def __init__(self, *args, **kwargs):
Runner.__init__(self, *args, **kwargs)
path = os.path.dirname(
os.path.dirname(sys.modules["metomi.rose"].__file__))
self.builtins_manager = SchemeHandlersManager([path], "rose.apps",
["run"], None, *args,
**kwargs)
self.date_time_oper = RoseDateTimeOperator()
def __init__(self, event_handler=None, popen=None):
if event_handler is None:
event_handler = Reporter()
self.event_handler = event_handler
if popen is None:
popen = RosePopener(self.event_handler)
self.popen = popen
path = os.path.dirname(
os.path.dirname(sys.modules["metomi.rosie"].__file__))
self.usertools_manager = SchemeHandlersManager([path],
"rosie.usertools",
["get_emails"])
def __init__(self, event_handler=None, popen=None, fs_util=None):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
path = os.path.dirname(
os.path.dirname(sys.modules["metomi.rose"].__file__))
SchemeHandlersManager.__init__(self, [path], "rose.config_processors",
["process"])
def __init__(self, event_handler=None, popen=None, fs_util=None):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
path = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__))
SchemeHandlersManager.__init__(
self, [path], ns="rose.loc_handlers", attrs=["parse", "pull"],
can_handle="can_pull")
def get_processor(
cls,
key=None,
event_handler=None,
popen=None,
fs_util=None,
host_selector=None,
):
"""Return a processor for the suite engine named by "key"."""
if cls.SCHEME_HANDLER_MANAGER is None:
path = os.path.dirname(
os.path.dirname(sys.modules["metomi.rose"].__file__))
cls.SCHEME_HANDLER_MANAGER = SchemeHandlersManager(
[path],
ns="rose.suite_engine_procs",
attrs=["SCHEME"],
can_handle=None,
event_handler=event_handler,
popen=popen,
fs_util=fs_util,
host_selector=host_selector,
)
if key is None:
key = cls.SCHEME_DEFAULT
x = cls.SCHEME_HANDLER_MANAGER.get_handler(key)
return x
def _run(self, dao, app_runner, config):
"""Transform and archive suite files.
This application is designed to work under "rose task-run" in a suite.
"""
compress_manager = SchemeHandlersManager(
[os.path.dirname(os.path.dirname(sys.modules["rose"].__file__))],
"rose.apps.rose_arch_compressions",
["compress_sources"],
None, app_runner)
# Set up the targets
s_key_tails = set()
targets = []
for t_key, t_node in sorted(config.value.items()):
if t_node.is_ignored() or ":" not in t_key:
continue
s_key_head, s_key_tail = t_key.split(":", 1)
if s_key_head != self.SECTION or not s_key_tail:
continue
# Determine target path.
s_key_tail = t_key.split(":", 1)[1]
try:
s_key_tail = env_var_process(s_key_tail)
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError([t_key, ""], "", exc)
# If parenthesised target is optional.
is_compulsory_target = True
if s_key_tail.startswith("(") and s_key_tail.endswith(")"):
s_key_tail = s_key_tail[1:-1]
is_compulsory_target = False
# Don't permit duplicate targets.
if s_key_tail in s_key_tails:
raise RoseArchDuplicateError([t_key], '', s_key_tail)
else:
s_key_tails.add(s_key_tail)
target = self._run_target_setup(
app_runner, compress_manager, config, t_key, s_key_tail,
t_node, is_compulsory_target)
old_target = dao.select(target.name)
if old_target is None or old_target != target:
dao.delete(target)
else:
target.status = target.ST_OLD
targets.append(target)
targets.sort(key=lambda target: target.name)
# Delete from database items that are no longer relevant
dao.delete_all(filter_targets=targets)
# Update the targets
for target in targets:
self._run_target_update(dao, app_runner, compress_manager, target)
return [target.status for target in targets].count(
RoseArchTarget.ST_BAD)
class AppRunner(Runner):
"""Invoke a Rose application."""
NAME = "app"
OPTIONS = [
"app_mode", "command_key", "conf_dir", "defines", "install_only_mode",
"new_mode", "no_overwrite_mode", "opt_conf_keys"
]
def __init__(self, *args, **kwargs):
Runner.__init__(self, *args, **kwargs)
path = os.path.dirname(
os.path.dirname(sys.modules["metomi.rose"].__file__))
self.builtins_manager = SchemeHandlersManager([path], "rose.apps",
["run"], None, *args,
**kwargs)
self.date_time_oper = RoseDateTimeOperator()
def run_impl(self, opts, args, uuid, work_files):
"""The actual logic for a run."""
# Preparation.
conf_tree = self.config_load(opts)
self._prep(conf_tree, opts)
self._poll(conf_tree)
# Run the application or the command.
app_mode = conf_tree.node.get_value(["mode"])
if app_mode is None:
app_mode = opts.app_mode
if app_mode is None:
app_mode = os.getenv("ROSE_APP_MODE")
if app_mode in [None, "command"]:
return self._command(conf_tree, opts, args)
else:
builtin_app = self.builtins_manager.get_handler(app_mode)
if builtin_app is None:
raise UnknownBuiltinAppError(app_mode)
return builtin_app.run(self, conf_tree, opts, args, uuid,
work_files)
def get_command(self, conf_tree, opts, args):
"""Get command to run."""
command = self.popen.list_to_shell_str(args)
if not command:
names = [
opts.command_key,
os.getenv("ROSE_APP_COMMAND_KEY"),
os.getenv("ROSE_TASK_NAME"), "default"
]
for name in names:
if not name:
continue
command = conf_tree.node.get_value(["command", name])
if command is not None:
break
return command
def _prep(self, conf_tree, opts):
"""Prepare to run the application."""
if opts.new_mode:
self._prep_new(opts)
# Dump the actual configuration as rose-app-run.conf
ConfigDumper()(conf_tree.node, "rose-app-run.conf")
# Environment variables: PATH
self._prep_path(conf_tree)
# Free format files not defined in the configuration file
file_section_prefix = self.config_pm.get_handler("file").PREFIX
for rel_path, conf_dir in conf_tree.files.items():
if not rel_path.startswith("file" + os.sep):
continue
name = rel_path[len("file" + os.sep):]
# No sub-directories, very slow otherwise
if os.sep in name:
name = name.split(os.sep, 1)[0]
target_key = file_section_prefix + name
target_node = conf_tree.node.get([target_key])
if target_node is None:
conf_tree.node.set([target_key])
target_node = conf_tree.node.get([target_key])
elif target_node.is_ignored():
continue
source_node = target_node.get(["source"])
if source_node is None:
target_node.set(["source"],
os.path.join(conf_dir, "file", name))
elif source_node.is_ignored():
continue
# Process Environment Variables
self.config_pm(conf_tree, "env")
# Process Files
self.config_pm(conf_tree,
"file",
no_overwrite_mode=opts.no_overwrite_mode)
def _prep_new(self, opts):
"""Clear out run directory on a --new option if possible."""
conf_dir = opts.conf_dir
if not conf_dir or os.path.abspath(conf_dir) == os.getcwd():
raise NewModeError(os.getcwd())
for path in os.listdir("."):
self.fs_util.delete(path)
@staticmethod
def _prep_path(conf_tree):
"""Add bin directories to the PATH seen by the app command."""
paths = []
for conf_dir in conf_tree.conf_dirs:
conf_bin_dir = os.path.join(conf_dir, "bin")
if os.path.isdir(conf_bin_dir):
paths.append(conf_bin_dir)
if paths:
value = os.pathsep.join(paths + [os.getenv("PATH")])
conf_tree.node.set(["env", "PATH"], value)
else:
conf_tree.node.set(["env", "PATH"], os.getenv("PATH"))
def _poll(self, conf_tree):
"""Run any configured file polling."""
poller = Poller(self.popen, self.handle_event)
poller.poll(conf_tree)
def _command(self, conf_tree, opts, args):
"""Run the command."""
command = self.get_command(conf_tree, opts, args)
if not command:
self.handle_event(CommandNotDefinedEvent())
return
if os.access("STDIN", os.F_OK | os.R_OK):
command += " <STDIN"
self.handle_event("command: %s" % command)
if opts.install_only_mode:
return
self.popen(command,
shell=True,
stdout=sys.stdout,
stderr=sys.stderr,
stdin=sys.stdin)
class RosieSvnPostCommitHook(RosieSvnHook):
"""A post-commit hook on a Rosie Subversion repository.
Update the Rosie discovery database on changes to info file.
Notify owner and users on access-list on trunk changes.
"""
KNOWN_KEYS_FILE_PATH = "R/O/S/I/E/trunk/rosie-keys"
REC_COPY_INFO = re.compile(r"\A\s+\(from\s(\S+):r(\d+)\)\s*\Z")
def __init__(self, event_handler=None, popen=None):
super(RosieSvnPostCommitHook, self).__init__(event_handler, popen)
self.usertools_manager = SchemeHandlersManager([self.path],
"rosie.usertools",
["get_emails"])
def run(self, repos, revision, no_notification=False):
"""Update database with changes in a changeset."""
# Lookup prefix of repos
# Do nothing if prefix is not registered
conf = ResourceLocator.default().get_conf()
metomi.rosie.db_node = conf.get(["rosie-db"], no_ignore=True)
for key, node in metomi.rosie.db_node.value.items():
if node.is_ignored() or not key.startswith("repos."):
continue
if os.path.realpath(repos) == os.path.realpath(node.value):
prefix = key[len("repos."):]
break
else:
return
# Locate Rosie DB of repos
dao = RosieWriteDAO(conf.get_value(["rosie-db", "db." + prefix]))
# Date-time of this commit
os.environ["TZ"] = "UTC"
date_time = self._svnlook("date", "-r", revision, repos)
date, dtime, _ = date_time.split(None, 2)
date = mktime(strptime(" ".join([date, dtime, "UTC"]), self.DATE_FMT))
# Detail of changes
changeset_attribs = {
"repos": repos,
"revision": revision,
"prefix": prefix,
"author": self._svnlook("author", "-r", revision, repos).strip(),
"date": date,
}
branch_attribs_dict = self._get_suite_branch_changes(repos, revision)
for key, branch_attribs in sorted(branch_attribs_dict.items()):
# Update known keys in suite info database meta table
if branch_attribs["has_changed_known_keys_file"]:
self._update_known_keys(dao, changeset_attribs)
# Update suite info database
self._update_info_db(dao, changeset_attribs, branch_attribs)
# Notification on trunk changes
# Notification on owner and access-list changes
if not no_notification and branch_attribs["branch"] == "trunk":
self._notify_trunk_changes(changeset_attribs, branch_attribs)
def _get_suite_branch_changes(self, repos, revision):
"""Retrieve changed statuses."""
branch_attribs_dict = {}
changed_lines = self._svnlook("changed", "--copy-info", "-r", revision,
repos).splitlines(True)
while changed_lines:
changed_line = changed_lines.pop(0)
# A normal status changed_line
# Column 1: content status
# Column 2: tree status
# Column 3: "+" sign denotes a copy history
# Column 5+: path
path = changed_line[4:].strip()
path_status = changed_line[0]
if path.endswith("/") and path_status == "_":
# Ignore property change on a directory
continue
# Path must be (under) a valid suite branch, including the special
# ROSIE suite
names = path.split("/", self.LEN_ID + 1)
if len(names) < self.LEN_ID + 1 or (
"".join(names[0:self.LEN_ID]) != "ROSIE" and any(
name not in id_chars
for name, id_chars in zip(names, self.ID_CHARS_LIST))):
continue
sid = "".join(names[0:self.LEN_ID])
branch = names[self.LEN_ID]
if branch:
# Change to a path in a suite branch
if (sid, branch) not in branch_attribs_dict:
branch_attribs_dict[(
sid,
branch)] = self._new_suite_branch_change(sid, branch)
branch_attribs = branch_attribs_dict[(sid, branch)]
try:
tail = names[self.LEN_ID + 1]
except IndexError:
tail = None
if tail == self.INFO_FILE:
# Suite info file change
if branch_attribs["info"] is None:
try:
branch_attribs["info"] = self._load_info(
repos,
sid,
branch,
revision=revision,
allow_popen_err=True,
)
except metomi.rose.config.ConfigSyntaxError as exc:
raise InfoFileError(InfoFileError.VALUE, exc)
if path_status != self.ST_ADDED:
branch_attribs["old_info"] = self._load_info(
repos,
sid,
branch,
revision=int(revision) - 1,
allow_popen_err=True,
)
if (branch_attribs["old_info"] !=
branch_attribs["info"]
and branch_attribs["status"] != self.ST_ADDED):
branch_attribs[
"status_info_file"] = self.ST_MODIFIED
elif tail:
# ROSIE meta known keys file change
if path == self.KNOWN_KEYS_FILE_PATH:
branch_attribs["has_changed_known_keys_file"] = True
if branch_attribs["status"] == self.ST_EMPTY:
branch_attribs["status"] = self.ST_MODIFIED
elif path_status in [self.ST_ADDED, self.ST_DELETED]:
# Branch add/delete
branch_attribs["status"] = path_status
# Load suite info and old info regardless
if branch_attribs["info"] is None:
try:
branch_attribs["info"] = self._load_info(
repos,
sid,
branch,
revision=revision,
allow_popen_err=True,
)
except metomi.rose.config.ConfigSyntaxError as exc:
raise InfoFileError(InfoFileError.VALUE, exc)
# Note: if (allowed) popen err, no DB entry will be created
if (branch_attribs["old_info"] is None
and branch_attribs["status"] == self.ST_DELETED):
branch_attribs["old_info"] = self._load_info(
repos,
sid,
branch,
revision=int(revision) - 1,
allow_popen_err=True,
)
# Append changed lines, so they can be used for notification
branch_attribs["changed_lines"].append(changed_line)
if changed_line[2] == "+":
changed_line_2 = changed_lines.pop(0)
branch_attribs["changed_lines"].append(changed_line_2)
if path_status != self.ST_ADDED or tail:
continue
# A line containing the copy info for a branch
# Column 5+ looks like: (from PATH:rREV)
match = self.REC_COPY_INFO.match(changed_line_2)
if match:
from_path, from_rev = match.groups()
branch_attribs_dict[(sid, branch)].update({
"from_path":
from_path,
"from_rev":
from_rev
})
elif path_status == self.ST_DELETED:
# The suite has been deleted
tree_out = self._svnlook("tree", "-r", str(int(revision) - 1),
"-N", repos, path)
# Include all branches of the suite in the deletion info
for tree_line in tree_out.splitlines()[1:]:
del_branch = tree_line.strip().rstrip("/")
branch_attribs_dict[(
sid, del_branch)] = self._new_suite_branch_change(
sid,
del_branch,
{
"old_info":
self._load_info(
repos,
sid,
del_branch,
revision=int(revision) - 1,
allow_popen_err=True,
),
"status":
self.ST_DELETED,
"status_info_file":
self.ST_EMPTY,
"changed_lines":
["D %s/%s/" % (path, del_branch)],
},
)
return branch_attribs_dict
def _new_suite_branch_change(self, sid, branch, attribs=None):
"""Return a dict to represent a suite branch change."""
branch_attribs = {
"sid": sid,
"branch": branch,
"from_path": None,
"from_rev": None,
"has_changed_known_keys_file": False,
"old_info": None,
"info": None,
"status": self.ST_EMPTY,
"status_info_file": self.ST_EMPTY,
"changed_lines": [],
}
if attribs:
branch_attribs.update(attribs)
return branch_attribs
def _notify_trunk_changes(self, changeset_attribs, branch_attribs):
"""Email owner and/or access-list users on changes to trunk."""
# Notify only if users' email addresses can be determined
conf = ResourceLocator.default().get_conf()
user_tool_name = conf.get_value(["rosa-svn", "user-tool"])
if not user_tool_name:
return
notify_who_str = conf.get_value(
["rosa-svn", "notify-who-on-trunk-commit"], "")
if not notify_who_str.strip():
return
notify_who = shlex.split(notify_who_str)
# Build the message text
info_file_path = "%s/trunk/%s" % (
"/".join(branch_attribs["sid"]),
self.INFO_FILE,
)
text = ""
for changed_line in branch_attribs["changed_lines"]:
text += changed_line
# For suite info file change, add diff as well
if (changed_line[4:].strip() == info_file_path and
branch_attribs["status_info_file"] == self.ST_MODIFIED):
old_strio = StringIO()
metomi.rose.config.dump(branch_attribs["old_info"], old_strio)
new_strio = StringIO()
metomi.rose.config.dump(branch_attribs["info"], new_strio)
for diff_line in unified_diff(
old_strio.getvalue().splitlines(True),
new_strio.getvalue().splitlines(True),
"@%d" % (int(changeset_attribs["revision"]) - 1),
"@%d" % (int(changeset_attribs["revision"])),
):
text += " " * 4 + diff_line
# Determine who to notify
users = set()
for key in ["old_info", "info"]:
if branch_attribs[key] is not None:
info_conf = branch_attribs[key]
if "owner" in notify_who:
users.add(info_conf.get_value(["owner"]))
if "access-list" in notify_who:
users.update(
info_conf.get_value(["access-list"], "").split())
users.discard("*")
# Determine email addresses
user_tool = self.usertools_manager.get_handler(user_tool_name)
if "author" in notify_who:
users.add(changeset_attribs["author"])
else:
users.discard(changeset_attribs["author"])
emails = sorted(user_tool.get_emails(users))
if not emails:
return
# Send notification
msg = MIMEText(text)
msg.set_charset("utf-8")
msg["From"] = conf.get_value(
["rosa-svn", "notification-from"],
"notications@" + socket.getfqdn(),
)
msg["To"] = ", ".join(emails)
msg["Subject"] = "%s-%s/trunk@%d" % (
changeset_attribs["prefix"],
branch_attribs["sid"],
int(changeset_attribs["revision"]),
)
smtp_host = conf.get_value(["rosa-svn", "smtp-host"],
default="localhost")
smtp = SMTP(smtp_host)
smtp.sendmail(msg["From"], emails, msg.as_string())
smtp.quit()
def _update_info_db(self, dao, changeset_attribs, branch_attribs):
"""Update the suite info database for a suite branch."""
idx = "{0}-{1}".format(changeset_attribs["prefix"],
branch_attribs["sid"])
vc_attrs = {
"idx": idx,
"branch": branch_attribs["branch"],
"revision": changeset_attribs["revision"],
}
# Latest table
try:
dao.delete(
LATEST_TABLE_NAME,
idx=vc_attrs["idx"],
branch=vc_attrs["branch"],
)
except al.exc.IntegrityError:
# idx and branch were just added: there is no previous record.
pass
if branch_attribs["status"] != self.ST_DELETED:
dao.insert(LATEST_TABLE_NAME, **vc_attrs)
# N.B. deleted suite branch only has old info
info_key = "info"
if branch_attribs["status"] == self.ST_DELETED:
info_key = "old_info"
if branch_attribs[info_key] is None:
return
# Main table
cols = dict(vc_attrs)
cols.update({
"author": changeset_attribs["author"],
"date": changeset_attribs["date"],
})
for name in ["owner", "project", "title"]:
cols[name] = branch_attribs[info_key].get_value([name], "null")
if branch_attribs["from_path"] and vc_attrs["branch"] == "trunk":
from_names = branch_attribs["from_path"].split("/")[:self.LEN_ID]
cols["from_idx"] = "{0}-{1}".format(changeset_attribs["prefix"],
"".join(from_names))
cols["status"] = (branch_attribs["status"] +
branch_attribs["status_info_file"])
dao.insert(MAIN_TABLE_NAME, **cols)
# Optional table
for name in branch_attribs[info_key].value:
if name in ["owner", "project", "title"]:
continue
value = branch_attribs[info_key].get_value([name])
if value is None: # setting may have ignore flag (!)
continue
cols = dict(vc_attrs)
cols.update({"name": name, "value": value})
dao.insert(OPTIONAL_TABLE_NAME, **cols)
def _update_known_keys(self, dao, changeset_attribs):
"""Update the known_keys in the meta table."""
repos = changeset_attribs["repos"]
revision = changeset_attribs["revision"]
keys_str = self._svnlook("cat", "-r", revision, repos,
self.KNOWN_KEYS_FILE_PATH)
keys_str = " ".join(shlex.split(keys_str))
if keys_str:
try:
dao.insert(META_TABLE_NAME, name="known_keys", value=keys_str)
except al.exc.IntegrityError:
dao.update(
META_TABLE_NAME,
("name", ),
name="known_keys",
value=keys_str,
)
def __init__(self, event_handler=None, popen=None):
super(RosieSvnPostCommitHook, self).__init__(event_handler, popen)
self.usertools_manager = SchemeHandlersManager([self.path],
"rosie.usertools",
["get_emails"])
class RosieSvnPreCommitHook(object):
"""A pre-commit hook on a Rosie Subversion repository.
Ensure that commits conform to the rules of Rosie.
"""
IGNORES = "svnperms.conf"
RE_ID_NAMES = [r"[Ra-z]", r"[Oa-z]", r"[S\d]", r"[I\d]", r"[E\d]"]
LEN_ID = len(RE_ID_NAMES)
ST_ADD = "A"
ST_DELETE = "D"
ST_UPDATE = "U"
TRUNK_INFO_FILE = "trunk/rose-suite.info"
TRUNK_KNOWN_KEYS_FILE = "trunk/rosie-keys"
def __init__(self, event_handler=None, popen=None):
if event_handler is None:
event_handler = Reporter()
self.event_handler = event_handler
if popen is None:
popen = RosePopener(self.event_handler)
self.popen = popen
path = os.path.dirname(
os.path.dirname(sys.modules["metomi.rosie"].__file__))
self.usertools_manager = SchemeHandlersManager([path],
"rosie.usertools",
["verify_users"])
def _get_access_info(self, info_node):
"""Return (owner, access_list) from "info_node"."""
owner = info_node.get_value(["owner"])
access_list = info_node.get_value(["access-list"], "").split()
access_list.sort()
return owner, access_list
def _get_info(self, repos, path_head, txn=None):
"""Return a ConfigNode for the "rose-suite.info" of a suite.
The suite is located under path_head.
"""
opt_txn = []
if txn is not None:
opt_txn = ["-t", txn]
t_handle = tempfile.TemporaryFile()
path = path_head + self.TRUNK_INFO_FILE
t_handle.write(self._svnlook("cat", repos, path, *opt_txn).encode())
t_handle.seek(0)
info_node = ConfigLoader()(t_handle)
t_handle.close()
return info_node
def _svnlook(self, *args):
"""Return the standard output from "svnlook"."""
command = ["svnlook"] + list(args)
data = self.popen(*command, stderr=sys.stderr)[0]
return data.decode()
def _verify_users(self, status, path, txn_owner, txn_access_list,
bad_changes):
"""Check txn_owner and txn_access_list.
For any invalid users, append to bad_changes and return True.
"""
# The owner and names in access list must be real users
conf = ResourceLocator.default().get_conf()
user_tool_name = conf.get_value(["rosa-svn", "user-tool"])
if not user_tool_name:
return False
user_tool = self.usertools_manager.get_handler(user_tool_name)
txn_users = set([txn_owner] + txn_access_list)
txn_users.discard("*")
bad_users = user_tool.verify_users(txn_users)
for bad_user in bad_users:
if txn_owner == bad_user:
bad_change = BadChange(status, path, BadChange.USER,
"owner=" + bad_user)
bad_changes.append(bad_change)
if bad_user in txn_access_list:
bad_change = BadChange(status, path, BadChange.USER,
"access-list=" + bad_user)
bad_changes.append(bad_change)
return bool(bad_users)
def run(self, repos, txn):
"""Apply the rule engine on transaction "txn" to repository "repos"."""
changes = set() # set([(status, path), ...])
for line in self._svnlook("changed", "-t", txn, repos).splitlines():
status, path = line.split(None, 1)
changes.add((status, path))
bad_changes = []
author = None
super_users = None
rev_info_map = {} # {path-id: (owner, access-list), ...}
txn_info_map = {}
conf = ResourceLocator.default().get_conf()
ignores_str = conf.get_value(["rosa-svn", "ignores"], self.IGNORES)
ignores = shlex.split(ignores_str)
for status, path in sorted(changes):
if any(fnmatch(path, ignore) for ignore in ignores):
continue
names = path.split("/", self.LEN_ID + 1)
tail = None
if not names[-1]:
tail = names.pop()
# Directories above the suites must match the ID patterns
is_bad = False
for name, pattern in zip(names, self.RE_ID_NAMES):
if not re.compile(r"\A" + pattern + r"\Z").match(name):
is_bad = True
break
if is_bad:
bad_changes.append(BadChange(status, path))
continue
# Can only add directories at levels above the suites
if len(names) < self.LEN_ID:
if status[0] != self.ST_ADD:
bad_changes.append(BadChange(status, path))
continue
else:
is_meta_suite = "".join(names[0:self.LEN_ID]) == "ROSIE"
# A file at the branch level
if len(names) == self.LEN_ID + 1 and tail is None:
bad_changes.append(BadChange(status, path))
continue
# No need to check non-trunk changes
if len(names) > self.LEN_ID and names[self.LEN_ID] != "trunk":
continue
# New suite should have an info file
if status[0] == self.ST_ADD and len(names) == self.LEN_ID:
if (self.ST_ADD, path + "trunk/") not in changes:
bad_changes.append(
BadChange(status, path, BadChange.NO_TRUNK))
continue
path_trunk_info_file = path + self.TRUNK_INFO_FILE
if ((self.ST_ADD, path_trunk_info_file) not in changes and
(self.ST_UPDATE, path_trunk_info_file) not in changes):
bad_changes.append(
BadChange(status, path, BadChange.NO_INFO))
continue
# The rest are trunk changes in a suite
path_head = "/".join(names[0:self.LEN_ID]) + "/"
path_tail = path[len(path_head):]
# For meta suite, make sure keys in keys file can be parsed
if is_meta_suite and path_tail == self.TRUNK_KNOWN_KEYS_FILE:
out = self._svnlook("cat", "-t", txn, repos, path)
try:
shlex.split(out)
except ValueError:
bad_changes.append(BadChange(status, path,
BadChange.VALUE))
continue
# Suite trunk information file must have an owner
# User IDs of owner and access list must be real
if (status not in self.ST_DELETE
and path_tail == self.TRUNK_INFO_FILE):
if path_head not in txn_info_map:
txn_info_map[path_head] = self._get_info(
repos, path_head, txn)
txn_owner, txn_access_list = self._get_access_info(
txn_info_map[path_head])
if not txn_owner:
bad_changes.append(
BadChange(status, path, BadChange.NO_OWNER))
continue
if self._verify_users(status, path, txn_owner, txn_access_list,
bad_changes):
continue
reports = DefaultValidators().validate(
txn_info_map[path_head],
load_meta_config(txn_info_map[path_head],
config_type=metomi.rose.INFO_CONFIG_NAME))
if reports:
reports_str = get_reports_as_text({None: reports}, path)
bad_changes.append(
BadChange(status, path, BadChange.VALUE, reports_str))
continue
# Can only remove trunk information file with suite
if status == self.ST_DELETE and path_tail == self.TRUNK_INFO_FILE:
if (self.ST_DELETE, path_head) not in changes:
bad_changes.append(
BadChange(status, path, BadChange.NO_INFO))
continue
# Can only remove trunk with suite
if status == self.ST_DELETE and path_tail == "trunk/":
if (self.ST_DELETE, path_head) not in changes:
bad_changes.append(
BadChange(status, path, BadChange.NO_TRUNK))
continue
# New suite trunk: ignore the rest
if (self.ST_ADD, path_head + "trunk/") in changes:
continue
# See whether author has permission to make changes
if author is None:
author = self._svnlook("author", "-t", txn, repos).strip()
if super_users is None:
super_users = []
for s_key in ["rosa-svn", "rosa-svn-pre-commit"]:
value = conf.get_value([s_key, "super-users"])
if value is not None:
super_users = shlex.split(value)
break
if path_head not in rev_info_map:
rev_info_map[path_head] = self._get_info(repos, path_head)
owner, access_list = self._get_access_info(rev_info_map[path_head])
admin_users = super_users + [owner]
# Only admin users can remove the suite
if author not in admin_users and not path_tail:
bad_changes.append(BadChange(status, path))
continue
# Admin users and those in access list can modify everything in
# trunk apart from specific entries in the trunk info file
if "*" in access_list or author in admin_users + access_list:
if path_tail != self.TRUNK_INFO_FILE:
continue
else:
bad_changes.append(BadChange(status, path))
continue
# The owner must not be deleted
if path_head not in txn_info_map:
txn_info_map[path_head] = self._get_info(repos, path_head, txn)
txn_owner, txn_access_list = self._get_access_info(
txn_info_map[path_head])
if not txn_owner:
bad_changes.append(BadChange(status, path, BadChange.NO_OWNER))
continue
# Only the admin users can change owner and access list
if owner == txn_owner and access_list == txn_access_list:
continue
if author not in admin_users:
if owner != txn_owner:
bad_changes.append(
BadChange(status, path, BadChange.PERM,
"owner=" + txn_owner))
else: # access list
bad_change = BadChange(
status, path, BadChange.PERM,
"access-list=" + " ".join(txn_access_list))
bad_changes.append(bad_change)
continue
if bad_changes:
raise BadChanges(bad_changes)
__call__ = run
class RosieSvnPreCommitHook(RosieSvnHook):
"""A pre-commit hook on a Rosie Subversion repository.
Ensure that commits conform to the rules of Rosie.
"""
IGNORES = "svnperms.conf"
RE_ID_NAMES = [r"[Ra-z]", r"[Oa-z]", r"[S\d]", r"[I\d]", r"[E\d]"]
TRUNK_KNOWN_KEYS_FILE = "trunk/rosie-keys"
def __init__(self, event_handler=None, popen=None):
super(RosieSvnPreCommitHook, self).__init__(event_handler, popen)
self.usertools_manager = SchemeHandlersManager(
[self.path], "rosie.usertools", ["verify_users"]
)
def _get_access_info(self, info_node):
"""Return (owner, access_list) from "info_node"."""
owner = info_node.get_value(["owner"])
access_list = info_node.get_value(["access-list"], "").split()
access_list.sort()
return owner, access_list
def _verify_users(
self, status, path, txn_owner, txn_access_list, bad_changes
):
"""Check txn_owner and txn_access_list.
For any invalid users, append to bad_changes and return True.
"""
# The owner and names in access list must be real users
conf = ResourceLocator.default().get_conf()
user_tool_name = conf.get_value(["rosa-svn", "user-tool"])
if not user_tool_name:
return False
user_tool = self.usertools_manager.get_handler(user_tool_name)
txn_users = set([txn_owner] + txn_access_list)
txn_users.discard("*")
bad_users = user_tool.verify_users(txn_users)
for bad_user in bad_users:
if txn_owner == bad_user:
bad_change = BadChange(
status, path, BadChange.USER, "owner=" + bad_user
)
bad_changes.append(bad_change)
if bad_user in txn_access_list:
bad_change = BadChange(
status, path, BadChange.USER, "access-list=" + bad_user
)
bad_changes.append(bad_change)
return bool(bad_users)
def run(self, repos, txn):
"""Apply the rule engine on transaction "txn" to repository "repos"."""
changes = set() # set([(status, path), ...])
for line in self._svnlook("changed", "-t", txn, repos).splitlines():
status, path = line.split(None, 1)
changes.add((status, path))
bad_changes = []
author = None
super_users = None
rev_info_map = {}
txn_info_map = {}
conf = ResourceLocator.default().get_conf()
ignores_str = conf.get_value(["rosa-svn", "ignores"], self.IGNORES)
ignores = shlex.split(ignores_str)
for status, path in sorted(changes):
if any(fnmatch(path, ignore) for ignore in ignores):
continue
names = path.split("/", self.LEN_ID + 1)
tail = None
if not names[-1]:
tail = names.pop()
# Directories above the suites must match the ID patterns
is_bad = False
for name, pattern in zip(names, self.RE_ID_NAMES):
if not re.compile(r"\A" + pattern + r"\Z").match(name):
is_bad = True
break
if is_bad:
msg = "Directories above the suites must match the ID patterns"
bad_changes.append(BadChange(status, path, content=msg))
continue
# At levels above the suites, can only add directories
if len(names) < self.LEN_ID:
if status[0] != self.ST_ADDED:
msg = (
"At levels above the suites, "
"can only add directories"
)
bad_changes.append(BadChange(status, path, content=msg))
continue
# Cannot have a file at the branch level
if len(names) == self.LEN_ID + 1 and tail is None:
msg = "Cannot have a file at the branch level"
bad_changes.append(BadChange(status, path, content=msg))
continue
# New suite should have an info file
if len(names) == self.LEN_ID and status == self.ST_ADDED:
if (self.ST_ADDED, path + "trunk/") not in changes:
bad_changes.append(
BadChange(status, path, BadChange.NO_TRUNK)
)
continue
path_trunk_info_file = path + self.TRUNK_INFO_FILE
if (self.ST_ADDED, path_trunk_info_file) not in changes and (
self.ST_UPDATED,
path_trunk_info_file,
) not in changes:
bad_changes.append(
BadChange(status, path, BadChange.NO_INFO)
)
continue
sid = "".join(names[0 : self.LEN_ID])
branch = names[self.LEN_ID] if len(names) > self.LEN_ID else None
path_head = "/".join(sid) + "/"
path_tail = path[len(path_head) :]
is_meta_suite = sid == "ROSIE"
if status != self.ST_DELETED:
# Check info file
if sid not in txn_info_map:
try:
txn_info_map[sid] = self._load_info(
repos, sid, branch=branch, transaction=txn
)
err = None
except ConfigSyntaxError as exc:
err = InfoFileError(InfoFileError.VALUE, exc)
except RosePopenError as exc:
err = InfoFileError(InfoFileError.NO_INFO, exc.stderr)
if err:
bad_changes.append(err)
txn_info_map[sid] = err
continue
# Suite must have an owner
txn_owner, txn_access_list = self._get_access_info(
txn_info_map[sid]
)
if not txn_owner:
bad_changes.append(
InfoFileError(InfoFileError.NO_OWNER)
)
continue
# No need to check other non-trunk changes
if branch and branch != "trunk":
continue
# For meta suite, make sure keys in keys file can be parsed
if is_meta_suite and path_tail == self.TRUNK_KNOWN_KEYS_FILE:
out = self._svnlook("cat", "-t", txn, repos, path)
try:
shlex.split(out)
except ValueError:
bad_changes.append(
BadChange(status, path, BadChange.VALUE)
)
continue
# User IDs of owner and access list must be real
if (
status != self.ST_DELETED
and path_tail == self.TRUNK_INFO_FILE
and not isinstance(txn_info_map[sid], InfoFileError)
):
txn_owner, txn_access_list = self._get_access_info(
txn_info_map[sid]
)
if self._verify_users(
status, path, txn_owner, txn_access_list, bad_changes
):
continue
reports = DefaultValidators().validate(
txn_info_map[sid],
load_meta_config(
txn_info_map[sid],
config_type=metomi.rose.INFO_CONFIG_NAME,
),
)
if reports:
reports_str = get_reports_as_text({None: reports}, path)
bad_changes.append(
BadChange(status, path, BadChange.VALUE, reports_str)
)
continue
# Can only remove trunk information file with suite
if status == self.ST_DELETED and path_tail == self.TRUNK_INFO_FILE:
if (self.ST_DELETED, path_head) not in changes:
bad_changes.append(
BadChange(status, path, BadChange.NO_INFO)
)
continue
# Can only remove trunk with suite
# (Don't allow replacing trunk with a copy from elsewhere, either)
if status == self.ST_DELETED and path_tail == "trunk/":
if (self.ST_DELETED, path_head) not in changes:
bad_changes.append(
BadChange(status, path, BadChange.NO_TRUNK)
)
continue
# New suite trunk: ignore the rest
if (self.ST_ADDED, path_head + "trunk/") in changes:
continue
# See whether author has permission to make changes
if author is None:
author = self._svnlook("author", "-t", txn, repos).strip()
if super_users is None:
super_users = []
for s_key in ["rosa-svn", "rosa-svn-pre-commit"]:
value = conf.get_value([s_key, "super-users"])
if value is not None:
super_users = shlex.split(value)
break
if sid not in rev_info_map:
rev_info_map[sid] = self._load_info(repos, sid, branch=branch)
owner, access_list = self._get_access_info(rev_info_map[sid])
admin_users = super_users + [owner]
# Only admin users can remove the suite
if author not in admin_users and not path_tail:
msg = "Only the suite owner can remove the suite"
bad_changes.append(BadChange(status, path, content=msg))
continue
# Admin users and those in access list can modify everything in
# trunk apart from specific entries in the trunk info file
if "*" in access_list or author in admin_users + access_list:
if path_tail != self.TRUNK_INFO_FILE:
continue
else:
msg = "User not in access list"
bad_changes.append(BadChange(status, path, content=msg))
continue
# Only the admin users can change owner and access list
if owner == txn_owner and access_list == txn_access_list:
continue
if author not in admin_users:
if owner != txn_owner:
bad_changes.append(
BadChange(
status, path, BadChange.PERM, "owner=" + txn_owner
)
)
else: # access list
bad_change = BadChange(
status,
path,
BadChange.PERM,
"access-list=" + " ".join(txn_access_list),
)
bad_changes.append(bad_change)
continue
if bad_changes:
raise BadChanges(bad_changes)
__call__ = run