class RosieSvnPostCommitHook(object):
"""A post-commit hook on a Rosie Subversion repository.
Update the Rosie discovery database on changes to info file.
Notify owner and users on access-list on trunk changes.
"""
DATE_FMT = "%Y-%m-%d %H:%M:%S %Z"
ID_CHARS_LIST = ["abcdefghijklmnopqrstuvwxyz"] * 2 + ["0123456789"] * 3
LEN_ID = len(ID_CHARS_LIST)
INFO_FILE = "rose-suite.info"
KNOWN_KEYS_FILE_PATH = "R/O/S/I/E/trunk/rosie-keys"
REC_COPY_INFO = re.compile(r"\A\s+\(from\s(\S+):r(\d+)\)\s*\Z")
ST_ADDED = "A"
ST_DELETED = "D"
ST_MODIFIED = "M"
ST_EMPTY = " "
TRUNK = "trunk"
def __init__(self, event_handler=None, popen=None):
if event_handler is None:
event_handler = Reporter()
self.event_handler = event_handler
if popen is None:
popen = RosePopener(self.event_handler)
self.popen = popen
path = os.path.dirname(os.path.dirname(sys.modules["rosie"].__file__))
self.usertools_manager = SchemeHandlersManager(
[path], "rosie.usertools", ["get_emails"])
def run(self, repos, revision, no_notification=False):
"""Update database with changes in a changeset."""
# Lookup prefix of repos
# Do nothing if prefix is not registered
conf = ResourceLocator.default().get_conf()
rosie_db_node = conf.get(["rosie-db"], no_ignore=True)
for key, node in rosie_db_node.value.items():
if node.is_ignored() or not key.startswith("repos."):
continue
if os.path.realpath(repos) == os.path.realpath(node.value):
prefix = key[len("repos."):]
break
else:
return
# Locate Rosie DB of repos
dao = RosieWriteDAO(conf.get_value(["rosie-db", "db." + prefix]))
# Date-time of this commit
os.environ["TZ"] = "UTC"
date_time_str = self._svnlook("date", "-r", revision, repos)
date, dtime, _ = date_time_str.split(None, 2)
date = mktime(strptime(" ".join([date, dtime, "UTC"]), self.DATE_FMT))
# Detail of changes
changeset_attribs = {
"repos": repos,
"revision": revision,
"prefix": prefix,
"author": self._svnlook("author", "-r", revision, repos).strip(),
"date": date}
branch_attribs_dict = self._get_suite_branch_changes(repos, revision)
for key, branch_attribs in sorted(branch_attribs_dict.items()):
# Update known keys in suite info database meta table
if branch_attribs["has_changed_known_keys_file"]:
self._update_known_keys(dao, changeset_attribs)
# Update suite info database
self._update_info_db(dao, changeset_attribs, branch_attribs)
# Notification on trunk changes
# Notification on owner and access-list changes
if not no_notification and branch_attribs["branch"] == "trunk":
self._notify_trunk_changes(
changeset_attribs, branch_attribs)
def _get_suite_branch_changes(self, repos, revision):
"""Retrieve changed statuses."""
branch_attribs_dict = {}
changed_lines = self._svnlook(
"changed", "--copy-info", "-r", revision, repos).splitlines(True)
while changed_lines:
changed_line = changed_lines.pop(0)
# A normal status changed_line
# Column 1: content status
# Column 2: tree status
# Column 3: "+" sign denotes a copy history
# Column 5+: path
path = changed_line[4:].strip()
path_status = changed_line[0]
if path.endswith("/") and path_status == "_":
# Ignore property change on a directory
continue
# Path must be (under) a valid suite branch, including the special
# ROSIE suite
names = path.split("/", self.LEN_ID + 1)
if (len(names) < self.LEN_ID + 1 or (
"".join(names[0:self.LEN_ID]) != "ROSIE" and
any(name not in id_chars for name, id_chars in
zip(names, self.ID_CHARS_LIST)))):
continue
sid = "".join(names[0:self.LEN_ID])
branch = names[self.LEN_ID]
if branch:
# Change to a path in a suite branch
if (sid, branch) not in branch_attribs_dict:
branch_attribs_dict[(sid, branch)] = (
self._new_suite_branch_change(sid, branch))
branch_attribs = branch_attribs_dict[(sid, branch)]
try:
tail = names[self.LEN_ID + 1]
except IndexError:
tail = None
if tail == self.INFO_FILE:
# Suite info file change
if branch_attribs["info"] is None:
branch_attribs["info"] = self._load_info(
repos, revision, sid, branch)
if path_status != self.ST_ADDED:
branch_attribs["old_info"] = self._load_info(
repos, int(revision) - 1, sid, branch)
if (branch_attribs["old_info"] !=
branch_attribs["info"] and
branch_attribs["status"] != self.ST_ADDED):
branch_attribs["status_info_file"] = (
self.ST_MODIFIED)
elif tail:
# ROSIE meta known keys file change
if path == self.KNOWN_KEYS_FILE_PATH:
branch_attribs["has_changed_known_keys_file"] = True
if branch_attribs["status"] == self.ST_EMPTY:
branch_attribs["status"] = self.ST_MODIFIED
elif path_status in [self.ST_ADDED, self.ST_DELETED]:
# Branch add/delete
branch_attribs["status"] = path_status
# Load suite info and old info regardless
if branch_attribs["info"] is None:
branch_attribs["info"] = self._load_info(
repos, revision, sid, branch)
if (branch_attribs["old_info"] is None and
branch_attribs["status"] == self.ST_DELETED):
branch_attribs["old_info"] = self._load_info(
repos, int(revision) - 1, sid, branch)
# Append changed lines, so they can be used for notification
branch_attribs["changed_lines"].append(changed_line)
if changed_line[2] == "+":
changed_line_2 = changed_lines.pop(0)
branch_attribs["changed_lines"].append(changed_line_2)
if path_status != self.ST_ADDED or tail:
continue
# A line containing the copy info for a branch
# Column 5+ looks like: (from PATH:rREV)
match = self.REC_COPY_INFO.match(changed_line_2)
if match:
from_path, from_rev = match.groups()
branch_attribs_dict[(sid, branch)].update({
"from_path": from_path, "from_rev": from_rev})
elif path_status == self.ST_DELETED:
# The suite has been deleted
tree_out = self._svnlook(
"tree", "-r", str(int(revision) - 1), "-N", repos, path)
# Include all branches of the suite in the deletion info
for tree_line in tree_out.splitlines()[1:]:
del_branch = tree_line.strip().rstrip("/")
branch_attribs_dict[(sid, del_branch)] = (
self._new_suite_branch_change(sid, del_branch, {
"old_info": self._load_info(
repos, int(revision) - 1, sid, del_branch),
"status": self.ST_DELETED,
"status_info_file": self.ST_EMPTY,
"changed_lines": [
"D %s/%s/" % (path, del_branch)]}))
return branch_attribs_dict
def _load_info(self, repos, revision, sid, branch):
"""Load info file from branch_path in repos @revision."""
info_file_path = "%s/%s/%s" % ("/".join(sid), branch, self.INFO_FILE)
t_handle = TemporaryFile()
try:
t_handle.write(self._svnlook(
"cat", "-r", str(revision), repos, info_file_path))
except RosePopenError:
return None
t_handle.seek(0)
config = rose.config.load(t_handle)
t_handle.close()
return config
def _new_suite_branch_change(self, sid, branch, attribs=None):
"""Return a dict to represent a suite branch change."""
branch_attribs = {
"sid": sid,
"branch": branch,
"from_path": None,
"from_rev": None,
"has_changed_known_keys_file": False,
"old_info": None,
"info": None,
"status": self.ST_EMPTY,
"status_info_file": self.ST_EMPTY,
"changed_lines": []}
if attribs:
branch_attribs.update(attribs)
return branch_attribs
def _notify_trunk_changes(self, changeset_attribs, branch_attribs):
"""Email owner and/or access-list users on changes to trunk."""
# Notify only if users' email addresses can be determined
conf = ResourceLocator.default().get_conf()
user_tool_name = conf.get_value(["rosa-svn", "user-tool"])
if not user_tool_name:
return
notify_who_str = conf.get_value(
["rosa-svn", "notify-who-on-trunk-commit"], "")
if not notify_who_str.strip():
return
notify_who = shlex.split(notify_who_str)
# Build the message text
info_file_path = "%s/trunk/%s" % (
"/".join(branch_attribs["sid"]), self.INFO_FILE)
text = ""
for changed_line in branch_attribs["changed_lines"]:
text += changed_line
# For suite info file change, add diff as well
if (changed_line[4:].strip() == info_file_path and
branch_attribs["status_info_file"] == self.ST_MODIFIED):
old_strio = StringIO()
rose.config.dump(branch_attribs["old_info"], old_strio)
new_strio = StringIO()
rose.config.dump(branch_attribs["info"], new_strio)
for diff_line in unified_diff(
old_strio.getvalue().splitlines(True),
new_strio.getvalue().splitlines(True),
"@%d" % (int(changeset_attribs["revision"]) - 1),
"@%d" % (int(changeset_attribs["revision"]))):
text += " " * 4 + diff_line
# Determine who to notify
users = set()
for key in ["old_info", "info"]:
if branch_attribs[key] is not None:
info_conf = branch_attribs[key]
if "owner" in notify_who:
users.add(info_conf.get_value(["owner"]))
if "access-list" in notify_who:
users.update(
info_conf.get_value(["access-list"], "").split())
users.discard("*")
# Determine email addresses
user_tool = self.usertools_manager.get_handler(user_tool_name)
if "author" in notify_who:
users.add(changeset_attribs["author"])
else:
users.discard(changeset_attribs["author"])
emails = sorted(user_tool.get_emails(users))
if not emails:
return
# Send notification
msg = MIMEText(text)
msg.set_charset("utf-8")
msg["From"] = conf.get_value(
["rosa-svn", "notification-from"],
"notications@" + socket.getfqdn())
msg["To"] = ", ".join(emails)
msg["Subject"] = "%s-%s/trunk@%d" % (
changeset_attribs["prefix"],
branch_attribs["sid"],
int(changeset_attribs["revision"]))
smtp_host = conf.get_value(
["rosa-svn", "smtp-host"], default="localhost")
smtp = SMTP(smtp_host)
smtp.sendmail(msg["From"], emails, msg.as_string())
smtp.quit()
def _svnlook(self, *args):
"""Return the standard output from "svnlook"."""
command = ["svnlook"] + list(args)
return self.popen(*command)[0]
def _update_info_db(self, dao, changeset_attribs, branch_attribs):
"""Update the suite info database for a suite branch."""
idx = changeset_attribs["prefix"] + "-" + branch_attribs["sid"]
vc_attrs = {
"idx": idx,
"branch": branch_attribs["branch"],
"revision": changeset_attribs["revision"]}
for key in vc_attrs:
vc_attrs[key] = vc_attrs[key].decode("utf-8")
# Latest table
try:
dao.delete(
LATEST_TABLE_NAME,
idx=vc_attrs["idx"], branch=vc_attrs["branch"])
except al.exc.IntegrityError:
# idx and branch were just added: there is no previous record.
pass
if branch_attribs["status"] != self.ST_DELETED:
dao.insert(LATEST_TABLE_NAME, **vc_attrs)
# N.B. deleted suite branch only has old info
info_key = "info"
if branch_attribs["status"] == self.ST_DELETED:
info_key = "old_info"
if branch_attribs[info_key] is None:
return
# Main table
cols = dict(vc_attrs)
cols.update({
"author": changeset_attribs["author"],
"date": changeset_attribs["date"]})
for name in ["owner", "project", "title"]:
cols[name] = branch_attribs[info_key].get_value([name])
if branch_attribs["from_path"] and vc_attrs["branch"] == u"trunk":
from_names = branch_attribs["from_path"].split("/")[:self.LEN_ID]
cols["from_idx"] = (
changeset_attribs["prefix"] + "-" + "".join(from_names))
cols["status"] = (
branch_attribs["status"] + branch_attribs["status_info_file"])
for key in cols:
try:
cols[key] = cols[key].decode("utf-8")
except AttributeError:
pass
dao.insert(MAIN_TABLE_NAME, **cols)
# Optional table
for name in branch_attribs[info_key].value:
if name in ["owner", "project", "title"]:
continue
value = branch_attribs[info_key].get_value([name])
if value is None: # setting may have ignore flag (!)
continue
cols = dict(vc_attrs)
cols.update({
"name": name.decode("utf-8"), "value": value.decode("utf-8")})
dao.insert(OPTIONAL_TABLE_NAME, **cols)
def _update_known_keys(self, dao, changeset_attribs):
"""Update the known_keys in the meta table."""
repos = changeset_attribs["repos"]
revision = changeset_attribs["revision"]
keys_str = self._svnlook(
"cat", "-r", revision, repos, self.KNOWN_KEYS_FILE_PATH)
keys_str = " ".join(shlex.split(keys_str)).decode("utf-8")
if keys_str:
try:
dao.insert(META_TABLE_NAME, name=u"known_keys", value=keys_str)
except al.exc.IntegrityError:
dao.update(
META_TABLE_NAME, (u"name",),
name=u"known_keys", value=keys_str)
def _run(self, dao, app_runner, config):
"""Transform and archive suite files.
This application is designed to work under "rose task-run" in a suite.
"""
path = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__))
compress_manager = SchemeHandlersManager(
[path], "rose.apps.rose_arch_compressions", ["compress_sources"],
None, app_runner)
# Set up the targets
cycle = os.getenv("ROSE_TASK_CYCLE_TIME")
targets = []
for t_key, t_node in sorted(config.value.items()):
if t_node.is_ignored() or ":" not in t_key:
continue
s_key_head, s_key_tail = t_key.split(":", 1)
if s_key_head != self.SECTION or not s_key_tail:
continue
target_prefix = self._get_conf(
config, t_node, "target-prefix", default="")
try:
s_key_tail = env_var_process(s_key_tail)
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError([t_key, ""], "", exc)
target_name = target_prefix + s_key_tail
target = RoseArchTarget(target_name)
target.command_format = self._get_conf(
config, t_node, "command-format", compulsory=True)
try:
target.command_format % {"sources": "", "target": ""}
except KeyError as exc:
target.status = target.ST_BAD
app_runner.handle_event(
RoseArchValueError(
target.name,
"command-format",
target.command_format,
type(exc).__name__,
exc
)
)
source_str = self._get_conf(
config, t_node, "source", compulsory=True)
source_prefix = self._get_conf(
config, t_node, "source-prefix", default="")
target.source_edit_format = self._get_conf(
config, t_node, "source-edit-format", default="")
try:
target.source_edit_format % {"in": "", "out": ""}
except KeyError as exc:
target.status = target.ST_BAD
app_runner.handle_event(
RoseArchValueError(
target.name,
"source-edit-format",
target.source_edit_format,
type(exc).__name__,
exc
)
)
update_check_str = self._get_conf(
config, t_node, "update-check", default="md5sum")
try:
checksum_func = get_checksum_func(update_check_str)
except KeyError as exc:
raise RoseArchValueError(
target.name,
"update-check",
update_check_str,
type(exc).__name__,
exc
)
for source_glob in shlex.split(source_str):
paths = glob(source_prefix + source_glob)
if not paths:
exc = OSError(errno.ENOENT, os.strerror(errno.ENOENT),
source_glob)
app_runner.handle_event(ConfigValueError(
[t_key, "source"], source_glob, exc))
target.status = target.ST_BAD
continue
for path in paths:
# N.B. source_prefix may not be a directory
name = path[len(source_prefix):]
for path_, checksum, _ in get_checksum(
path, checksum_func):
if checksum is None: # is directory
continue
if path_:
target.sources[checksum] = RoseArchSource(
checksum,
os.path.join(name, path_),
os.path.join(path, path_))
else: # path is a file
target.sources[checksum] = RoseArchSource(
checksum, name, path)
target.compress_scheme = self._get_conf(config, t_node, "compress")
if target.compress_scheme:
if (compress_manager.get_handler(target.compress_scheme) is
None):
app_runner.handle_event(ConfigValueError(
[t_key, "compress"],
target.compress_scheme,
KeyError(target.compress_scheme)))
target.status = target.ST_BAD
else:
target_base = target.name
if "/" in target.name:
target_base = target.name.rsplit("/", 1)[1]
if "." in target_base:
tail = target_base.split(".", 1)[1]
if compress_manager.get_handler(tail):
target.compress_scheme = tail
rename_format = self._get_conf(config, t_node, "rename-format")
if rename_format:
rename_parser_str = self._get_conf(config, t_node,
"rename-parser")
if rename_parser_str:
try:
rename_parser = re.compile(rename_parser_str)
except re.error as exc:
raise RoseArchValueError(
target.name,
"rename-parser",
rename_parser_str,
type(exc).__name__,
exc
)
else:
rename_parser = None
for source in target.sources.values():
dict_ = {"cycle": cycle, "name": source.name}
if rename_parser:
match = rename_parser.match(source.name)
if match:
dict_.update(match.groupdict())
try:
source.name = rename_format % dict_
except (KeyError, ValueError) as exc:
raise RoseArchValueError(
target.name,
"rename-format",
rename_format,
type(exc).__name__,
exc
)
old_target = dao.select(target.name)
if old_target is None or old_target != target:
dao.delete(target)
else:
target.status = target.ST_OLD
targets.append(target)
# Delete from database items that are no longer relevant
dao.delete_all(filter_targets=targets)
# Update the targets
for target in targets:
if target.status == target.ST_OLD:
app_runner.handle_event(RoseArchEvent(target))
continue
target.command_rc = 1
dao.insert(target)
if target.status == target.ST_BAD:
app_runner.handle_event(RoseArchEvent(target))
continue
work_dir = mkdtemp()
t_init = time()
t_tran, t_arch = t_init, t_init
ret_code = None
try:
# Rename/edit sources
target.status = target.ST_BAD
rename_required = False
for source in target.sources.values():
if source.name != source.orig_name:
rename_required = True
break
if rename_required or target.source_edit_format:
for source in target.sources.values():
source.path = os.path.join(work_dir, source.name)
source_path_d = os.path.dirname(source.path)
app_runner.fs_util.makedirs(source_path_d)
if target.source_edit_format:
fmt_args = {"in": source.orig_path,
"out": source.path}
command = target.source_edit_format % fmt_args
app_runner.popen.run_ok(command, shell=True)
else:
app_runner.fs_util.symlink(source.orig_path,
source.path)
# Compress sources
if target.compress_scheme:
handler = compress_manager.get_handler(
target.compress_scheme)
handler.compress_sources(target, work_dir)
t_tran = time()
# Run archive command
sources = []
if target.work_source_path:
sources = [target.work_source_path]
else:
for source in target.sources.values():
sources.append(source.path)
sources_str = app_runner.popen.list_to_shell_str(sources)
target_str = app_runner.popen.list_to_shell_str([target.name])
command = target.command_format % {"sources": sources_str,
"target": target_str}
ret_code, out, err = app_runner.popen.run(command, shell=True)
t_arch = time()
if ret_code:
app_runner.handle_event(
RosePopenError([command], ret_code, out, err))
else:
target.status = target.ST_NEW
app_runner.handle_event(err, kind=Event.KIND_ERR)
app_runner.handle_event(out)
app_runner.handle_event(out)
target.command_rc = ret_code
dao.update_command_rc(target)
finally:
app_runner.fs_util.delete(work_dir)
app_runner.handle_event(
RoseArchEvent(target, [t_init, t_tran, t_arch], ret_code))
return [target.status for target in targets].count(
RoseArchTarget.ST_BAD)
class RosieSvnPostCommitHook(object):
"""A post-commit hook on a Rosie Subversion repository.
Update the Rosie discovery database on changes to info file.
Notify owner and users on access-list on trunk changes.
"""
DATE_FMT = "%Y-%m-%d %H:%M:%S %Z"
ID_CHARS_LIST = ["abcdefghijklmnopqrstuvwxyz"] * 2 + ["0123456789"] * 3
LEN_ID = len(ID_CHARS_LIST)
INFO_FILE = "rose-suite.info"
KNOWN_KEYS_FILE_PATH = "R/O/S/I/E/trunk/rosie-keys"
REC_COPY_INFO = re.compile(r"\A\s+\(from\s(\S+):r(\d+)\)\s*\Z")
ST_ADDED = "A"
ST_DELETED = "D"
ST_MODIFIED = "M"
ST_EMPTY = " "
TRUNK = "trunk"
def __init__(self, event_handler=None, popen=None):
if event_handler is None:
event_handler = Reporter()
self.event_handler = event_handler
if popen is None:
popen = RosePopener(self.event_handler)
self.popen = popen
path = os.path.dirname(os.path.dirname(sys.modules["rosie"].__file__))
self.usertools_manager = SchemeHandlersManager([path],
"rosie.usertools",
["get_emails"])
def run(self, repos, revision, no_notification=False):
"""Update database with changes in a changeset."""
# Lookup prefix of repos
# Do nothing if prefix is not registered
conf = ResourceLocator.default().get_conf()
rosie_db_node = conf.get(["rosie-db"], no_ignore=True)
for key, node in rosie_db_node.value.items():
if node.is_ignored() or not key.startswith("repos."):
continue
if os.path.realpath(repos) == os.path.realpath(node.value):
prefix = key[len("repos."):]
break
else:
return
# Locate Rosie DB of repos
dao = RosieWriteDAO(conf.get_value(["rosie-db", "db." + prefix]))
# Date-time of this commit
os.environ["TZ"] = "UTC"
date_time_str = self._svnlook("date", "-r", revision, repos)
date, dtime, _ = date_time_str.split(None, 2)
date = mktime(strptime(" ".join([date, dtime, "UTC"]), self.DATE_FMT))
# Detail of changes
changeset_attribs = {
"repos": repos,
"revision": revision,
"prefix": prefix,
"author": self._svnlook("author", "-r", revision, repos).strip(),
"date": date
}
branch_attribs_dict = self._get_suite_branch_changes(repos, revision)
for key, branch_attribs in sorted(branch_attribs_dict.items()):
# Update known keys in suite info database meta table
if branch_attribs["has_changed_known_keys_file"]:
self._update_known_keys(dao, changeset_attribs)
# Update suite info database
self._update_info_db(dao, changeset_attribs, branch_attribs)
# Notification on trunk changes
# Notification on owner and access-list changes
if not no_notification and branch_attribs["branch"] == "trunk":
self._notify_trunk_changes(changeset_attribs, branch_attribs)
def _get_suite_branch_changes(self, repos, revision):
"""Retrieve changed statuses."""
branch_attribs_dict = {}
changed_lines = self._svnlook("changed", "--copy-info", "-r", revision,
repos).splitlines(True)
while changed_lines:
changed_line = changed_lines.pop(0)
# A normal status changed_line
# Column 1: content status
# Column 2: tree status
# Column 3: "+" sign denotes a copy history
# Column 5+: path
path = changed_line[4:].strip()
path_status = changed_line[0]
if path.endswith("/") and path_status == "_":
# Ignore property change on a directory
continue
# Path must be (under) a valid suite branch, including the special
# ROSIE suite
names = path.split("/", self.LEN_ID + 1)
if (len(names) < self.LEN_ID + 1 or
("".join(names[0:self.LEN_ID]) != "ROSIE" and any(
name not in id_chars
for name, id_chars in zip(names, self.ID_CHARS_LIST)))):
continue
sid = "".join(names[0:self.LEN_ID])
branch = names[self.LEN_ID]
if branch:
# Change to a path in a suite branch
if (sid, branch) not in branch_attribs_dict:
branch_attribs_dict[(sid, branch)] = (
self._new_suite_branch_change(sid, branch))
branch_attribs = branch_attribs_dict[(sid, branch)]
try:
tail = names[self.LEN_ID + 1]
except IndexError:
tail = None
if tail == self.INFO_FILE:
# Suite info file change
if branch_attribs["info"] is None:
branch_attribs["info"] = self._load_info(
repos, revision, sid, branch)
if path_status != self.ST_ADDED:
branch_attribs["old_info"] = self._load_info(
repos,
int(revision) - 1, sid, branch)
if (branch_attribs["old_info"] !=
branch_attribs["info"]
and branch_attribs["status"] != self.ST_ADDED):
branch_attribs["status_info_file"] = (
self.ST_MODIFIED)
elif tail:
# ROSIE meta known keys file change
if path == self.KNOWN_KEYS_FILE_PATH:
branch_attribs["has_changed_known_keys_file"] = True
if branch_attribs["status"] == self.ST_EMPTY:
branch_attribs["status"] = self.ST_MODIFIED
elif path_status in [self.ST_ADDED, self.ST_DELETED]:
# Branch add/delete
branch_attribs["status"] = path_status
# Load suite info and old info regardless
if branch_attribs["info"] is None:
branch_attribs["info"] = self._load_info(
repos, revision, sid, branch)
if (branch_attribs["old_info"] is None
and branch_attribs["status"] == self.ST_DELETED):
branch_attribs["old_info"] = self._load_info(
repos,
int(revision) - 1, sid, branch)
# Append changed lines, so they can be used for notification
branch_attribs["changed_lines"].append(changed_line)
if changed_line[2] == "+":
changed_line_2 = changed_lines.pop(0)
branch_attribs["changed_lines"].append(changed_line_2)
if path_status != self.ST_ADDED or tail:
continue
# A line containing the copy info for a branch
# Column 5+ looks like: (from PATH:rREV)
match = self.REC_COPY_INFO.match(changed_line_2)
if match:
from_path, from_rev = match.groups()
branch_attribs_dict[(sid, branch)].update({
"from_path":
from_path,
"from_rev":
from_rev
})
elif path_status == self.ST_DELETED:
# The suite has been deleted
tree_out = self._svnlook("tree", "-r", str(int(revision) - 1),
"-N", repos, path)
# Include all branches of the suite in the deletion info
for tree_line in tree_out.splitlines()[1:]:
del_branch = tree_line.strip().rstrip("/")
branch_attribs_dict[(sid, del_branch)] = (
self._new_suite_branch_change(
sid, del_branch, {
"old_info":
self._load_info(repos,
int(revision) - 1, sid,
del_branch),
"status":
self.ST_DELETED,
"status_info_file":
self.ST_EMPTY,
"changed_lines":
["D %s/%s/" % (path, del_branch)]
}))
return branch_attribs_dict
def _load_info(self, repos, revision, sid, branch):
"""Load info file from branch_path in repos @revision."""
info_file_path = "%s/%s/%s" % ("/".join(sid), branch, self.INFO_FILE)
t_handle = TemporaryFile()
try:
t_handle.write(
self._svnlook("cat", "-r", str(revision), repos,
info_file_path))
except RosePopenError:
return None
t_handle.seek(0)
config = rose.config.load(t_handle)
t_handle.close()
return config
def _new_suite_branch_change(self, sid, branch, attribs=None):
"""Return a dict to represent a suite branch change."""
branch_attribs = {
"sid": sid,
"branch": branch,
"from_path": None,
"from_rev": None,
"has_changed_known_keys_file": False,
"old_info": None,
"info": None,
"status": self.ST_EMPTY,
"status_info_file": self.ST_EMPTY,
"changed_lines": []
}
if attribs:
branch_attribs.update(attribs)
return branch_attribs
def _notify_trunk_changes(self, changeset_attribs, branch_attribs):
"""Email owner and/or access-list users on changes to trunk."""
# Notify only if users' email addresses can be determined
conf = ResourceLocator.default().get_conf()
user_tool_name = conf.get_value(["rosa-svn", "user-tool"])
if not user_tool_name:
return
notify_who_str = conf.get_value(
["rosa-svn", "notify-who-on-trunk-commit"], "")
if not notify_who_str.strip():
return
notify_who = shlex.split(notify_who_str)
# Build the message text
info_file_path = "%s/trunk/%s" % ("/".join(
branch_attribs["sid"]), self.INFO_FILE)
text = ""
for changed_line in branch_attribs["changed_lines"]:
text += changed_line
# For suite info file change, add diff as well
if (changed_line[4:].strip() == info_file_path and
branch_attribs["status_info_file"] == self.ST_MODIFIED):
old_strio = StringIO()
rose.config.dump(branch_attribs["old_info"], old_strio)
new_strio = StringIO()
rose.config.dump(branch_attribs["info"], new_strio)
for diff_line in unified_diff(
old_strio.getvalue().splitlines(True),
new_strio.getvalue().splitlines(True),
"@%d" % (int(changeset_attribs["revision"]) - 1),
"@%d" % (int(changeset_attribs["revision"]))):
text += " " * 4 + diff_line
# Determine who to notify
users = set()
for key in ["old_info", "info"]:
if branch_attribs[key] is not None:
info_conf = branch_attribs[key]
if "owner" in notify_who:
users.add(info_conf.get_value(["owner"]))
if "access-list" in notify_who:
users.update(
info_conf.get_value(["access-list"], "").split())
users.discard("*")
# Determine email addresses
user_tool = self.usertools_manager.get_handler(user_tool_name)
if "author" in notify_who:
users.add(changeset_attribs["author"])
else:
users.discard(changeset_attribs["author"])
emails = sorted(user_tool.get_emails(users))
if not emails:
return
# Send notification
msg = MIMEText(text)
msg.set_charset("utf-8")
msg["From"] = conf.get_value(["rosa-svn", "notification-from"],
"notications@" + socket.getfqdn())
msg["To"] = ", ".join(emails)
msg["Subject"] = "%s-%s/trunk@%d" % (
changeset_attribs["prefix"], branch_attribs["sid"],
int(changeset_attribs["revision"]))
smtp_host = conf.get_value(["rosa-svn", "smtp-host"],
default="localhost")
smtp = SMTP(smtp_host)
smtp.sendmail(msg["From"], emails, msg.as_string())
smtp.quit()
def _svnlook(self, *args):
"""Return the standard output from "svnlook"."""
command = ["svnlook"] + list(args)
return self.popen(*command)[0]
def _update_info_db(self, dao, changeset_attribs, branch_attribs):
"""Update the suite info database for a suite branch."""
idx = changeset_attribs["prefix"] + "-" + branch_attribs["sid"]
vc_attrs = {
"idx": idx,
"branch": branch_attribs["branch"],
"revision": changeset_attribs["revision"]
}
for key in vc_attrs:
vc_attrs[key] = vc_attrs[key].decode("utf-8")
# Latest table
try:
dao.delete(LATEST_TABLE_NAME,
idx=vc_attrs["idx"],
branch=vc_attrs["branch"])
except al.exc.IntegrityError:
# idx and branch were just added: there is no previous record.
pass
if branch_attribs["status"] != self.ST_DELETED:
dao.insert(LATEST_TABLE_NAME, **vc_attrs)
# N.B. deleted suite branch only has old info
info_key = "info"
if branch_attribs["status"] == self.ST_DELETED:
info_key = "old_info"
if branch_attribs[info_key] is None:
return
# Main table
cols = dict(vc_attrs)
cols.update({
"author": changeset_attribs["author"],
"date": changeset_attribs["date"]
})
for name in ["owner", "project", "title"]:
cols[name] = branch_attribs[info_key].get_value([name])
if branch_attribs["from_path"] and vc_attrs["branch"] == u"trunk":
from_names = branch_attribs["from_path"].split("/")[:self.LEN_ID]
cols["from_idx"] = (changeset_attribs["prefix"] + "-" +
"".join(from_names))
cols["status"] = (branch_attribs["status"] +
branch_attribs["status_info_file"])
for key in cols:
try:
cols[key] = cols[key].decode("utf-8")
except AttributeError:
pass
dao.insert(MAIN_TABLE_NAME, **cols)
# Optional table
for name in branch_attribs[info_key].value:
if name in ["owner", "project", "title"]:
continue
value = branch_attribs[info_key].get_value([name])
if value is None: # setting may have ignore flag (!)
continue
cols = dict(vc_attrs)
cols.update({
"name": name.decode("utf-8"),
"value": value.decode("utf-8")
})
dao.insert(OPTIONAL_TABLE_NAME, **cols)
def _update_known_keys(self, dao, changeset_attribs):
"""Update the known_keys in the meta table."""
repos = changeset_attribs["repos"]
revision = changeset_attribs["revision"]
keys_str = self._svnlook("cat", "-r", revision, repos,
self.KNOWN_KEYS_FILE_PATH)
keys_str = " ".join(shlex.split(keys_str)).decode("utf-8")
if keys_str:
try:
dao.insert(META_TABLE_NAME, name=u"known_keys", value=keys_str)
except al.exc.IntegrityError:
dao.update(META_TABLE_NAME, (u"name", ),
name=u"known_keys",
value=keys_str)
class AppRunner(Runner):
"""Invoke a Rose application."""
OLD_DURATION_UNITS = {"h": 3600, "m": 60, "s": 1}
NAME = "app"
OPTIONS = ["app_mode", "command_key", "conf_dir", "defines",
"install_only_mode", "new_mode", "no_overwrite_mode",
"opt_conf_keys"]
def __init__(self, *args, **kwargs):
Runner.__init__(self, *args, **kwargs)
path = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__))
self.builtins_manager = SchemeHandlersManager(
[path], "rose.apps", ["run"], None, *args, **kwargs)
self.duration_parser = DurationParser()
def run_impl(self, opts, args, uuid, work_files):
"""The actual logic for a run."""
# Preparation.
conf_tree = self.config_load(opts)
self._prep(conf_tree, opts)
self._poll(conf_tree)
# Run the application or the command.
app_mode = conf_tree.node.get_value(["mode"])
if app_mode is None:
app_mode = opts.app_mode
if app_mode in [None, "command"]:
return self._command(conf_tree, opts, args)
else:
builtin_app = self.builtins_manager.get_handler(app_mode)
if builtin_app is None:
raise UnknownBuiltinAppError(app_mode)
return builtin_app.run(self, conf_tree, opts, args, uuid,
work_files)
def _poll(self, conf_tree):
"""Poll for prerequisites of applications."""
# Poll configuration
poll_test = conf_tree.node.get_value(["poll", "test"])
poll_all_files_value = conf_tree.node.get_value(["poll", "all-files"])
poll_all_files = []
if poll_all_files_value:
try:
poll_all_files = shlex.split(
env_var_process(poll_all_files_value))
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError(["poll", "all-files"],
poll_all_files_value, exc)
poll_any_files_value = conf_tree.node.get_value(["poll", "any-files"])
poll_any_files = []
if poll_any_files_value:
try:
poll_any_files = shlex.split(
env_var_process(poll_any_files_value))
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError(["poll", "any-files"],
poll_any_files_value, exc)
poll_file_test = None
if poll_all_files or poll_any_files:
poll_file_test = conf_tree.node.get_value(["poll", "file-test"])
if poll_file_test and "{}" not in poll_file_test:
raise ConfigValueError(["poll", "file-test"], poll_file_test,
ConfigValueError.SYNTAX)
poll_delays = []
if poll_test or poll_all_files or poll_any_files:
# Parse something like this: delays=10,4*PT30S,PT2M30S,2*PT1H
# R*DURATION: repeat the value R times
conf_keys = ["poll", "delays"]
poll_delays_value = conf_tree.node.get_value(
conf_keys, default="").strip()
if poll_delays_value:
is_legacy0 = None
for item in poll_delays_value.split(","):
value = item.strip()
repeat = 1
if "*" in value:
repeat, value = value.split("*", 1)
try:
repeat = int(repeat)
except ValueError as exc:
raise ConfigValueError(conf_keys,
poll_delays_value,
ConfigValueError.SYNTAX)
try:
value = self.duration_parser.parse(value).get_seconds()
is_legacy = False
except ISO8601SyntaxError:
# Legacy mode: nnnU
# nnn is a float, U is the unit
# No unit or s: seconds
# m: minutes
# h: hours
unit = None
if value[-1].lower() in self.OLD_DURATION_UNITS:
unit = self.OLD_DURATION_UNITS[value[-1].lower()]
value = value[:-1]
try:
value = float(value)
except ValueError as exc:
raise ConfigValueError(conf_keys,
poll_delays_value,
ConfigValueError.SYNTAX)
if unit:
value *= unit
is_legacy = True
if is_legacy0 is None:
is_legacy0 = is_legacy
elif is_legacy0 != is_legacy:
raise ConfigValueError(
conf_keys,
poll_delays_value,
ConfigValueError.DURATION_LEGACY_MIX)
poll_delays += [value] * repeat
else:
poll_delays = [0] # poll once without a delay
# Poll
t_init = get_timepoint_for_now()
while poll_delays and (poll_test or poll_any_files or poll_all_files):
poll_delay = poll_delays.pop(0)
if poll_delay:
sleep(poll_delay)
if poll_test:
ret_code = self.popen.run(
poll_test, shell=True,
stdout=sys.stdout, stderr=sys.stderr)[0]
self.handle_event(PollEvent(time(), poll_test, ret_code == 0))
if ret_code == 0:
poll_test = None
any_files = list(poll_any_files)
for file_ in any_files:
if self._poll_file(file_, poll_file_test):
self.handle_event(PollEvent(time(), "any-files", True))
poll_any_files = []
break
all_files = list(poll_all_files)
for file_ in all_files:
if self._poll_file(file_, poll_file_test):
poll_all_files.remove(file_)
if all_files and not poll_all_files:
self.handle_event(PollEvent(time(), "all-files", True))
failed_items = []
if poll_test:
failed_items.append("test")
if poll_any_files:
failed_items.append("any-files")
if poll_all_files:
failed_items.append("all-files:" +
self.popen.list_to_shell_str(poll_all_files))
if failed_items:
now = get_timepoint_for_now()
raise PollTimeoutError(now, now - t_init, failed_items)
def _poll_file(self, file_, poll_file_test):
"""Poll for existence of a file."""
is_done = False
if poll_file_test:
test = poll_file_test.replace(
"{}", self.popen.list_to_shell_str([file_]))
is_done = self.popen.run(
test, shell=True, stdout=sys.stdout, stderr=sys.stderr)[0] == 0
else:
is_done = bool(glob(file_))
self.handle_event(PollEvent(time(), "file:" + file_, is_done))
return is_done
def _prep(self, conf_tree, opts):
"""Prepare to run the application."""
if opts.new_mode:
conf_dir = opts.conf_dir
if not conf_dir or os.path.abspath(conf_dir) == os.getcwd():
raise NewModeError(os.getcwd())
for path in os.listdir("."):
self.fs_util.delete(path)
# Dump the actual configuration as rose-app-run.conf
ConfigDumper()(conf_tree.node, "rose-app-run.conf")
# Environment variables: PATH
paths = []
for conf_dir in conf_tree.conf_dirs:
conf_bin_dir = os.path.join(conf_dir, "bin")
if os.path.isdir(conf_bin_dir):
paths.append(conf_bin_dir)
if paths:
value = os.pathsep.join(paths + [os.getenv("PATH")])
conf_tree.node.set(["env", "PATH"], value)
else:
conf_tree.node.set(["env", "PATH"], os.getenv("PATH"))
# Free format files not defined in the configuration file
file_section_prefix = self.config_pm.get_handler("file").PREFIX
for rel_path, conf_dir in conf_tree.files.items():
if not rel_path.startswith("file" + os.sep):
continue
name = rel_path[len("file" + os.sep):]
# No sub-directories, very slow otherwise
if os.sep in name:
name = name.split(os.sep, 1)[0]
target_key = file_section_prefix + name
target_node = conf_tree.node.get([target_key])
if target_node is None:
conf_tree.node.set([target_key])
target_node = conf_tree.node.get([target_key])
elif target_node.is_ignored():
continue
source_node = target_node.get("source")
if source_node is None:
target_node.set(["source"],
os.path.join(conf_dir, "file", name))
elif source_node.is_ignored():
continue
# Process Environment Variables
self.config_pm(conf_tree, "env")
# Process Files
self.config_pm(conf_tree, "file",
no_overwrite_mode=opts.no_overwrite_mode)
def _command(self, conf_tree, opts, args):
"""Run the command."""
command = self.popen.list_to_shell_str(args)
if not command:
names = [opts.command_key, os.getenv("ROSE_TASK_NAME"), "default"]
for name in names:
if not name:
continue
command = conf_tree.node.get_value(["command", name])
if command is not None:
break
else:
self.handle_event(CommandNotDefinedEvent())
return
if os.access("STDIN", os.F_OK | os.R_OK):
command += " <STDIN"
self.handle_event("command: %s" % command)
if opts.install_only_mode:
return
self.popen(command, shell=True, stdout=sys.stdout, stderr=sys.stderr)
class RosieSvnPostCommitHook(object):
"""A post-commit hook on a Rosie Subversion repository.
Update the Rosie discovery database on changes.
"""
DATE_FMT = "%Y-%m-%d %H:%M:%S %Z"
RE_ID_NAMES = [r"[a-z]", r"[a-z]", r"\d", r"\d", r"\d"]
LEN_ID = len(RE_ID_NAMES)
INFO_FILE = "rose-suite.info"
KNOWN_KEYS_FILE = "rosie-keys"
REC_COPY_INFO = re.compile("^\s+\(from\s([^\s]+)\)$")
ST_ADDED = "A"
ST_DELETED = "D"
ST_MODIFIED = "M"
ST_UPDATED = "U"
TRUNK = "trunk"
def __init__(self, event_handler=None, popen=None):
if event_handler is None:
event_handler = Reporter()
self.event_handler = event_handler
if popen is None:
popen = RosePopener(self.event_handler)
self.popen = popen
path = os.path.dirname(os.path.dirname(sys.modules["rosie"].__file__))
self.usertools_manager = SchemeHandlersManager(
[path], "rosie.usertools", ["get_emails"])
def _svnlook(self, *args):
"""Return the standard output from "svnlook"."""
command = ["svnlook"] + list(args)
return self.popen(*command)[0]
def _check_path_is_sid(self, path):
"""Return whether the path contains a suffix-id."""
names = path.split("/")
if len(names) < self.LEN_ID + 1:
return False
if "".join(names)[:self.LEN_ID] == "ROSIE":
return True
for name, pattern in zip(names, self.RE_ID_NAMES):
if not re.compile(r"\A" + pattern + r"\Z").match(name):
return False
return True
def run(self, repos, revision):
"""Update database with changes in a changeset."""
conf = ResourceLocator.default().get_conf()
rosie_db_node = conf.get(["rosie-db"], no_ignore=True)
for key, node in rosie_db_node.value.items():
if node.is_ignored() or not key.startswith("repos."):
continue
if os.path.realpath(repos) == os.path.realpath(node.value):
prefix = key[len("repos."):]
break
else:
return
dao = RosieWriteDAO(conf.get_value(["rosie-db", "db." + prefix]))
idx_branch_rev_info = {}
author = self._svnlook("author", "-r", revision, repos).strip()
os.environ["TZ"] = "UTC"
date_time_str = self._svnlook("date", "-r", revision, repos)
date, dtime, _ = date_time_str.split(None, 2)
date = time.mktime(time.strptime(" ".join([date, dtime, "UTC"]),
self.DATE_FMT))
# Retrieve copied information on the suite-idx-level.
path_copies = {}
path_num = -1
copy_changes = self._svnlook("changed", "--copy-info", "-r", revision,
repos)
for i, change in enumerate(copy_changes.splitlines()):
copy_match = self.REC_COPY_INFO.match(change)
if copy_match:
path_copies[path_num] = copy_match.groups()[0]
else:
path_num += 1
changes = self._svnlook("changed", "-r", revision, repos)
suite_statuses = {}
path_statuses = []
for i, line in enumerate(changes.splitlines()):
path_status, path = line.rsplit(None, 1)
if not self._check_path_is_sid(path):
# The path must contain a full suite id (e.g. a/a/0/0/1/).
continue
path_statuses.append((path_status, path, i))
# Loop through a stack of statuses, paths, and copy info pointers.
configs = {}
while path_statuses:
path_status, path, path_num = path_statuses.pop(0)
names = path.split("/")
sid = "".join(names[0:self.LEN_ID])
idx = prefix + "-" + sid
branch = names[self.LEN_ID]
branch_path = "/".join(sid) + "/" + branch
info_file_path = branch_path + "/" + self.INFO_FILE
if not branch and path_status[0] == self.ST_DELETED:
# The suite has been deleted at the a/a/0/0/1/ level.
out = self._svnlook("tree", "-r", str(int(revision) - 1),
"-N", repos, path)
# Include all branches of the suite in the deletion info.
for line in out.splitlines()[1:]:
del_branch = line.strip().rstrip("/")
path_statuses.append((path_status, path.rstrip("/") +
"/" + del_branch, None))
if (sid == "ROSIE" and branch == self.TRUNK and
path == branch_path + "/" + self.KNOWN_KEYS_FILE):
# The known keys in the special R/O/S/I/E/ suite have changed.
keys_str = self._svnlook("cat", "-r", revision, repos, path)
keys_str = " ".join(shlex.split(keys_str))
if keys_str:
try:
dao.insert(META_TABLE_NAME, name="known_keys",
value=keys_str)
except al.exc.IntegrityError:
dao.update(META_TABLE_NAME, ("name",),
name="known_keys", value=keys_str)
status_0 = " "
from_idx = None
if (path_num in path_copies and
self._check_path_is_sid(path_copies[path_num])):
copy_names = path_copies[path_num].split("/")
copy_sid = "".join(copy_names[:self.LEN_ID])
if copy_sid != sid and branch:
# This has been copied from a different suite.
from_idx = prefix + "-" + copy_sid
# Figure out our status information.
if path.rstrip("/") == branch_path:
if path_status[0] == self.ST_DELETED:
status_0 = self.ST_DELETED
if path_status[0] == self.ST_ADDED:
status_0 = self.ST_ADDED
if (len(path.rstrip("/")) > len(branch_path) and
path != info_file_path and status_0.isspace()):
status_0 = self.ST_MODIFIED
suite_statuses.setdefault((idx, branch, revision),
{0: status_0, 1: " "})
status_info = suite_statuses[(idx, branch, revision)]
if not branch:
continue
if path.rstrip("/") not in [branch_path, info_file_path]:
if branch_path not in [i[1] for i in path_statuses]:
# Make sure the branch gets noticed.
path_statuses.append((self.ST_UPDATED, branch_path, None))
continue
# Start populating the idx+branch+revision info for this suite.
suite_info = idx_branch_rev_info.setdefault(
(idx, branch, revision), {})
suite_info["author"] = author
suite_info["date"] = date
suite_info.setdefault("from_idx", None)
if from_idx is not None:
suite_info["from_idx"] = from_idx
suite_info.setdefault("owner", "")
suite_info.setdefault("project", "")
suite_info.setdefault("title", "")
suite_info.setdefault("optional", {})
if (idx, branch, revision) not in configs:
new_config = self._get_config_node(
repos, info_file_path, revision)
old_config = self._get_config_node(
repos, info_file_path, str(int(revision) - 1))
configs[(idx, branch, revision)] = (new_config, old_config)
if branch == self.TRUNK:
self._notify_access_changes(
"%s/%s@%s" % (idx, branch, revision),
suite_info["author"],
old_config,
new_config)
new_config, old_config = configs[(idx, branch, revision)]
if new_config is None and old_config is None:
# A technically-invalid commit (likely to be historical).
idx_branch_rev_info.pop((idx, branch, revision))
continue
if new_config is None and status_info[0] == self.ST_DELETED:
new_config = old_config
if old_config is None and status_info[0] == self.ST_ADDED:
old_config = new_config
if self._get_configs_differ(old_config, new_config):
status_info[1] = self.ST_MODIFIED
for key, node in new_config.value.items():
if node.is_ignored():
continue
if key in ["owner", "project", "title"]:
suite_info[key] = node.value
else:
suite_info["optional"][key] = node.value
# Now loop over all idx+branch+revision suite groups.
for suite_id, suite_info in idx_branch_rev_info.items():
idx, branch, revision = suite_id
status = suite_statuses.get((idx, branch, revision),
{0: " ", 1: " "})
suite_info["status"] = (status[0] +
status[1])
optional = suite_info.pop("optional")
for key, value in optional.items():
dao.insert(OPTIONAL_TABLE_NAME, idx=idx, branch=branch,
revision=revision, name=key, value=value)
dao.insert(MAIN_TABLE_NAME, idx=idx, branch=branch,
revision=revision, **suite_info)
try:
dao.delete(LATEST_TABLE_NAME, idx=idx, branch=branch)
except al.exc.IntegrityError:
# idx and branch were just added: there is no previous record.
pass
if suite_info["status"][0] != self.ST_DELETED:
dao.insert(LATEST_TABLE_NAME, idx=idx, branch=branch,
revision=revision)
__call__ = run
def _get_config_node(self, repos, info_file_path, revision):
"""Load configuration file from info_file_path in repos @revision."""
t_handle = tempfile.TemporaryFile()
try:
t_handle.write(
self._svnlook("cat", "-r", revision, repos, info_file_path))
except RosePopenError:
return None
t_handle.seek(0)
config = rose.config.load(t_handle)
t_handle.close()
return config
@classmethod
def _get_configs_differ(cls, old_config, new_config):
"""Return True if old_config differs from new_config."""
for keys1, node1 in old_config.walk(no_ignore=True):
node2 = new_config.get(keys1, no_ignore=True)
if type(node1) != type(node2):
return True
if (not isinstance(node1.value, dict) and
node1.value != node2.value):
return True
if node1.comments != node2.comments:
return True
for keys2, node2 in new_config.walk(no_ignore=True):
node1 = old_config.get(keys2, no_ignore=True)
if node1 is None:
return True
return False
def _notify_access_changes(self, full_id, author, old_config, new_config):
"""Email owner and/or access-list users on changes."""
conf = ResourceLocator.default().get_conf()
user_tool_name = conf.get_value(["rosa-svn", "user-tool"])
if not user_tool_name:
return
users = set()
if old_config is None:
new_access_str = new_config.get_value(["access-list"], "")
changes = {
("owner", "+"): new_config.get_value(["owner"]),
("access-list", "+"): new_access_str,
}
users.update(new_access_str.split())
elif new_config is None:
old_owner = old_config.get_value(["owner"])
old_access_str = old_config.get_value(["access-list"], "")
changes = {
("owner", "-"): old_owner,
("access-list", "-"): old_access_str,
}
users.add(old_owner)
users.update(old_access_str.split())
else:
changes = {}
old_owner = old_config.get_value(["owner"])
new_owner = new_config.get_value(["owner"])
if old_owner != new_owner:
changes[("owner", "-")] = old_owner
changes[("owner", "+")] = new_owner
users.add(old_owner)
users.add(new_owner)
old_access_str = old_config.get_value(["access-list"], "")
new_access_str = new_config.get_value(["access-list"], "")
old_access_set = set(old_access_str.split())
new_access_set = set(new_access_str.split())
if old_access_set != new_access_set:
changes[("access-list", "-")] = old_access_str
changes[("access-list", "+")] = new_access_str
users.update(old_access_set ^ new_access_set)
users.discard("*")
users.discard(author)
if not users or users == set([author]):
return
user_tool = self.usertools_manager.get_handler(user_tool_name)
users.add(author)
emails = sorted(user_tool.get_emails(users))
from_email = conf.get_value(["rosa-svn", "notification-from"],
"notications@" + socket.getfqdn())
text = ""
for key, status in [
("owner", "-"),
("owner", "+"),
("access-list", "-"),
("access-list", "+")]:
if (key, status) in changes:
text += "%s %s=%s\n" % (status, key, changes[(key, status)])
msg = MIMEText(text)
msg.set_charset("utf-8")
msg["From"] = from_email
msg["To"] = ", ".join(emails)
msg["Subject"] = "[%s] owner/access-list change" % full_id
smtp_host = conf.get_value(["rosa-svn", "smtp-host"],
default="localhost")
smtp = SMTP(smtp_host)
smtp.sendmail(msg["From"], emails, msg.as_string())
smtp.quit()
class AppRunner(Runner):
"""Invoke a Rose application."""
NAME = "app"
OPTIONS = [
"app_mode", "command_key", "conf_dir", "defines", "install_only_mode",
"new_mode", "no_overwrite_mode", "opt_conf_keys"
]
def __init__(self, *args, **kwargs):
Runner.__init__(self, *args, **kwargs)
path = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__))
self.builtins_manager = SchemeHandlersManager([path], "rose.apps",
["run"], None, *args,
**kwargs)
self.date_time_oper = RoseDateTimeOperator()
def run_impl(self, opts, args, uuid, work_files):
"""The actual logic for a run."""
# Preparation.
conf_tree = self.config_load(opts)
self._prep(conf_tree, opts)
self._poll(conf_tree)
# Run the application or the command.
app_mode = conf_tree.node.get_value(["mode"])
if app_mode is None:
app_mode = opts.app_mode
if app_mode is None:
app_mode = os.getenv("ROSE_APP_MODE")
if app_mode in [None, "command"]:
return self._command(conf_tree, opts, args)
else:
builtin_app = self.builtins_manager.get_handler(app_mode)
if builtin_app is None:
raise UnknownBuiltinAppError(app_mode)
return builtin_app.run(self, conf_tree, opts, args, uuid,
work_files)
def get_command(self, conf_tree, opts, args):
"""Get command to run."""
command = self.popen.list_to_shell_str(args)
if not command:
names = [
opts.command_key,
os.getenv("ROSE_APP_COMMAND_KEY"),
os.getenv("ROSE_TASK_NAME"), "default"
]
for name in names:
if not name:
continue
command = conf_tree.node.get_value(["command", name])
if command is not None:
break
return command
def _prep(self, conf_tree, opts):
"""Prepare to run the application."""
if opts.new_mode:
self._prep_new(opts)
# Dump the actual configuration as rose-app-run.conf
ConfigDumper()(conf_tree.node, "rose-app-run.conf")
# Environment variables: PATH
self._prep_path(conf_tree)
# Free format files not defined in the configuration file
file_section_prefix = self.config_pm.get_handler("file").PREFIX
for rel_path, conf_dir in conf_tree.files.items():
if not rel_path.startswith("file" + os.sep):
continue
name = rel_path[len("file" + os.sep):]
# No sub-directories, very slow otherwise
if os.sep in name:
name = name.split(os.sep, 1)[0]
target_key = file_section_prefix + name
target_node = conf_tree.node.get([target_key])
if target_node is None:
conf_tree.node.set([target_key])
target_node = conf_tree.node.get([target_key])
elif target_node.is_ignored():
continue
source_node = target_node.get(["source"])
if source_node is None:
target_node.set(["source"],
os.path.join(conf_dir, "file", name))
elif source_node.is_ignored():
continue
# Process Environment Variables
self.config_pm(conf_tree, "env")
# Process Files
self.config_pm(conf_tree,
"file",
no_overwrite_mode=opts.no_overwrite_mode)
def _prep_new(self, opts):
"""Clear out run directory on a --new option if possible."""
conf_dir = opts.conf_dir
if not conf_dir or os.path.abspath(conf_dir) == os.getcwd():
raise NewModeError(os.getcwd())
for path in os.listdir("."):
self.fs_util.delete(path)
@staticmethod
def _prep_path(conf_tree):
"""Add bin directories to the PATH seen by the app command."""
paths = []
for conf_dir in conf_tree.conf_dirs:
conf_bin_dir = os.path.join(conf_dir, "bin")
if os.path.isdir(conf_bin_dir):
paths.append(conf_bin_dir)
if paths:
value = os.pathsep.join(paths + [os.getenv("PATH")])
conf_tree.node.set(["env", "PATH"], value)
else:
conf_tree.node.set(["env", "PATH"], os.getenv("PATH"))
def _poll(self, conf_tree):
"""Run any configured file polling."""
poller = Poller(self.popen, self.handle_event)
poller.poll(conf_tree)
def _command(self, conf_tree, opts, args):
"""Run the command."""
command = self.get_command(conf_tree, opts, args)
if not command:
self.handle_event(CommandNotDefinedEvent())
return
if os.access("STDIN", os.F_OK | os.R_OK):
command += " <STDIN"
self.handle_event("command: %s" % command)
if opts.install_only_mode:
return
self.popen(command,
shell=True,
stdout=sys.stdout,
stderr=sys.stderr,
stdin=sys.stdin)
class RosieSvnPreCommitHook(object):
"""A pre-commit hook on a Rosie Subversion repository.
Ensure that commits conform to the rules of Rosie.
"""
IGNORES = "svnperms.conf"
RE_ID_NAMES = [r"[Ra-z]", r"[Oa-z]", r"[S\d]", r"[I\d]", r"[E\d]"]
LEN_ID = len(RE_ID_NAMES)
ST_ADD = "A"
ST_DELETE = "D"
ST_UPDATE = "U"
TRUNK_INFO_FILE = "trunk/rose-suite.info"
TRUNK_KNOWN_KEYS_FILE = "trunk/rosie-keys"
def __init__(self, event_handler=None, popen=None):
if event_handler is None:
event_handler = Reporter()
self.event_handler = event_handler
if popen is None:
popen = RosePopener(self.event_handler)
self.popen = popen
path = os.path.dirname(os.path.dirname(sys.modules["rosie"].__file__))
self.usertools_manager = SchemeHandlersManager(
[path], "rosie.usertools", ["verify_users"])
def _get_access_info(self, info_node):
"""Return (owner, access_list) from "info_node"."""
owner = info_node.get_value(["owner"])
access_list = info_node.get_value(["access-list"], "").split()
access_list.sort()
return owner, access_list
def _get_info(self, repos, path_head, txn=None):
"""Return a ConfigNode for the "rose-suite.info" of a suite.
The suite is located under path_head.
"""
opt_txn = []
if txn is not None:
opt_txn = ["-t", txn]
t_handle = tempfile.TemporaryFile()
path = path_head + self.TRUNK_INFO_FILE
t_handle.write(self._svnlook("cat", repos, path, *opt_txn))
t_handle.seek(0)
info_node = ConfigLoader()(t_handle)
t_handle.close()
return info_node
def _svnlook(self, *args):
"""Return the standard output from "svnlook"."""
command = ["svnlook"] + list(args)
return self.popen(*command, stderr=sys.stderr)[0]
def _verify_users(self, status, path, txn_owner, txn_access_list,
bad_changes):
"""Check txn_owner and txn_access_list.
For any invalid users, append to bad_changes and return True.
"""
# The owner and names in access list must be real users
conf = ResourceLocator.default().get_conf()
user_tool_name = conf.get_value(["rosa-svn", "user-tool"])
if not user_tool_name:
return False
user_tool = self.usertools_manager.get_handler(user_tool_name)
txn_users = set([txn_owner] + txn_access_list)
txn_users.discard("*")
bad_users = user_tool.verify_users(txn_users)
for bad_user in bad_users:
if txn_owner == bad_user:
bad_change = BadChange(
status,
path,
BadChange.USER,
"owner=" + bad_user)
bad_changes.append(bad_change)
if bad_user in txn_access_list:
bad_change = BadChange(
status,
path,
BadChange.USER,
"access-list=" + bad_user)
bad_changes.append(bad_change)
return bool(bad_users)
def run(self, repos, txn):
"""Apply the rule engine on transaction "txn" to repository "repos"."""
changes = set() # set([(status, path), ...])
for line in self._svnlook("changed", "-t", txn, repos).splitlines():
status, path = line.split(None, 1)
changes.add((status, path))
bad_changes = []
author = None
super_users = None
rev_info_map = {} # {path-id: (owner, access-list), ...}
txn_info_map = {}
conf = ResourceLocator.default().get_conf()
ignores_str = conf.get_value(["rosa-svn", "ignores"], self.IGNORES)
ignores = shlex.split(ignores_str)
for status, path in sorted(changes):
if any([fnmatch(path, ignore) for ignore in ignores]):
continue
names = path.split("/", self.LEN_ID + 1)
if not names[-1]:
names.pop()
# Directories above the suites must match the ID patterns
is_bad = False
for name, pattern in zip(names, self.RE_ID_NAMES):
if not re.compile(r"\A" + pattern + r"\Z").match(name):
is_bad = True
break
if is_bad:
bad_changes.append(BadChange(status, path))
continue
# Can only add directories at levels above the suites
if len(names) < self.LEN_ID:
if status[0] != self.ST_ADD:
bad_changes.append(BadChange(status, path))
continue
else:
is_meta_suite = "".join(names[0:self.LEN_ID]) == "ROSIE"
# No need to check non-trunk changes
if len(names) > self.LEN_ID and names[self.LEN_ID] != "trunk":
continue
# New suite should have an info file
if status[0] == self.ST_ADD and len(names) == self.LEN_ID:
if (self.ST_ADD, path + "trunk/") not in changes:
bad_changes.append(
BadChange(status, path, BadChange.NO_TRUNK))
continue
path_trunk_info_file = path + self.TRUNK_INFO_FILE
if ((self.ST_ADD, path_trunk_info_file) not in changes and
(self.ST_UPDATE, path_trunk_info_file) not in changes):
bad_changes.append(
BadChange(status, path, BadChange.NO_INFO))
continue
# The rest are trunk changes in a suite
path_head = "/".join(names[0:self.LEN_ID]) + "/"
path_tail = path[len(path_head):]
# For meta suite, make sure keys in keys file can be parsed
if is_meta_suite and path_tail == self.TRUNK_KNOWN_KEYS_FILE:
out = self._svnlook("cat", "-t", txn, repos, path)
try:
shlex.split(out)
except ValueError:
bad_changes.append(
BadChange(status, path, BadChange.VALUE))
continue
# Suite trunk information file must have an owner
# User IDs of owner and access list must be real
if (status not in self.ST_DELETE and
path_tail == self.TRUNK_INFO_FILE):
if path_head not in txn_info_map:
txn_info_map[path_head] = self._get_info(
repos, path_head, txn)
txn_owner, txn_access_list = self._get_access_info(
txn_info_map[path_head])
if not txn_owner:
bad_changes.append(
BadChange(status, path, BadChange.NO_OWNER))
continue
if self._verify_users(
status, path, txn_owner, txn_access_list, bad_changes):
continue
reports = DefaultValidators().validate(
txn_info_map[path_head],
load_meta_config(
txn_info_map[path_head],
config_type=rose.INFO_CONFIG_NAME))
if reports:
reports_str = get_reports_as_text({None: reports}, path)
bad_changes.append(
BadChange(status, path, BadChange.VALUE, reports_str))
continue
# Can only remove trunk information file with suite
if status == self.ST_DELETE and path_tail == self.TRUNK_INFO_FILE:
if (self.ST_DELETE, path_head) not in changes:
bad_changes.append(
BadChange(status, path, BadChange.NO_INFO))
continue
# Can only remove trunk with suite
if status == self.ST_DELETE and path_tail == "trunk/":
if (self.ST_DELETE, path_head) not in changes:
bad_changes.append(
BadChange(status, path, BadChange.NO_TRUNK))
continue
# New suite trunk: ignore the rest
if (self.ST_ADD, path_head + "trunk/") in changes:
continue
# See whether author has permission to make changes
if author is None:
author = self._svnlook("author", "-t", txn, repos).strip()
if super_users is None:
super_users = []
for s_key in ["rosa-svn", "rosa-svn-pre-commit"]:
value = conf.get_value([s_key, "super-users"])
if value is not None:
super_users = shlex.split(value)
break
if path_head not in rev_info_map:
rev_info_map[path_head] = self._get_info(
repos, path_head)
owner, access_list = self._get_access_info(rev_info_map[path_head])
admin_users = super_users + [owner]
# Only admin users can remove the suite
if author not in admin_users and not path_tail:
bad_changes.append(BadChange(status, path))
continue
# Admin users and those in access list can modify everything in
# trunk apart from specific entries in the trunk info file
if "*" in access_list or author in admin_users + access_list:
if path_tail != self.TRUNK_INFO_FILE:
continue
else:
bad_changes.append(BadChange(status, path))
continue
# The owner must not be deleted
if path_head not in txn_info_map:
txn_info_map[path_head] = self._get_info(
repos, path_head, txn)
txn_owner, txn_access_list = self._get_access_info(
txn_info_map[path_head])
if not txn_owner:
bad_changes.append(BadChange(status, path, BadChange.NO_OWNER))
continue
# Only the admin users can change owner and access list
if owner == txn_owner and access_list == txn_access_list:
continue
if author not in admin_users:
if owner != txn_owner:
bad_changes.append(BadChange(status, path, BadChange.PERM,
"owner=" + txn_owner))
else: # access list
bad_change = BadChange(
status, path, BadChange.PERM,
"access-list=" + " ".join(txn_access_list))
bad_changes.append(bad_change)
continue
if bad_changes:
raise BadChanges(bad_changes)
__call__ = run
class RosieSvnPreCommitHook(object):
"""A pre-commit hook on a Rosie Subversion repository.
Ensure that commits conform to the rules of Rosie.
"""
IGNORES = "svnperms.conf"
RE_ID_NAMES = [r"[Ra-z]", r"[Oa-z]", r"[S\d]", r"[I\d]", r"[E\d]"]
LEN_ID = len(RE_ID_NAMES)
ST_ADD = "A"
ST_DELETE = "D"
ST_UPDATE = "U"
TRUNK_INFO_FILE = "trunk/rose-suite.info"
TRUNK_KNOWN_KEYS_FILE = "trunk/rosie-keys"
def __init__(self, event_handler=None, popen=None):
if event_handler is None:
event_handler = Reporter()
self.event_handler = event_handler
if popen is None:
popen = RosePopener(self.event_handler)
self.popen = popen
path = os.path.dirname(os.path.dirname(sys.modules["rosie"].__file__))
self.usertools_manager = SchemeHandlersManager(
[path], "rosie.usertools", ["verify_users"])
def _get_access_info(self, repos, path_head, txn=None):
"""Return the owner and the access list of a suite (path_head)."""
opt_txn = []
if txn is not None:
opt_txn = ["-t", txn]
t_handle = tempfile.TemporaryFile()
path = path_head + self.TRUNK_INFO_FILE
t_handle.write(self._svnlook("cat", repos, path, *opt_txn))
t_handle.seek(0)
node = ConfigLoader()(t_handle)
t_handle.close()
owner = node.get_value(["owner"])
access_list = node.get_value(["access-list"], "").split()
access_list.sort()
return owner, access_list
def _svnlook(self, *args):
"""Return the standard output from "svnlook"."""
command = ["svnlook"] + list(args)
return self.popen(*command, stderr=sys.stderr)[0]
def _verify_users(self, status, path, txn_owner, txn_access_list,
bad_changes):
"""Check txn_owner and txn_access_list.
For any invalid users, append to bad_changes and return True.
"""
# The owner and names in access list must be real users
conf = ResourceLocator.default().get_conf()
user_tool_name = conf.get_value(["rosa-svn", "user-tool"])
if not user_tool_name:
return False
user_tool = self.usertools_manager.get_handler(user_tool_name)
txn_users = set([txn_owner] + txn_access_list)
txn_users.discard("*")
bad_users = user_tool.verify_users(txn_users)
for bad_user in bad_users:
if txn_owner == bad_user:
bad_change = BadChange(
status,
path,
BadChange.USER,
"owner=" + bad_user)
bad_changes.append(bad_change)
if bad_user in txn_access_list:
bad_change = BadChange(
status,
path,
BadChange.USER,
"access-list=" + bad_user)
bad_changes.append(bad_change)
return bool(bad_users)
def run(self, repos, txn):
"""Apply the rule engine on transaction "txn" to repository "repos"."""
changes = set() # set([(status, path), ...])
for line in self._svnlook("changed", "-t", txn, repos).splitlines():
status, path = line.split(None, 1)
changes.add((status, path))
bad_changes = []
author = None
super_users = None
access_info_map = {} # {path-id: (owner, access-list), ...}
txn_access_info_map = {}
conf = ResourceLocator.default().get_conf()
ignores_str = conf.get_value(["rosa-svn", "ignores"], self.IGNORES)
ignores = shlex.split(ignores_str)
for status, path in sorted(changes):
if any([fnmatch(path, ignore) for ignore in ignores]):
continue
names = path.split("/", self.LEN_ID + 1)
if not names[-1]:
names.pop()
# Directories above the suites must match the ID patterns
is_bad = False
for name, pattern in zip(names, self.RE_ID_NAMES):
if not re.compile(r"\A" + pattern + r"\Z").match(name):
is_bad = True
break
if is_bad:
bad_changes.append(BadChange(status, path))
continue
# Can only add directories at levels above the suites
if len(names) < self.LEN_ID:
if status[0] != self.ST_ADD:
bad_changes.append(BadChange(status, path))
continue
else:
is_meta_suite = "".join(names[0:self.LEN_ID]) == "ROSIE"
# No need to check non-trunk changes
if len(names) > self.LEN_ID and names[self.LEN_ID] != "trunk":
continue
# New suite should have an info file
if status[0] == self.ST_ADD and len(names) == self.LEN_ID:
if (self.ST_ADD, path + "trunk/") not in changes:
bad_changes.append(
BadChange(status, path, BadChange.NO_TRUNK))
continue
path_trunk_info_file = path + self.TRUNK_INFO_FILE
if ((self.ST_ADD, path_trunk_info_file) not in changes and
(self.ST_UPDATE, path_trunk_info_file) not in changes):
bad_changes.append(
BadChange(status, path, BadChange.NO_INFO))
continue
# The rest are trunk changes in a suite
path_head = "/".join(names[0:self.LEN_ID]) + "/"
path_tail = path[len(path_head):]
# For meta suite, make sure keys in keys file can be parsed
if is_meta_suite and path_tail == self.TRUNK_KNOWN_KEYS_FILE:
out = self._svnlook("cat", "-t", txn, repos, path)
try:
shlex.split(out)
except ValueError:
bad_changes.append(
BadChange(status, path, BadChange.VALUE))
continue
# Suite trunk information file must have an owner
# User IDs of owner and access list must be real
if (status not in self.ST_DELETE and
path_tail == self.TRUNK_INFO_FILE):
txn_owner, txn_access_list = self._get_access_info(
repos, path_head, txn)
if not txn_owner:
bad_changes.append(
BadChange(status, path, BadChange.NO_OWNER))
continue
if self._verify_users(
status, path, txn_owner, txn_access_list, bad_changes):
continue
# New suite trunk: ignore the rest
if (self.ST_ADD, path_head + "trunk/") in changes:
continue
# Can only remove trunk information file with suite
if status == self.ST_DELETE and path_tail == self.TRUNK_INFO_FILE:
if (self.ST_DELETE, path_head) not in changes:
bad_changes.append(
BadChange(status, path, BadChange.NO_INFO))
continue
# Can only remove trunk with suite
if status == self.ST_DELETE and path_tail == "trunk/":
if (self.ST_DELETE, path_head) not in changes:
bad_changes.append(
BadChange(status, path, BadChange.NO_TRUNK))
continue
# See whether author has permission to make changes
if author is None:
author = self._svnlook("author", "-t", txn, repos).strip()
if super_users is None:
super_users = []
for s_key in ["rosa-svn", "rosa-svn-pre-commit"]:
value = conf.get_value([s_key, "super-users"])
if value is not None:
super_users = shlex.split(value)
break
if path_head not in access_info_map:
access_info = self._get_access_info(repos, path_head)
access_info_map[path_head] = access_info
owner, access_list = access_info_map[path_head]
admin_users = super_users + [owner]
# Only admin users can remove the suite
if author not in admin_users and not path_tail:
bad_changes.append(BadChange(status, path))
continue
# Admin users and those in access list can modify everything in
# trunk apart from specific entries in the trunk info file
if "*" in access_list or author in admin_users + access_list:
if path_tail != self.TRUNK_INFO_FILE:
continue
else:
bad_changes.append(BadChange(status, path))
continue
# The owner must not be deleted
if path_head not in txn_access_info_map:
txn_access_info = self._get_access_info(repos, path_head, txn)
txn_access_info_map[path_head] = txn_access_info
txn_owner, txn_access_list = txn_access_info_map[path_head]
if not txn_owner:
bad_changes.append(BadChange(status, path, BadChange.NO_OWNER))
continue
# Only the admin users can change owner and access list
if owner == txn_owner and access_list == txn_access_list:
continue
if author not in admin_users:
if owner != txn_owner:
bad_changes.append(BadChange(status, path, BadChange.PERM,
"owner=" + txn_owner))
else: # access list
bad_change = BadChange(
status, path, BadChange.PERM,
"access-list=" + " ".join(txn_access_list))
bad_changes.append(bad_change)
continue
if bad_changes:
raise BadChanges(bad_changes)
__call__ = run
def _run(self, dao, app_runner, config):
"""Transform and archive suite files.
This application is designed to work under "rose task-run" in a suite.
"""
path = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__))
compress_manager = SchemeHandlersManager(
[path], "rose.apps.rose_arch_compressions", ["compress_sources"],
None, app_runner)
# Set up the targets
cycle = os.getenv("ROSE_TASK_CYCLE_TIME")
targets = []
for t_key, t_node in sorted(config.value.items()):
if t_node.is_ignored() or ":" not in t_key:
continue
s_key_head, s_key_tail = t_key.split(":", 1)
if s_key_head != self.SECTION or not s_key_tail:
continue
target_prefix = self._get_conf(config,
t_node,
"target-prefix",
default="")
try:
s_key_tail = env_var_process(s_key_tail)
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError([t_key, ""], "", exc)
target_name = target_prefix + s_key_tail
target = RoseArchTarget(target_name)
target.command_format = self._get_conf(config,
t_node,
"command-format",
compulsory=True)
try:
target.command_format % {"sources": "", "target": ""}
except KeyError as exc:
target.status = target.ST_BAD
app_runner.handle_event(
RoseArchValueError(target.name, "command-format",
target.command_format,
type(exc).__name__, exc))
source_str = self._get_conf(config,
t_node,
"source",
compulsory=True)
source_prefix = self._get_conf(config,
t_node,
"source-prefix",
default="")
target.source_edit_format = self._get_conf(config,
t_node,
"source-edit-format",
default="")
try:
target.source_edit_format % {"in": "", "out": ""}
except KeyError as exc:
target.status = target.ST_BAD
app_runner.handle_event(
RoseArchValueError(target.name, "source-edit-format",
target.source_edit_format,
type(exc).__name__, exc))
update_check_str = self._get_conf(config,
t_node,
"update-check",
default="md5sum")
try:
checksum_func = get_checksum_func(update_check_str)
except KeyError as exc:
raise RoseArchValueError(target.name, "update-check",
update_check_str,
type(exc).__name__, exc)
for source_glob in shlex.split(source_str):
paths = glob(source_prefix + source_glob)
if not paths:
exc = OSError(errno.ENOENT, os.strerror(errno.ENOENT),
source_glob)
app_runner.handle_event(
ConfigValueError([t_key, "source"], source_glob, exc))
target.status = target.ST_BAD
continue
for path in paths:
# N.B. source_prefix may not be a directory
name = path[len(source_prefix):]
for path_, checksum, _ in get_checksum(
path, checksum_func):
if checksum is None: # is directory
continue
if path_:
target.sources[checksum] = RoseArchSource(
checksum, os.path.join(name, path_),
os.path.join(path, path_))
else: # path is a file
target.sources[checksum] = RoseArchSource(
checksum, name, path)
target.compress_scheme = self._get_conf(config, t_node, "compress")
if target.compress_scheme:
if (compress_manager.get_handler(target.compress_scheme) is
None):
app_runner.handle_event(
ConfigValueError([t_key, "compress"],
target.compress_scheme,
KeyError(target.compress_scheme)))
target.status = target.ST_BAD
else:
target_base = target.name
if "/" in target.name:
target_base = target.name.rsplit("/", 1)[1]
if "." in target_base:
tail = target_base.split(".", 1)[1]
if compress_manager.get_handler(tail):
target.compress_scheme = tail
rename_format = self._get_conf(config, t_node, "rename-format")
if rename_format:
rename_parser_str = self._get_conf(config, t_node,
"rename-parser")
if rename_parser_str:
try:
rename_parser = re.compile(rename_parser_str)
except re.error as exc:
raise RoseArchValueError(target.name, "rename-parser",
rename_parser_str,
type(exc).__name__, exc)
else:
rename_parser = None
for source in target.sources.values():
dict_ = {"cycle": cycle, "name": source.name}
if rename_parser:
match = rename_parser.match(source.name)
if match:
dict_.update(match.groupdict())
try:
source.name = rename_format % dict_
except (KeyError, ValueError) as exc:
raise RoseArchValueError(target.name, "rename-format",
rename_format,
type(exc).__name__, exc)
old_target = dao.select(target.name)
if old_target is None or old_target != target:
dao.delete(target)
else:
target.status = target.ST_OLD
targets.append(target)
# Delete from database items that are no longer relevant
dao.delete_all(filter_targets=targets)
# Update the targets
for target in targets:
if target.status == target.ST_OLD:
app_runner.handle_event(RoseArchEvent(target))
continue
target.command_rc = 1
dao.insert(target)
if target.status == target.ST_BAD:
app_runner.handle_event(RoseArchEvent(target))
continue
work_dir = mkdtemp()
t_init = time()
t_tran, t_arch = t_init, t_init
ret_code = None
try:
# Rename/edit sources
target.status = target.ST_BAD
rename_required = False
for source in target.sources.values():
if source.name != source.orig_name:
rename_required = True
break
if rename_required or target.source_edit_format:
for source in target.sources.values():
source.path = os.path.join(work_dir, source.name)
source_path_d = os.path.dirname(source.path)
app_runner.fs_util.makedirs(source_path_d)
if target.source_edit_format:
fmt_args = {
"in": source.orig_path,
"out": source.path
}
command = target.source_edit_format % fmt_args
app_runner.popen.run_ok(command, shell=True)
else:
app_runner.fs_util.symlink(source.orig_path,
source.path)
# Compress sources
if target.compress_scheme:
handler = compress_manager.get_handler(
target.compress_scheme)
handler.compress_sources(target, work_dir)
t_tran = time()
# Run archive command
sources = []
if target.work_source_path:
sources = [target.work_source_path]
else:
for source in target.sources.values():
sources.append(source.path)
sources_str = app_runner.popen.list_to_shell_str(sources)
target_str = app_runner.popen.list_to_shell_str([target.name])
command = target.command_format % {
"sources": sources_str,
"target": target_str
}
ret_code, out, err = app_runner.popen.run(command, shell=True)
t_arch = time()
if ret_code:
app_runner.handle_event(
RosePopenError([command], ret_code, out, err))
else:
target.status = target.ST_NEW
app_runner.handle_event(err, kind=Event.KIND_ERR)
app_runner.handle_event(out)
target.command_rc = ret_code
dao.update_command_rc(target)
finally:
app_runner.fs_util.delete(work_dir)
app_runner.handle_event(
RoseArchEvent(target, [t_init, t_tran, t_arch], ret_code))
return [target.status
for target in targets].count(RoseArchTarget.ST_BAD)
class AppRunner(Runner):
"""Invoke a Rose application."""
NAME = "app"
OPTIONS = ["app_mode", "command_key", "conf_dir", "defines",
"install_only_mode", "new_mode", "no_overwrite_mode",
"opt_conf_keys"]
def __init__(self, *args, **kwargs):
Runner.__init__(self, *args, **kwargs)
path = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__))
self.builtins_manager = SchemeHandlersManager(
[path], "rose.apps", ["run"], None, *args, **kwargs)
self.date_time_oper = RoseDateTimeOperator()
def run_impl(self, opts, args, uuid, work_files):
"""The actual logic for a run."""
# Preparation.
conf_tree = self.config_load(opts)
self._prep(conf_tree, opts)
self._poll(conf_tree)
# Run the application or the command.
app_mode = conf_tree.node.get_value(["mode"])
if app_mode is None:
app_mode = opts.app_mode
if app_mode is None:
app_mode = os.getenv("ROSE_APP_MODE")
if app_mode in [None, "command"]:
return self._command(conf_tree, opts, args)
else:
builtin_app = self.builtins_manager.get_handler(app_mode)
if builtin_app is None:
raise UnknownBuiltinAppError(app_mode)
return builtin_app.run(self, conf_tree, opts, args, uuid,
work_files)
def get_command(self, conf_tree, opts, args):
"""Get command to run."""
command = self.popen.list_to_shell_str(args)
if not command:
names = [opts.command_key, os.getenv("ROSE_APP_COMMAND_KEY"),
os.getenv("ROSE_TASK_NAME"), "default"]
for name in names:
if not name:
continue
command = conf_tree.node.get_value(["command", name])
if command is not None:
break
return command
def _prep(self, conf_tree, opts):
"""Prepare to run the application."""
if opts.new_mode:
self._prep_new(opts)
# Dump the actual configuration as rose-app-run.conf
ConfigDumper()(conf_tree.node, "rose-app-run.conf")
# Environment variables: PATH
self._prep_path(conf_tree)
# Free format files not defined in the configuration file
file_section_prefix = self.config_pm.get_handler("file").PREFIX
for rel_path, conf_dir in conf_tree.files.items():
if not rel_path.startswith("file" + os.sep):
continue
name = rel_path[len("file" + os.sep):]
# No sub-directories, very slow otherwise
if os.sep in name:
name = name.split(os.sep, 1)[0]
target_key = file_section_prefix + name
target_node = conf_tree.node.get([target_key])
if target_node is None:
conf_tree.node.set([target_key])
target_node = conf_tree.node.get([target_key])
elif target_node.is_ignored():
continue
source_node = target_node.get(["source"])
if source_node is None:
target_node.set(["source"],
os.path.join(conf_dir, "file", name))
elif source_node.is_ignored():
continue
# Process Environment Variables
self.config_pm(conf_tree, "env")
# Process Files
self.config_pm(conf_tree, "file",
no_overwrite_mode=opts.no_overwrite_mode)
def _prep_new(self, opts):
"""Clear out run directory on a --new option if possible."""
conf_dir = opts.conf_dir
if not conf_dir or os.path.abspath(conf_dir) == os.getcwd():
raise NewModeError(os.getcwd())
for path in os.listdir("."):
self.fs_util.delete(path)
@staticmethod
def _prep_path(conf_tree):
"""Add bin directories to the PATH seen by the app command."""
paths = []
for conf_dir in conf_tree.conf_dirs:
conf_bin_dir = os.path.join(conf_dir, "bin")
if os.path.isdir(conf_bin_dir):
paths.append(conf_bin_dir)
if paths:
value = os.pathsep.join(paths + [os.getenv("PATH")])
conf_tree.node.set(["env", "PATH"], value)
else:
conf_tree.node.set(["env", "PATH"], os.getenv("PATH"))
def _poll(self, conf_tree):
"""Run any configured file polling."""
poller = Poller(self.popen, self.handle_event)
poller.poll(conf_tree)
def _command(self, conf_tree, opts, args):
"""Run the command."""
command = self.get_command(conf_tree, opts, args)
if not command:
self.handle_event(CommandNotDefinedEvent())
return
if os.access("STDIN", os.F_OK | os.R_OK):
command += " <STDIN"
self.handle_event("command: %s" % command)
if opts.install_only_mode:
return
self.popen(
command, shell=True, stdout=sys.stdout, stderr=sys.stderr,
stdin=sys.stdin)
class AppRunner(Runner):
"""Invoke a Rose application."""
OLD_DURATION_UNITS = {"h": 3600, "m": 60, "s": 1}
NAME = "app"
OPTIONS = [
"app_mode", "command_key", "conf_dir", "defines", "install_only_mode",
"new_mode", "no_overwrite_mode", "opt_conf_keys"
]
def __init__(self, *args, **kwargs):
Runner.__init__(self, *args, **kwargs)
path = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__))
self.builtins_manager = SchemeHandlersManager([path], "rose.apps",
["run"], None, *args,
**kwargs)
self.duration_parser = DurationParser()
def run_impl(self, opts, args, uuid, work_files):
"""The actual logic for a run."""
# Preparation.
conf_tree = self.config_load(opts)
self._prep(conf_tree, opts)
self._poll(conf_tree)
# Run the application or the command.
app_mode = conf_tree.node.get_value(["mode"])
if app_mode is None:
app_mode = opts.app_mode
if app_mode in [None, "command"]:
return self._command(conf_tree, opts, args)
else:
builtin_app = self.builtins_manager.get_handler(app_mode)
if builtin_app is None:
raise UnknownBuiltinAppError(app_mode)
return builtin_app.run(self, conf_tree, opts, args, uuid,
work_files)
def _poll(self, conf_tree):
"""Poll for prerequisites of applications."""
# Poll configuration
poll_test = conf_tree.node.get_value(["poll", "test"])
poll_all_files_value = conf_tree.node.get_value(["poll", "all-files"])
poll_all_files = []
if poll_all_files_value:
try:
poll_all_files = shlex.split(
env_var_process(poll_all_files_value))
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError(["poll", "all-files"],
poll_all_files_value, exc)
poll_any_files_value = conf_tree.node.get_value(["poll", "any-files"])
poll_any_files = []
if poll_any_files_value:
try:
poll_any_files = shlex.split(
env_var_process(poll_any_files_value))
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError(["poll", "any-files"],
poll_any_files_value, exc)
poll_file_test = None
if poll_all_files or poll_any_files:
poll_file_test = conf_tree.node.get_value(["poll", "file-test"])
if poll_file_test and "{}" not in poll_file_test:
raise ConfigValueError(["poll", "file-test"], poll_file_test,
ConfigValueError.SYNTAX)
poll_delays = []
if poll_test or poll_all_files or poll_any_files:
# Parse something like this: delays=10,4*PT30S,PT2M30S,2*PT1H
# R*DURATION: repeat the value R times
conf_keys = ["poll", "delays"]
poll_delays_value = conf_tree.node.get_value(conf_keys,
default="").strip()
if poll_delays_value:
is_legacy0 = None
for item in poll_delays_value.split(","):
value = item.strip()
repeat = 1
if "*" in value:
repeat, value = value.split("*", 1)
try:
repeat = int(repeat)
except ValueError as exc:
raise ConfigValueError(conf_keys,
poll_delays_value,
ConfigValueError.SYNTAX)
try:
value = self.duration_parser.parse(value).get_seconds()
is_legacy = False
except ISO8601SyntaxError:
# Legacy mode: nnnU
# nnn is a float, U is the unit
# No unit or s: seconds
# m: minutes
# h: hours
unit = None
if value[-1].lower() in self.OLD_DURATION_UNITS:
unit = self.OLD_DURATION_UNITS[value[-1].lower()]
value = value[:-1]
try:
value = float(value)
except ValueError as exc:
raise ConfigValueError(conf_keys,
poll_delays_value,
ConfigValueError.SYNTAX)
if unit:
value *= unit
is_legacy = True
if is_legacy0 is None:
is_legacy0 = is_legacy
elif is_legacy0 != is_legacy:
raise ConfigValueError(
conf_keys, poll_delays_value,
ConfigValueError.DURATION_LEGACY_MIX)
poll_delays += [value] * repeat
else:
poll_delays = [0] # poll once without a delay
# Poll
t_init = get_timepoint_for_now()
while poll_delays and (poll_test or poll_any_files or poll_all_files):
poll_delay = poll_delays.pop(0)
if poll_delay:
sleep(poll_delay)
if poll_test:
ret_code = self.popen.run(poll_test,
shell=True,
stdout=sys.stdout,
stderr=sys.stderr)[0]
self.handle_event(PollEvent(time(), poll_test, ret_code == 0))
if ret_code == 0:
poll_test = None
any_files = list(poll_any_files)
for file_ in any_files:
if self._poll_file(file_, poll_file_test):
self.handle_event(PollEvent(time(), "any-files", True))
poll_any_files = []
break
all_files = list(poll_all_files)
for file_ in all_files:
if self._poll_file(file_, poll_file_test):
poll_all_files.remove(file_)
if all_files and not poll_all_files:
self.handle_event(PollEvent(time(), "all-files", True))
failed_items = []
if poll_test:
failed_items.append("test")
if poll_any_files:
failed_items.append("any-files")
if poll_all_files:
failed_items.append("all-files:" +
self.popen.list_to_shell_str(poll_all_files))
if failed_items:
now = get_timepoint_for_now()
raise PollTimeoutError(now, now - t_init, failed_items)
def _poll_file(self, file_, poll_file_test):
"""Poll for existence of a file."""
is_done = False
if poll_file_test:
test = poll_file_test.replace(
"{}", self.popen.list_to_shell_str([file_]))
is_done = self.popen.run(test,
shell=True,
stdout=sys.stdout,
stderr=sys.stderr)[0] == 0
else:
is_done = bool(glob(file_))
self.handle_event(PollEvent(time(), "file:" + file_, is_done))
return is_done
def _prep(self, conf_tree, opts):
"""Prepare to run the application."""
if opts.new_mode:
conf_dir = opts.conf_dir
if not conf_dir or os.path.abspath(conf_dir) == os.getcwd():
raise NewModeError(os.getcwd())
for path in os.listdir("."):
self.fs_util.delete(path)
# Dump the actual configuration as rose-app-run.conf
ConfigDumper()(conf_tree.node, "rose-app-run.conf")
# Environment variables: PATH
paths = []
for conf_dir in conf_tree.conf_dirs:
conf_bin_dir = os.path.join(conf_dir, "bin")
if os.path.isdir(conf_bin_dir):
paths.append(conf_bin_dir)
if paths:
value = os.pathsep.join(paths + [os.getenv("PATH")])
conf_tree.node.set(["env", "PATH"], value)
else:
conf_tree.node.set(["env", "PATH"], os.getenv("PATH"))
# Free format files not defined in the configuration file
file_section_prefix = self.config_pm.get_handler("file").PREFIX
for rel_path, conf_dir in conf_tree.files.items():
if not rel_path.startswith("file" + os.sep):
continue
name = rel_path[len("file" + os.sep):]
# No sub-directories, very slow otherwise
if os.sep in name:
name = name.split(os.sep, 1)[0]
target_key = file_section_prefix + name
target_node = conf_tree.node.get([target_key])
if target_node is None:
conf_tree.node.set([target_key])
target_node = conf_tree.node.get([target_key])
elif target_node.is_ignored():
continue
source_node = target_node.get("source")
if source_node is None:
target_node.set(["source"],
os.path.join(conf_dir, "file", name))
elif source_node.is_ignored():
continue
# Process Environment Variables
self.config_pm(conf_tree, "env")
# Process Files
self.config_pm(conf_tree,
"file",
no_overwrite_mode=opts.no_overwrite_mode)
def _command(self, conf_tree, opts, args):
"""Run the command."""
command = self.popen.list_to_shell_str(args)
if not command:
names = [opts.command_key, os.getenv("ROSE_TASK_NAME"), "default"]
for name in names:
if not name:
continue
command = conf_tree.node.get_value(["command", name])
if command is not None:
break
else:
self.handle_event(CommandNotDefinedEvent())
return
if os.access("STDIN", os.F_OK | os.R_OK):
command += " <STDIN"
self.handle_event("command: %s" % command)
if opts.install_only_mode:
return
self.popen(command, shell=True, stdout=sys.stdout, stderr=sys.stderr)
class AppRunner(Runner):
"""Invoke a Rose application."""
NAME = "app"
OPTIONS = ["app_mode", "command_key", "conf_dir", "defines",
"install_only_mode", "new_mode", "no_overwrite_mode",
"opt_conf_keys"]
def __init__(self, *args, **kwargs):
Runner.__init__(self, *args, **kwargs)
p = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__))
self.builtins_manager = SchemeHandlersManager(
[p], "rose.apps", ["run"], None, *args, **kwargs)
def run_impl(self, opts, args, uuid, work_files):
"""The actual logic for a run."""
# Preparation.
conf_tree = self.config_load(opts)
self._prep(conf_tree, opts, args, uuid, work_files)
self._poll(conf_tree, opts, args, uuid, work_files)
# Run the application or the command.
app_mode = conf_tree.node.get_value(["mode"])
if app_mode is None:
app_mode = opts.app_mode
if app_mode in [None, "command"]:
return self._command(conf_tree, opts, args, uuid, work_files)
else:
builtin_app = self.builtins_manager.get_handler(app_mode)
if builtin_app is None:
raise UnknownBuiltinAppError(app_mode)
return builtin_app.run(self, conf_tree, opts, args, uuid,
work_files)
def _poll(self, conf_tree, opts, args, uuid, work_files):
"""Poll for prerequisites of applications."""
# Poll configuration
poll_test = conf_tree.node.get_value(["poll", "test"])
poll_all_files_value = conf_tree.node.get_value(["poll", "all-files"])
poll_all_files = []
if poll_all_files_value:
try:
poll_all_files = shlex.split(
env_var_process(poll_all_files_value))
except UnboundEnvironmentVariableError as e:
raise ConfigValueError(["poll", "all-files"],
poll_all_files_value, e)
poll_any_files_value = conf_tree.node.get_value(["poll", "any-files"])
poll_any_files = []
if poll_any_files_value:
try:
poll_any_files = shlex.split(
env_var_process(poll_any_files_value))
except UnboundEnvironmentVariableError as e:
raise ConfigValueError(["poll", "any-files"],
poll_any_files_value, e)
poll_file_test = None
if poll_all_files or poll_any_files:
poll_file_test = conf_tree.node.get_value(["poll", "file-test"])
if poll_file_test and "{}" not in poll_file_test:
raise ConfigValueError(["poll", "file-test"], poll_file_test,
ConfigValueError.SYNTAX)
poll_delays = []
if poll_test or poll_all_files or poll_any_files:
# Parse something like this: delays=10,4*30s,2.5m,2*1h
# No unit or s: seconds
# m: minutes
# h: hours
# N*: repeat the value N times
poll_delays_value = conf_tree.node.get_value(["poll", "delays"],
default="")
poll_delays_value = poll_delays_value.strip()
units = {"h": 3600, "m": 60, "s": 1}
if poll_delays_value:
for item in poll_delays_value.split(","):
value = item.strip()
repeat = 1
if "*" in value:
repeat, value = value.split("*", 1)
try:
repeat = int(repeat)
except ValueError as e:
raise ConfigValueError(["poll", "delays"],
poll_delays_value,
ConfigValueError.SYNTAX)
unit = None
if value[-1].lower() in units.keys():
unit = units[value[-1]]
value = value[:-1]
try:
value = float(value)
except ValueError as e:
raise ConfigValueError(["poll", "delays"],
poll_delays_value,
ConfigValueError.SYNTAX)
if unit:
value *= unit
for i in range(repeat):
poll_delays.append(value)
else:
poll_delays = [0] # poll once without a delay
# Poll
t_init = time()
while poll_delays and (poll_test or poll_any_files or poll_all_files):
poll_delay = poll_delays.pop(0)
if poll_delay:
sleep(poll_delay)
if poll_test:
rc, out, err = self.popen.run(poll_test, shell=True,
stdout=sys.stdout,
stderr=sys.stderr)
self.handle_event(PollEvent(time(), poll_test, rc == 0))
if rc == 0:
poll_test = None
any_files = list(poll_any_files)
for file in any_files:
if self._poll_file(file, poll_file_test):
self.handle_event(PollEvent(time(), "any-files", True))
poll_any_files = []
break
all_files = list(poll_all_files)
for file in all_files:
if self._poll_file(file, poll_file_test):
poll_all_files.remove(file)
if all_files and not poll_all_files:
self.handle_event(PollEvent(time(), "all-files", True))
failed_items = []
if poll_test:
failed_items.append("test")
if poll_any_files:
failed_items.append("any-files")
if poll_all_files:
failed_items.append("all-files:" +
self.popen.list_to_shell_str(poll_all_files))
if failed_items:
now = time()
raise PollTimeoutError(now, now - t_init, failed_items)
def _poll_file(self, file, poll_file_test):
ok = False
if poll_file_test:
test = poll_file_test.replace(
"{}", self.popen.list_to_shell_str([file]))
rc, out, err = self.popen.run(test, shell=True,
stdout=sys.stdout, stderr=sys.stderr)
ok = rc == 0
else:
ok = os.path.exists(file)
self.handle_event(PollEvent(time(), "file:" + file, ok))
return ok
def _prep(self, conf_tree, opts, args, uuid, work_files):
"""Prepare to run the application."""
if opts.new_mode:
conf_dir = opts.conf_dir
if not conf_dir or os.path.abspath(conf_dir) == os.getcwd():
raise NewModeError(os.getcwd())
for p in os.listdir("."):
self.fs_util.delete(p)
# Dump the actual configuration as rose-app-run.conf
ConfigDumper()(conf_tree.node, "rose-app-run.conf")
# Environment variables: PATH
paths = []
for conf_dir in conf_tree.conf_dirs:
conf_bin_dir = os.path.join(conf_dir, "bin")
if os.path.isdir(conf_bin_dir):
paths.append(conf_bin_dir)
if paths:
value = os.pathsep.join(paths + [os.getenv("PATH")])
conf_tree.node.set(["env", "PATH"], value)
else:
conf_tree.node.set(["env", "PATH"], os.getenv("PATH"))
# Free format files not defined in the configuration file
file_section_prefix = self.config_pm.get_handler("file").PREFIX
for rel_path, conf_dir in conf_tree.files.items():
if not rel_path.startswith("file" + os.sep):
continue
name = rel_path[len("file" + os.sep):]
section = file_section_prefix + name
if conf_tree.node.get([section], no_ignore=True) is None:
conf_tree.node.set([section, "source"],
os.path.join(conf_dir, rel_path))
# Process Environment Variables
self.config_pm(conf_tree, "env")
# Process Files
self.config_pm(conf_tree, "file",
no_overwrite_mode=opts.no_overwrite_mode)
def _command(self, conf_tree, opts, args, uuid, work_files):
"""Run the command."""
command = self.popen.list_to_shell_str(args)
if not command:
names = [opts.command_key, os.getenv("ROSE_TASK_NAME"), "default"]
for name in names:
if not name:
continue
command = conf_tree.node.get_value(["command", name])
if command is not None:
break
else:
self.handle_event(CommandNotDefinedEvent())
return
if os.access("STDIN", os.F_OK | os.R_OK):
command += " <STDIN"
self.handle_event("command: %s" % command)
if opts.install_only_mode:
return
# TODO: allow caller of app_run to specify stdout and stderr?
self.popen(command, shell=True, stdout=sys.stdout, stderr=sys.stderr)