def listFilterHooks(db, user):
cursor = db.cursor()
installs = Extension.getInstalls(db, user)
filterhooks = []
for extension_id, version_id, version_sha1, is_universal in installs:
if version_id is not None:
cursor.execute(
"""SELECT 1
FROM extensionroles
JOIN extensionfilterhookroles ON (role=id)
WHERE version=%s""", (version_id, ))
if not cursor.fetchone():
continue
extension = Extension.fromId(db, extension_id)
manifest = extension.getManifest(sha1=version_sha1)
else:
try:
extension = Extension.fromId(db, extension_id)
except ExtensionError:
# If the author/hosting user no longer exists, or the extension
# directory no longer exists or is inaccessible, ignore the
# extension.
continue
try:
manifest = Manifest.load(extension.getPath())
except ManifestError:
# If the MANIFEST is missing or invalid, we can't know whether
# the extension has any filter hook roles, so assume it doesn't
# and ignore it.
continue
if not any(
isinstance(role, FilterHookRole)
for role in manifest.roles):
continue
filterhooks.append((extension, manifest,
sorted((role for role in manifest.roles
if isinstance(role, FilterHookRole)),
key=lambda role: role.title)))
return sorted(filterhooks,
key=lambda (extension, manifest, roles): extension.getKey())
def listFilterHooks(db, user):
cursor = db.cursor()
installs = Extension.getInstalls(db, user)
filterhooks = []
for extension_id, version_id, version_sha1, is_universal in installs:
if version_id is not None:
cursor.execute("""SELECT 1
FROM extensionroles
JOIN extensionfilterhookroles ON (role=id)
WHERE version=%s""",
(version_id,))
if not cursor.fetchone():
continue
extension = Extension.fromId(db, extension_id)
manifest = extension.getManifest(sha1=version_sha1)
else:
try:
extension = Extension.fromId(db, extension_id)
except ExtensionError:
# If the author/hosting user no longer exists, or the extension
# directory no longer exists or is inaccessible, ignore the
# extension.
continue
try:
manifest = Manifest.load(extension.getPath())
except ManifestError:
# If the MANIFEST is missing or invalid, we can't know whether
# the extension has any filter hook roles, so assume it doesn't
# and ignore it.
continue
if not any(isinstance(role, FilterHookRole)
for role in manifest.roles):
continue
filterhooks.append((extension, manifest, sorted(
(role for role in manifest.roles
if isinstance(role, FilterHookRole)),
key=lambda role: role.title)))
return sorted(filterhooks,
key=lambda (extension, manifest, roles): extension.getKey())
def getExtension(author_name, extension_name):
"""Create an Extension object ignoring whether it is valid"""
try:
return Extension(author_name, extension_name)
except ExtensionError as error:
if error.extension is None:
raise error
return error.extension
def getFilterHookRole(db, filter_id):
cursor = db.cursor()
cursor.execute("""SELECT extension, uid, name
FROM extensionhookfilters
WHERE id=%s""",
(filter_id,))
extension_id, user_id, filterhook_name = cursor.fetchone()
extension = Extension.fromId(db, extension_id)
user = dbutils.User.fromId(db, user_id)
installed_sha1, _ = extension.getInstalledVersion(db, user)
if installed_sha1 is False:
return
manifest = extension.getManifest(sha1=installed_sha1)
for role in manifest.roles:
if isinstance(role, FilterHookRole) and role.name == filterhook_name:
return role
def getFilterHookRole(db, filter_id):
cursor = db.cursor()
cursor.execute(
"""SELECT extension, uid, name
FROM extensionhookfilters
WHERE id=%s""", (filter_id, ))
extension_id, user_id, filterhook_name = cursor.fetchone()
extension = Extension.fromId(db, extension_id)
user = dbutils.User.fromId(db, user_id)
installed_sha1, _ = extension.getInstalledVersion(db, user)
if installed_sha1 is False:
return
manifest = extension.getManifest(sha1=installed_sha1)
for role in manifest.roles:
if isinstance(role, FilterHookRole) and role.name == filterhook_name:
return role
def processFilterHookEvent(db, event_id, logfn):
cursor = db.cursor()
cursor.execute("""SELECT filters.extension, filters.uid, filters.path,
filters.name, events.review, events.uid, events.data
FROM extensionfilterhookevents AS events
JOIN extensionhookfilters AS filters ON (filters.id=events.filter)
WHERE events.id=%s""",
(event_id,))
# Note:
# - filter_user_id / filter_user represent the user whose filter was
# triggered.
# - user_id /user represent the user that added commits and thereby
# triggered the filter.
(extension_id, filter_user_id, filter_path,
filterhook_name, review_id, user_id, filter_data) = cursor.fetchone()
extension = Extension.fromId(db, extension_id)
filter_user = dbutils.User.fromId(db, filter_user_id)
installed_sha1, _ = extension.getInstalledVersion(db, filter_user)
if installed_sha1 is False:
# Invalid event (user doesn't have extension installed); do nothing.
# The event will be deleted by the caller.
return
manifest = extension.getManifest(sha1=installed_sha1)
for role in manifest.roles:
if isinstance(role, FilterHookRole) and role.name == filterhook_name:
break
else:
# Invalid event (installed version of extension doesn't have the named
# filter hook role); do nothing. The event will be deleted by the
# caller.
return
cursor.execute("""SELECT commit
FROM extensionfilterhookcommits
WHERE event=%s""",
(event_id,))
commit_ids = [commit_id for (commit_id,) in cursor]
cursor.execute("""SELECT file
FROM extensionfilterhookfiles
WHERE event=%s""",
(event_id,))
file_ids = [file_id for (file_id,) in cursor]
argv = """
(function () {
var review = new critic.Review(%(review_id)d);
var user = new critic.User(%(user_id)d);
var repository = review.repository;
var commits = new critic.CommitSet(
%(commit_ids)r.map(
function (commit_id) {
return repository.getCommit(commit_id);
}));
var files = %(file_ids)r.map(
function (file_id) {
return critic.File.find(file_id);
});
return [%(filter_data)s, review, user, commits, files];
})()
""" % { "filter_data": htmlutils.jsify(filter_data),
"review_id": review_id,
"user_id": user_id,
"commit_ids": commit_ids,
"file_ids": file_ids }
argv = re.sub("[ \n]+", " ", argv.strip())
logfn("argv=%r" % argv)
logfn("script=%r" % role.script)
logfn("function=%r" % role.function)
try:
executeProcess(
manifest, "filterhook", role.script, role.function, extension_id,
filter_user_id, argv, configuration.extensions.LONG_TIMEOUT)
except (ProcessTimeout, ProcessError) as error:
review = dbutils.Review.fromId(db, review_id)
recipients = set([filter_user])
author = extension.getAuthor(db)
if author is None:
recipients.update(dbutils.User.withRole(db, "administrator"))
else:
recipients.add(author)
body = """\
An error occurred while processing an extension hook filter event!
Filter details:
Extension: %(extension.title)s
Filter hook: %(role.title)s
Repository: %(repository.name)s
Path: %(filter.path)s
Data: %(filter.data)s
Event details:
Review: r/%(review.id)d "%(review.summary)s"
Commits: %(commits)s
Error details:
Error: %(error.message)s
Output:%(error.output)s
-- critic"""
commits = (gitutils.Commit.fromId(db, review.repository, commit_id)
for commit_id in commit_ids)
commits_text = "\n ".join(
('%s "%s"' % (commit.sha1[:8], commit.niceSummary())
for commit in commits))
if isinstance(error, ProcessTimeout):
error_output = " N/A"
else:
error_output = "\n\n " + "\n ".join(error.stderr.splitlines())
body = body % { "extension.title": extension.getTitle(db),
"role.title": role.title,
"repository.name": review.repository.name,
"filter.path": filter_path,
"filter.data": htmlutils.jsify(filter_data),
"review.id": review.id,
"review.summary": review.summary,
"commits": commits_text,
"error.message": error.message,
"error.output": error_output }
mailutils.sendMessage(
recipients=list(recipients),
subject="Failed: " + role.title,
body=body)
def installExtension(db, user, author_name, extension_name, version):
doInstallExtension(db, user, Extension(author_name, extension_name),
version)
db.commit()
def execute(db, user, review, all_commits, old_head, new_head, output):
cursor = db.cursor()
installs = Extension.getInstalls(db, user)
data = None
for extension_id, version_id, version_sha1, is_universal in installs:
handlers = []
extension = Extension.fromId(db, extension_id)
if version_id is not None:
cursor.execute("""SELECT script, function
FROM extensionroles
JOIN extensionprocesscommitsroles ON (role=id)
WHERE version=%s
ORDER BY id ASC""",
(version_id,))
handlers.extend(cursor)
if not handlers:
continue
extension_path = getExtensionInstallPath(version_sha1)
manifest = Manifest.load(extension_path)
else:
manifest = Manifest.load(extension.getPath())
for role in manifest.roles:
if isinstance(role, ProcessCommitsRole):
handlers.append((role.script, role.function))
if not handlers:
continue
if data is None:
commitset = log.commitset.CommitSet(all_commits)
assert old_head is None or old_head in commitset.getTails()
assert new_head in commitset.getHeads()
assert len(commitset.getHeads()) == 1
tails = commitset.getFilteredTails(review.repository)
if len(tails) == 1:
tail = gitutils.Commit.fromSHA1(db, review.repository, tails.pop())
changeset_id = changeset.utils.createChangeset(
db, user, review.repository, from_commit=tail, to_commit=new_head)[0].id
changeset_arg = "repository.getChangeset(%d)" % changeset_id
else:
changeset_arg = "null"
commits_arg = "[%s]" % ",".join(
[("repository.getCommit(%d)" % commit.getId(db))
for commit in all_commits])
data = { "review_id": review.id,
"changeset": changeset_arg,
"commits": commits_arg }
for script, function in handlers:
class Error(Exception):
pass
def print_header():
header = "%s::%s()" % (script, function)
print >>output, ("\n[%s] %s\n[%s] %s"
% (extension.getName(), header,
extension.getName(), "=" * len(header)))
try:
argv = """
(function ()
{
var review = new critic.Review(%(review_id)d);
var repository = review.repository;
var changeset = %(changeset)s;
var commitset = new critic.CommitSet(%(commits)s);
return [review, changeset, commitset];
})()
""" % data
argv = re.sub("[ \n]+", " ", argv.strip())
try:
stdout_data = executeProcess(
manifest, "processcommits", script, function, extension_id, user.id,
argv, configuration.extensions.SHORT_TIMEOUT)
except ProcessTimeout:
raise Error("Timeout after %d seconds." % configuration.extensions.SHORT_TIMEOUT)
except ProcessError as error:
if error.returncode < 0:
raise Error("Process terminated by signal %d." % -error.returncode)
else:
raise Error("Process returned %d.\n%s" % (error.returncode, error.stderr))
if stdout_data.strip():
print_header()
for line in stdout_data.splitlines():
print >>output, "[%s] %s" % (extension.getName(), line)
except Error as error:
print_header()
print >>output, "[%s] Extension error: %s" % (extension.getName(), error.message)
def executeProcess(db, manifest, role_name, script, function, extension_id,
user_id, argv, timeout, stdin=None, rlimit_rss=256):
# If |user_id| is not the same as |db.user|, then one user's access of the
# system is triggering an extension on behalf of another user. This will
# for instance happen when one user is adding changes to a review,
# triggering an extension filter hook set up by another user.
#
# In this case, we need to check that the other user can access the
# extension.
#
# If |user_id| is the same as |db.user|, we need to use |db.profiles|, which
# may contain a profile associated with an access token that was used to
# authenticate the user.
if user_id != db.user.id:
user = dbutils.User.fromId(db, user_id)
authentication_labels = auth.DATABASE.getAuthenticationLabels(user)
profiles = [auth.AccessControlProfile.forUser(
db, user, authentication_labels)]
else:
authentication_labels = db.authentication_labels
profiles = db.profiles
extension = Extension.fromId(db, extension_id)
if not auth.AccessControlProfile.isAllowedExtension(
profiles, "execute", extension):
raise auth.AccessDenied("Access denied to extension: execute %s"
% extension.getKey())
flavor = manifest.flavor
if manifest.flavor not in configuration.extensions.FLAVORS:
flavor = configuration.extensions.DEFAULT_FLAVOR
stdin_data = "%s\n" % json_encode({
"library_path": configuration.extensions.FLAVORS[flavor]["library"],
"rlimit": { "rss": rlimit_rss },
"hostname": configuration.base.HOSTNAME,
"dbname": configuration.database.PARAMETERS["database"],
"dbuser": configuration.database.PARAMETERS["user"],
"git": configuration.executables.GIT,
"python": configuration.executables.PYTHON,
"python_path": "%s:%s" % (configuration.paths.CONFIG_DIR,
configuration.paths.INSTALL_DIR),
"repository_work_copy_path": configuration.extensions.WORKCOPY_DIR,
"changeset_address": configuration.services.CHANGESET["address"],
"branchtracker_pid_path": configuration.services.BRANCHTRACKER["pidfile_path"],
"maildelivery_pid_path": configuration.services.MAILDELIVERY["pidfile_path"],
"is_development": configuration.debug.IS_DEVELOPMENT,
"extension_path": manifest.path,
"extension_id": extension_id,
"user_id": user_id,
"authentication_labels": list(authentication_labels),
"role": role_name,
"script_path": script,
"fn": function,
"argv": argv })
if stdin is not None:
stdin_data += stdin
# Double the timeout. Timeouts are primarily handled by the extension runner
# service, which returns an error response on timeout. This deadline here is
# thus mostly to catch the extension runner service itself timing out.
deadline = time.time() + timeout * 2
try:
connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
connection.settimeout(max(0, deadline - time.time()))
connection.connect(configuration.services.EXTENSIONRUNNER["address"])
connection.sendall(json_encode({
"stdin": stdin_data,
"flavor": flavor,
"timeout": timeout
}))
connection.shutdown(socket.SHUT_WR)
data = ""
while True:
connection.settimeout(max(0, deadline - time.time()))
try:
received = connection.recv(4096)
except socket.error as error:
if error.errno == errno.EINTR:
continue
raise
if not received:
break
data += received
connection.close()
except socket.timeout as error:
raise ProcessTimeout(timeout)
except socket.error as error:
raise ProcessError("failed to read response: %s" % error)
try:
data = json_decode(data)
except ValueError as error:
raise ProcessError("failed to decode response: %s" % error)
if data["status"] == "timeout":
raise ProcessTimeout(timeout)
if data["status"] == "error":
raise ProcessError(data["error"])
if data["returncode"] != 0:
raise ProcessFailure(data["returncode"], data["stderr"])
return data["stdout"]
def execute(db, req, user):
cursor = db.cursor()
installs = Extension.getInstalls(db, user)
argv = None
stdin_data = None
for extension_id, version_id, version_sha1, is_universal in installs:
handlers = []
if version_id is not None:
cursor.execute(
"""SELECT script, function, path
FROM extensionroles
JOIN extensionpageroles ON (role=id)
WHERE version=%s
ORDER BY id ASC""", (version_id, ))
for script, function, path_regexp in cursor:
if re.match(path_regexp, req.path):
handlers.append((script, function))
if not handlers:
continue
extension_path = getExtensionInstallPath(version_sha1)
manifest = Manifest.load(extension_path)
else:
try:
extension = Extension.fromId(db, extension_id)
except ExtensionError:
# If the author/hosting user no longer exists, or the extension
# directory no longer exists or is inaccessible, ignore the
# extension.
continue
try:
manifest = Manifest.load(extension.getPath())
except ManifestError:
# If the MANIFEST is missing or invalid, we can't know whether
# the extension has a page role handling the path, so assume it
# doesn't and ignore it.
continue
for role in manifest.roles:
if isinstance(role, PageRole) and re.match(
role.regexp, req.path):
handlers.append((role.script, role.function))
if not handlers:
continue
if argv is None:
def param(raw):
parts = raw.split("=", 1)
if len(parts) == 1:
return "%s: null" % jsify(decodeURIComponent(raw))
else:
return "%s: %s" % (jsify(decodeURIComponent(
parts[0])), jsify(decodeURIComponent(parts[1])))
if req.query:
query = (
"Object.freeze({ raw: %s, params: Object.freeze({ %s }) })"
% (jsify(req.query), ", ".join(
map(param, req.query.split("&")))))
else:
query = "null"
headers = ("Object.freeze({ %s })" % ", ".join(
("%s: %s" % (jsify(name), jsify(value)))
for name, value in req.getRequestHeaders().items()))
argv = ("[%(method)s, %(path)s, %(query)s, %(headers)s]" % {
'method': jsify(req.method),
'path': jsify(req.path),
'query': query,
'headers': headers
})
if req.method == "POST":
if stdin_data is None:
stdin_data = req.read()
for script, function in handlers:
before = time.time()
try:
stdout_data = executeProcess(
manifest,
"page",
script,
function,
extension_id,
user.id,
argv,
configuration.extensions.LONG_TIMEOUT,
stdin=stdin_data)
except ProcessTimeout:
req.setStatus(500, "Extension Timeout")
return "Extension timed out!"
except ProcessError as error:
req.setStatus(500, "Extension Failure")
if error.returncode < 0:
return ("Extension failure: terminated by signal %d\n" %
-error.returncode)
else:
return ("Extension failure: returned %d\n%s" %
(error.returncode, error.stderr))
after = time.time()
status = None
headers = {}
if not stdout_data:
return False
while True:
try:
line, stdout_data = stdout_data.split("\n", 1)
except:
req.setStatus(500, "Extension Error")
return "Extension error: output format error.\n%r\n" % stdout_data
if status is None:
try:
status = int(line.strip())
except:
req.setStatus(500, "Extension Error")
return "Extension error: first line should contain only a numeric HTTP status code.\n%r\n" % line
elif not line:
break
else:
try:
name, value = line.split(":", 1)
except:
req.setStatus(500, "Extension Error")
return "Extension error: header line should be on 'name: value' format.\n%r\n" % line
headers[name.strip()] = value.strip()
if status is None:
req.setStatus(500, "Extension Error")
return "Extension error: first line should contain only a numeric HTTP status code.\n"
content_type = "text/plain"
for name, value in headers.items():
if name.lower() == "content-type":
content_type = value
del headers[name]
else:
headers[name] = value
req.setStatus(status)
req.setContentType(content_type)
for name, value in headers.items():
req.addResponseHeader(name, value)
if content_type == "text/tutorial":
req.setContentType("text/html")
return renderTutorial(db, user, stdout_data)
if content_type.startswith("text/html"):
stdout_data += "\n\n<!-- extension execution time: %.2f seconds -->\n" % (
after - before)
return stdout_data
return False
def __call__(self, value, context):
from extensions.extension import Extension
super(ExtensionId, self).__call__(value, context)
return Extension.fromId(context.db, value)
def execute(db, req, user):
cursor = db.cursor()
installs = Extension.getInstalls(db, user)
argv = None
stdin_data = None
for extension_id, version_id, version_sha1, is_universal in installs:
handlers = []
if version_id is not None:
cursor.execute("""SELECT script, function, path
FROM extensionroles
JOIN extensionpageroles ON (role=id)
WHERE version=%s
ORDER BY id ASC""",
(version_id,))
for script, function, path_regexp in cursor:
if re.match(path_regexp, req.path):
handlers.append((script, function))
if not handlers:
continue
extension_path = getExtensionInstallPath(version_sha1)
manifest = Manifest.load(extension_path)
else:
try:
extension = Extension.fromId(db, extension_id)
except ExtensionError:
# If the author/hosting user no longer exists, or the extension
# directory no longer exists or is inaccessible, ignore the
# extension.
continue
try:
manifest = Manifest.load(extension.getPath())
except ManifestError:
# If the MANIFEST is missing or invalid, we can't know whether
# the extension has a page role handling the path, so assume it
# doesn't and ignore it.
continue
for role in manifest.roles:
if isinstance(role, PageRole) and re.match(role.regexp, req.path):
handlers.append((role.script, role.function))
if not handlers:
continue
if argv is None:
def param(raw):
parts = raw.split("=", 1)
if len(parts) == 1:
return "%s: null" % jsify(decodeURIComponent(raw))
else:
return "%s: %s" % (jsify(decodeURIComponent(parts[0])),
jsify(decodeURIComponent(parts[1])))
if req.query:
query = ("Object.freeze({ raw: %s, params: Object.freeze({ %s }) })"
% (jsify(req.query),
", ".join(map(param, req.query.split("&")))))
else:
query = "null"
headers = ("Object.freeze({ %s })"
% ", ".join(("%s: %s" % (jsify(name), jsify(value)))
for name, value in req.getRequestHeaders().items()))
argv = ("[%(method)s, %(path)s, %(query)s, %(headers)s]"
% { 'method': jsify(req.method),
'path': jsify(req.path),
'query': query,
'headers': headers })
if req.method == "POST":
if stdin_data is None:
stdin_data = req.read()
for script, function in handlers:
before = time.time()
try:
stdout_data = executeProcess(
manifest, "page", script, function, extension_id, user.id,
argv, configuration.extensions.LONG_TIMEOUT, stdin=stdin_data)
except ProcessTimeout:
req.setStatus(500, "Extension Timeout")
return "Extension timed out!"
except ProcessError as error:
req.setStatus(500, "Extension Failure")
if error.returncode < 0:
return ("Extension failure: terminated by signal %d\n"
% -error.returncode)
else:
return ("Extension failure: returned %d\n%s"
% (error.returncode, error.stderr))
after = time.time()
status = None
headers = {}
if not stdout_data:
return False
while True:
try: line, stdout_data = stdout_data.split("\n", 1)
except:
req.setStatus(500, "Extension Error")
return "Extension error: output format error.\n%r\n" % stdout_data
if status is None:
try: status = int(line.strip())
except:
req.setStatus(500, "Extension Error")
return "Extension error: first line should contain only a numeric HTTP status code.\n%r\n" % line
elif not line:
break
else:
try: name, value = line.split(":", 1)
except:
req.setStatus(500, "Extension Error")
return "Extension error: header line should be on 'name: value' format.\n%r\n" % line
headers[name.strip()] = value.strip()
if status is None:
req.setStatus(500, "Extension Error")
return "Extension error: first line should contain only a numeric HTTP status code.\n"
content_type = "text/plain"
for name, value in headers.items():
if name.lower() == "content-type":
content_type = value
del headers[name]
else:
headers[name] = value
req.setStatus(status)
req.setContentType(content_type)
for name, value in headers.items():
req.addResponseHeader(name, value)
if content_type == "text/tutorial":
req.setContentType("text/html")
return renderTutorial(db, user, stdout_data)
if content_type.startswith("text/html"):
stdout_data += "\n\n<!-- extension execution time: %.2f seconds -->\n" % (after - before)
return stdout_data
return False
def doInstallExtension(db, user, extension, version):
auth.AccessControl.accessExtension(db, "install", extension)
is_universal = user is None
extension_id = extension.getExtensionID(db, create=True)
manifest = extension.getManifest(version)
# Detect conflicting extension installs.
current_installs = Extension.getInstalls(db, user)
for current_extension_id, _, _, current_is_universal in current_installs:
# Two installs never conflict if one is universal and one is not.
if is_universal != current_is_universal:
continue
try:
current_extension = Extension.fromId(db, current_extension_id)
except ExtensionError as error:
# Invalid extension => no conflict.
#
# But if there would be a conflict, should the installed extension
# later become valid again, then delete the installation.
if extension.getName() == error.extension.getName():
doUninstallExtension(db, user, error.extension)
continue
# Same extension => conflict
#
# The web UI will typically not let you try to do this; if the extension
# is already installed the UI will only let you uninstall or upgrade it.
# But you never know. Also, there's a UNIQUE constraint in the database
# that would prevent this, but with a significantly worse error message,
# of course.
if extension_id == current_extension_id:
raise InstallationError(
title="Conflicting install",
message=("The extension <code>%s</code> is already "
"%sinstalled."
% (current_extension.getTitle(db),
"universally " if is_universal else "")),
is_html=True)
# Different extensions, same name => also conflict
#
# Two extensions with the same name are probably simply two forks of the
# same extension, and are very likely to have overlapping and
# conflicting functionality. Also, extension resource paths only
# contain the extension name as an identifier, and thus will conflict
# between the two extensions, even if they are actually completely
# unrelated.
if extension.getName() == current_extension.getName():
raise InstallationError(
title="Conflicting install",
message=("The extension <code>%s</code> is already "
"%sinstalled, and conflicts with the extension "
"<code>%s</code> since they have the same name."
% (current_extension.getTitle(db),
"universally " if is_universal else "",
extension.getTitle(db))),
is_html=True)
cursor = db.cursor()
if is_universal:
user_id = None
else:
user_id = user.id
if version is not None:
sha1 = extension.prepareVersionSnapshot(version)
cursor.execute("""SELECT id
FROM extensionversions
WHERE extension=%s
AND name=%s
AND sha1=%s""",
(extension_id, version, sha1))
row = cursor.fetchone()
if row:
(version_id,) = row
else:
cursor.execute("""INSERT INTO extensionversions (extension, name, sha1)
VALUES (%s, %s, %s)
RETURNING id""",
(extension_id, version, sha1))
(version_id,) = cursor.fetchone()
for role in manifest.roles:
role.install(db, version_id)
else:
version_id = None
cursor.execute("""INSERT INTO extensioninstalls (uid, extension, version)
VALUES (%s, %s, %s)""",
(user_id, extension_id, version_id))
def reinstallExtension(db, user, author_name, extension_name, version):
doUninstallExtension(db, user, getExtension(author_name, extension_name))
doInstallExtension(db, user, Extension(author_name, extension_name),
version)
db.commit()
def executeProcess(db,
manifest,
role_name,
script,
function,
extension_id,
user_id,
argv,
timeout,
stdin=None,
rlimit_rss=256):
# If |user_id| is not the same as |db.user|, then one user's access of the
# system is triggering an extension on behalf of another user. This will
# for instance happen when one user is adding changes to a review,
# triggering an extension filter hook set up by another user.
#
# In this case, we need to check that the other user can access the
# extension.
#
# If |user_id| is the same as |db.user|, we need to use |db.profiles|, which
# may contain a profile associated with an access token that was used to
# authenticate the user.
if user_id != db.user.id:
user = dbutils.User.fromId(db, user_id)
authentication_labels = auth.DATABASE.getAuthenticationLabels(user)
profiles = [
auth.AccessControlProfile.forUser(db, user, authentication_labels)
]
else:
authentication_labels = db.authentication_labels
profiles = db.profiles
extension = Extension.fromId(db, extension_id)
if not auth.AccessControlProfile.isAllowedExtension(
profiles, "execute", extension):
raise auth.AccessDenied("Access denied to extension: execute %s" %
extension.getKey())
flavor = manifest.flavor
if manifest.flavor not in configuration.extensions.FLAVORS:
flavor = configuration.extensions.DEFAULT_FLAVOR
stdin_data = "%s\n" % json_encode({
"library_path":
configuration.extensions.FLAVORS[flavor]["library"],
"rlimit": {
"rss": rlimit_rss
},
"hostname":
configuration.base.HOSTNAME,
"dbname":
configuration.database.PARAMETERS["database"],
"dbuser":
configuration.database.PARAMETERS["user"],
"git":
configuration.executables.GIT,
"python":
configuration.executables.PYTHON,
"python_path":
"%s:%s" %
(configuration.paths.CONFIG_DIR, configuration.paths.INSTALL_DIR),
"repository_work_copy_path":
configuration.extensions.WORKCOPY_DIR,
"changeset_address":
configuration.services.CHANGESET["address"],
"branchtracker_pid_path":
configuration.services.BRANCHTRACKER["pidfile_path"],
"maildelivery_pid_path":
configuration.services.MAILDELIVERY["pidfile_path"],
"is_development":
configuration.debug.IS_DEVELOPMENT,
"extension_path":
manifest.path,
"extension_id":
extension_id,
"user_id":
user_id,
"authentication_labels":
list(authentication_labels),
"role":
role_name,
"script_path":
script,
"fn":
function,
"argv":
argv
})
if stdin is not None:
stdin_data += stdin
# Double the timeout. Timeouts are primarily handled by the extension runner
# service, which returns an error response on timeout. This deadline here is
# thus mostly to catch the extension runner service itself timing out.
deadline = time.time() + timeout * 2
try:
connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
connection.settimeout(max(0, deadline - time.time()))
connection.connect(configuration.services.EXTENSIONRUNNER["address"])
connection.sendall(
json_encode({
"stdin": stdin_data,
"flavor": flavor,
"timeout": timeout
}))
connection.shutdown(socket.SHUT_WR)
data = ""
while True:
connection.settimeout(max(0, deadline - time.time()))
try:
received = connection.recv(4096)
except socket.error as error:
if error.errno == errno.EINTR:
continue
raise
if not received:
break
data += received
connection.close()
except socket.timeout as error:
raise ProcessTimeout(timeout)
except socket.error as error:
raise ProcessError("failed to read response: %s" % error)
try:
data = json_decode(data)
except ValueError as error:
raise ProcessError("failed to decode response: %s" % error)
if data["status"] == "timeout":
raise ProcessTimeout(timeout)
if data["status"] == "error":
raise ProcessError(data["error"])
if data["returncode"] != 0:
raise ProcessFailure(data["returncode"], data["stderr"])
return data["stdout"]
def processFilterHookEvent(db, event_id, logfn):
cursor = db.cursor()
cursor.execute(
"""SELECT filters.extension, filters.uid, filters.path,
filters.name, events.review, events.uid, events.data
FROM extensionfilterhookevents AS events
JOIN extensionhookfilters AS filters ON (filters.id=events.filter)
WHERE events.id=%s""", (event_id, ))
# Note:
# - filter_user_id / filter_user represent the user whose filter was
# triggered.
# - user_id /user represent the user that added commits and thereby
# triggered the filter.
(extension_id, filter_user_id, filter_path, filterhook_name, review_id,
user_id, filter_data) = cursor.fetchone()
extension = Extension.fromId(db, extension_id)
filter_user = dbutils.User.fromId(db, filter_user_id)
installed_sha1, _ = extension.getInstalledVersion(db, filter_user)
if installed_sha1 is False:
# Invalid event (user doesn't have extension installed); do nothing.
# The event will be deleted by the caller.
return
manifest = extension.getManifest(sha1=installed_sha1)
for role in manifest.roles:
if isinstance(role, FilterHookRole) and role.name == filterhook_name:
break
else:
# Invalid event (installed version of extension doesn't have the named
# filter hook role); do nothing. The event will be deleted by the
# caller.
return
cursor.execute(
"""SELECT commit
FROM extensionfilterhookcommits
WHERE event=%s""", (event_id, ))
commit_ids = [commit_id for (commit_id, ) in cursor]
cursor.execute(
"""SELECT file
FROM extensionfilterhookfiles
WHERE event=%s""", (event_id, ))
file_ids = [file_id for (file_id, ) in cursor]
argv = """
(function () {
var review = new critic.Review(%(review_id)d);
var user = new critic.User(%(user_id)d);
var repository = review.repository;
var commits = new critic.CommitSet(
%(commit_ids)r.map(
function (commit_id) {
return repository.getCommit(commit_id);
}));
var files = %(file_ids)r.map(
function (file_id) {
return critic.File.find(file_id);
});
return [%(filter_data)s, review, user, commits, files];
})()
""" % {
"filter_data": htmlutils.jsify(filter_data),
"review_id": review_id,
"user_id": user_id,
"commit_ids": commit_ids,
"file_ids": file_ids
}
argv = re.sub("[ \n]+", " ", argv.strip())
logfn("argv=%r" % argv)
logfn("script=%r" % role.script)
logfn("function=%r" % role.function)
try:
executeProcess(manifest, "filterhook", role.script, role.function,
extension_id, filter_user_id, argv,
configuration.extensions.LONG_TIMEOUT)
except (ProcessTimeout, ProcessError) as error:
review = dbutils.Review.fromId(db, review_id)
recipients = set([filter_user])
author = extension.getAuthor(db)
if author is None:
recipients.update(dbutils.User.withRole(db, "administrator"))
else:
recipients.add(author)
body = """\
An error occurred while processing an extension hook filter event!
Filter details:
Extension: %(extension.title)s
Filter hook: %(role.title)s
Repository: %(repository.name)s
Path: %(filter.path)s
Data: %(filter.data)s
Event details:
Review: r/%(review.id)d "%(review.summary)s"
Commits: %(commits)s
Error details:
Error: %(error.message)s
Output:%(error.output)s
-- critic"""
commits = (gitutils.Commit.fromId(db, review.repository, commit_id)
for commit_id in commit_ids)
commits_text = "\n ".join(
('%s "%s"' % (commit.sha1[:8], commit.niceSummary())
for commit in commits))
if isinstance(error, ProcessTimeout):
error_output = " N/A"
else:
error_output = "\n\n " + "\n ".join(
error.stderr.splitlines())
body = body % {
"extension.title": extension.getTitle(db),
"role.title": role.title,
"repository.name": review.repository.name,
"filter.path": filter_path,
"filter.data": htmlutils.jsify(filter_data),
"review.id": review.id,
"review.summary": review.summary,
"commits": commits_text,
"error.message": error.message,
"error.output": error_output
}
mailutils.sendMessage(recipients=list(recipients),
subject="Failed: " + role.title,
body=body)
def execute(db, req, user, document, links, injected, profiler=None):
cursor = db.cursor()
installs = Extension.getInstalls(db, user)
def get_matching_path(path_regexp):
if re.match(path_regexp, req.path):
return (req.path, req.query)
elif re.match(path_regexp, req.original_path):
return (req.original_path, req.original_query)
else:
return None, None
query = None
for extension_id, version_id, version_sha1, is_universal in installs:
handlers = []
try:
if version_id is not None:
cursor.execute("""SELECT script, function, path
FROM extensionroles
JOIN extensioninjectroles ON (role=id)
WHERE version=%s
ORDER BY id ASC""",
(version_id,))
for script, function, path_regexp in cursor:
path, query = get_matching_path(path_regexp)
if path is not None:
handlers.append((path, query, script, function))
if not handlers:
continue
extension = Extension.fromId(db, extension_id)
manifest = Manifest.load(getExtensionInstallPath(version_sha1))
else:
extension = Extension.fromId(db, extension_id)
manifest = Manifest.load(extension.getPath())
for role in manifest.roles:
if isinstance(role, InjectRole):
path, query = get_matching_path(role.regexp)
if path is not None:
handlers.append((path, query, role.script, role.function))
if not handlers:
continue
def construct_query(query):
if not query:
return "null"
params = urlparse.parse_qs(query, keep_blank_values=True)
for key in params:
values = params[key]
if len(values) == 1:
if not values[0]:
params[key] = None
else:
params[key] = values[0]
return ("Object.freeze({ raw: %s, params: Object.freeze(%s) })"
% (json_encode(query), json_encode(params)))
preferences = None
commands = []
for path, query, script, function in handlers:
argv = "[%s, %s]" % (jsify(path), construct_query(query))
try:
stdout_data = executeProcess(
manifest, "inject", script, function, extension_id, user.id, argv,
configuration.extensions.SHORT_TIMEOUT)
except ProcessTimeout:
raise InjectError("Timeout after %d seconds." % configuration.extensions.SHORT_TIMEOUT)
except ProcessError as error:
if error.returncode < 0:
raise InjectError("Process terminated by signal %d." % -error.returncode)
else:
raise InjectError("Process returned %d.\n%s" % (error.returncode, error.stderr))
for line in stdout_data.splitlines():
if line.strip():
commands.append(processLine(path, line.strip()))
for command, value in commands:
if command == "script":
document.addExternalScript(value, use_static=False, order=1)
elif command == "stylesheet":
document.addExternalStylesheet(value, use_static=False, order=1)
elif command == "link":
for index, (_, label, _, _) in enumerate(links):
if label == value["label"]:
if value["url"] is None:
del links[index]
else:
links[index][0] = value["url"]
break
else:
if value["url"] is not None:
links.append([value["url"], value["label"], None, None])
elif command == "preference":
if not preferences:
preferences = []
injected.setdefault("preferences", []).append(
(extension.getName(), extension.getAuthor(db), preferences))
preferences.append(value)
if profiler:
profiler.check("inject: %s" % extension.getKey())
except ExtensionError as error:
document.comment("\n\n[%s] Extension error:\nInvalid extension:\n%s\n\n"
% (error.extension.getKey(), error.message))
except ManifestError as error:
document.comment("\n\n[%s] Extension error:\nInvalid MANIFEST:\n%s\n\n"
% (extension.getKey(), error.message))
except InjectError as error:
document.comment("\n\n[%s] Extension error:\n%s\n\n"
% (extension.getKey(), error.message))
def renderManageExtensions(req, db, user):
if not configuration.extensions.ENABLED:
administrators = dbutils.getAdministratorContacts(db, as_html=True)
raise page.utils.DisplayMessage(
title="Extension support not enabled",
body=(("<p>This Critic system does not support extensions.</p>"
"<p>Contact %s to have it enabled, or see the "
"<a href='/tutorial?item=administration#extensions'>"
"section on extensions</a> in the system administration "
"tutorial for more information.</p>")
% administrators),
html=True)
cursor = db.cursor()
what = req.getParameter("what", "available")
selected_versions = page.utils.json_decode(req.getParameter("select", "{}"))
focused = req.getParameter("focus", None)
if what == "installed":
title = "Installed Extensions"
listed_extensions = []
for extension_id, _, _, _ in Extension.getInstalls(db, user):
try:
listed_extensions.append(Extension.fromId(db, extension_id))
except ExtensionError as error:
listed_extensions.append(error)
else:
title = "Available Extensions"
listed_extensions = Extension.find(db)
req.content_type = "text/html; charset=utf-8"
document = htmlutils.Document(req)
document.setTitle("Manage Extensions")
html = document.html()
head = html.head()
body = html.body()
def generateRight(target):
target.a("button", href="tutorial?item=extensions").text("Tutorial")
target.text(" ")
target.a("button", href="tutorial?item=extensions-api").text("API Documentation")
page.utils.generateHeader(body, db, user, current_page="extensions", generate_right=generateRight)
document.addExternalStylesheet("resource/manageextensions.css")
document.addExternalScript("resource/manageextensions.js")
document.addInternalScript(user.getJS())
table = page.utils.PaleYellowTable(body, title)
def addTitleRightLink(url, label):
if user.name != req.user:
url += "&user=%s" % user.name
table.titleRight.text(" ")
table.titleRight.a(href=url).text("[" + label + " extensions]")
if what != "installed" or focused:
addTitleRightLink("/manageextensions?what=installed", "installed")
if what != "available" or focused:
addTitleRightLink("/manageextensions?what=available", "available")
for item in listed_extensions:
if isinstance(item, ExtensionError):
extension_error = item
extension = item.extension
else:
extension_error = None
extension = item
if focused and extension.getKey() != focused:
continue
extension_path = extension.getPath()
if extension.isSystemExtension():
hosting_user = None
else:
hosting_user = extension.getAuthor(db)
selected_version = selected_versions.get(extension.getKey(), False)
installed_sha1, installed_version = extension.getInstalledVersion(db, user)
universal_sha1, universal_version = extension.getInstalledVersion(db, None)
installed_upgradeable = universal_upgradeable = False
if extension_error is None:
if installed_sha1:
current_sha1 = extension.getCurrentSHA1(installed_version)
installed_upgradeable = installed_sha1 != current_sha1
if universal_sha1:
current_sha1 = extension.getCurrentSHA1(universal_version)
universal_upgradeable = universal_sha1 != current_sha1
def massage_version(version):
if version is None:
return "live"
elif version:
return "version/%s" % version
else:
return None
if selected_version is False:
selected_version = installed_version
if selected_version is False:
selected_version = universal_version
install_version = massage_version(selected_version)
installed_version = massage_version(installed_version)
universal_version = massage_version(universal_version)
manifest = None
if extension_error is None:
try:
if selected_version is False:
manifest = extension.getManifest()
else:
manifest = extension.getManifest(selected_version)
except ManifestError as error:
pass
elif installed_sha1:
manifest = extension.getManifest(installed_version, installed_sha1)
elif universal_sha1:
manifest = extension.getManifest(universal_version, universal_sha1)
if manifest:
if what == "available" and manifest.hidden:
# Hide from view unless the user is hosting the extension, or is
# an administrator and the extension is a system extension.
if extension.isSystemExtension():
if not user.hasRole(db, "administrator"):
continue
elif hosting_user != user:
continue
else:
if hosting_user != user:
continue
extension_id = extension.getExtensionID(db, create=False)
if not user.isAnonymous():
buttons = []
if extension_id is not None:
cursor.execute("""SELECT 1
FROM extensionstorage
WHERE extension=%s
AND uid=%s""",
(extension_id, user.id))
if cursor.fetchone():
buttons.append(("Clear storage",
("clearExtensionStorage(%s, %s)"
% (htmlutils.jsify(extension.getAuthorName()),
htmlutils.jsify(extension.getName())))))
if not installed_version:
if manifest and install_version and install_version != universal_version:
buttons.append(("Install",
("installExtension(%s, %s, %s)"
% (htmlutils.jsify(extension.getAuthorName()),
htmlutils.jsify(extension.getName()),
htmlutils.jsify(install_version)))))
else:
buttons.append(("Uninstall",
("uninstallExtension(%s, %s)"
% (htmlutils.jsify(extension.getAuthorName()),
htmlutils.jsify(extension.getName())))))
if manifest and (install_version != installed_version
or (installed_sha1 and installed_upgradeable)):
if install_version == installed_version:
label = "Upgrade"
else:
label = "Install"
buttons.append(("Upgrade",
("reinstallExtension(%s, %s, %s)"
% (htmlutils.jsify(extension.getAuthorName()),
htmlutils.jsify(extension.getName()),
htmlutils.jsify(install_version)))))
if user.hasRole(db, "administrator"):
if not universal_version:
if manifest and install_version:
buttons.append(("Install (universal)",
("installExtension(%s, %s, %s, true)"
% (htmlutils.jsify(extension.getAuthorName()),
htmlutils.jsify(extension.getName()),
htmlutils.jsify(install_version)))))
else:
buttons.append(("Uninstall (universal)",
("uninstallExtension(%s, %s, true)"
% (htmlutils.jsify(extension.getAuthorName()),
htmlutils.jsify(extension.getName())))))
if manifest and (install_version != universal_version
or (universal_sha1 and universal_upgradeable)):
if install_version == universal_version:
label = "Upgrade (universal)"
else:
label = "Install (universal)"
buttons.append((label,
("reinstallExtension(%s, %s, %s, true)"
% (htmlutils.jsify(extension.getAuthorName()),
htmlutils.jsify(extension.getName()),
htmlutils.jsify(universal_version)))))
else:
buttons = None
def renderItem(target):
target.span("name").innerHTML(extension.getTitle(db, html=True))
if hosting_user:
is_author = manifest and manifest.isAuthor(db, hosting_user)
is_sole_author = is_author and len(manifest.authors) == 1
else:
is_sole_author = False
if extension_error is None:
span = target.span("details")
span.b().text("Details: ")
select = span.select("details", critic_author=extension.getAuthorName(), critic_extension=extension.getName())
select.option(value='', selected="selected" if selected_version is False else None).text("Select version")
versions = extension.getVersions()
if versions:
optgroup = select.optgroup(label="Official Versions")
for version in versions:
optgroup.option(value="version/%s" % version, selected="selected" if selected_version == version else None).text("%s" % version.upper())
optgroup = select.optgroup(label="Development")
optgroup.option(value='live', selected="selected" if selected_version is None else None).text("LIVE")
if manifest:
is_installed = bool(installed_version)
if is_installed:
target.span("installed").text(" [installed]")
else:
is_installed = bool(universal_version)
if is_installed:
target.span("installed").text(" [installed (universal)]")
target.div("description").preformatted().text(manifest.description, linkify=True)
if not is_sole_author:
authors = target.div("authors")
authors.b().text("Author%s:" % ("s" if len(manifest.authors) > 1 else ""))
authors.text(", ".join(author.name for author in manifest.getAuthors()))
else:
is_installed = False
div = target.div("description broken").preformatted()
if extension_error is None:
anchor = div.a(href="loadmanifest?key=%s" % extension.getKey())
anchor.text("[This extension has an invalid MANIFEST file]")
else:
div.text("[This extension has been deleted or has become inaccessible]")
if selected_version is False:
return
pages = []
injects = []
processcommits = []
filterhooks = []
scheduled = []
if manifest:
for role in manifest.roles:
if isinstance(role, PageRole):
pages.append(role)
elif isinstance(role, InjectRole):
injects.append(role)
elif isinstance(role, ProcessCommitsRole):
processcommits.append(role)
elif isinstance(role, FilterHookRole):
filterhooks.append(role)
elif isinstance(role, ScheduledRole):
scheduled.append(role)
role_table = target.table("roles")
if pages:
role_table.tr().th(colspan=2).text("Pages")
for role in pages:
row = role_table.tr()
url = "%s/%s" % (dbutils.getURLPrefix(db, user), role.pattern)
if is_installed and "*" not in url:
row.td("pattern").a(href=url).text(url)
else:
row.td("pattern").text(url)
td = row.td("description")
td.text(role.description)
if injects:
role_table.tr().th(colspan=2).text("Page Injections")
for role in injects:
row = role_table.tr()
row.td("pattern").text("%s/%s" % (dbutils.getURLPrefix(db, user), role.pattern))
td = row.td("description")
td.text(role.description)
if processcommits:
role_table.tr().th(colspan=2).text("ProcessCommits hooks")
ul = role_table.tr().td(colspan=2).ul()
for role in processcommits:
li = ul.li()
li.text(role.description)
if filterhooks:
role_table.tr().th(colspan=2).text("FilterHook hooks")
for role in filterhooks:
row = role_table.tr()
row.td("title").text(role.title)
row.td("description").text(role.description)
if scheduled:
role_table.tr().th(colspan=2).text("Scheduled hooks")
for role in scheduled:
row = role_table.tr()
row.td("pattern").text("%s @ %s" % (role.frequency, role.at))
td = row.td("description")
td.text(role.description)
installed_by = ""
if extension_id is not None:
cursor.execute("""SELECT uid
FROM extensioninstalls
JOIN extensions ON (extensions.id=extensioninstalls.extension)
WHERE extensions.id=%s""",
(extension.getExtensionID(db, create=False),))
user_ids = set(user_id for user_id, in cursor.fetchall())
if user_ids:
installed_by = " (installed"
if None in user_ids:
installed_by += " universally"
user_ids.remove(None)
if user_ids:
installed_by += " and"
if user_ids:
installed_by += (" by %d user%s"
% (len(user_ids),
"s" if len(user_ids) > 1 else ""))
installed_by += ")"
table.addItem("Extension", renderItem, extension_path + "/" + installed_by, buttons)
document.addInternalScript("var selected_versions = %s;" % page.utils.json_encode(selected_versions))
return document
def execute(db, user, review, all_commits, old_head, new_head, output):
cursor = db.cursor()
installs = Extension.getInstalls(db, user)
data = None
for extension_id, version_id, version_sha1, is_universal in installs:
handlers = []
extension = Extension.fromId(db, extension_id)
if version_id is not None:
cursor.execute(
"""SELECT script, function
FROM extensionroles
JOIN extensionprocesscommitsroles ON (role=id)
WHERE version=%s
ORDER BY id ASC""", (version_id, ))
handlers.extend(cursor)
if not handlers:
continue
extension_path = getExtensionInstallPath(version_sha1)
manifest = Manifest.load(extension_path)
else:
manifest = Manifest.load(extension.getPath())
for role in manifest.roles:
if isinstance(role, ProcessCommitsRole):
handlers.append((role.script, role.function))
if not handlers:
continue
if data is None:
commitset = log.commitset.CommitSet(all_commits)
assert old_head is None or old_head in commitset.getTails()
assert new_head in commitset.getHeads()
assert len(commitset.getHeads()) == 1
tails = commitset.getFilteredTails(review.repository)
if len(tails) == 1:
tail = gitutils.Commit.fromSHA1(db, review.repository,
tails.pop())
changeset_id = changeset.utils.createChangeset(
db,
user,
review.repository,
from_commit=tail,
to_commit=new_head)[0].id
changeset_arg = "repository.getChangeset(%d)" % changeset_id
else:
changeset_arg = "null"
commits_arg = "[%s]" % ",".join(
[("repository.getCommit(%d)" % commit.getId(db))
for commit in all_commits])
data = {
"review_id": review.id,
"changeset": changeset_arg,
"commits": commits_arg
}
for script, function in handlers:
class Error(Exception):
pass
def print_header():
header = "%s::%s()" % (script, function)
print >> output, ("\n[%s] %s\n[%s] %s" %
(extension.getName(), header,
extension.getName(), "=" * len(header)))
try:
argv = """
(function ()
{
var review = new critic.Review(%(review_id)d);
var repository = review.repository;
var changeset = %(changeset)s;
var commitset = new critic.CommitSet(%(commits)s);
return [review, changeset, commitset];
})()
""" % data
argv = re.sub("[ \n]+", " ", argv.strip())
try:
stdout_data = executeProcess(
manifest, "processcommits", script, function,
extension_id, user.id, argv,
configuration.extensions.SHORT_TIMEOUT)
except ProcessTimeout:
raise Error("Timeout after %d seconds." %
configuration.extensions.SHORT_TIMEOUT)
except ProcessError as error:
if error.returncode < 0:
raise Error("Process terminated by signal %d." %
-error.returncode)
else:
raise Error("Process returned %d.\n%s" %
(error.returncode, error.stderr))
if stdout_data.strip():
print_header()
for line in stdout_data.splitlines():
print >> output, "[%s] %s" % (extension.getName(),
line)
except Error as error:
print_header()
print >> output, "[%s] Extension error: %s" % (
extension.getName(), error.message)
def generateHeader(target,
db,
user,
generate_right=None,
current_page=None,
extra_links=[],
profiler=None):
target.addExternalStylesheet("resource/third-party/jquery-ui.css")
target.addExternalStylesheet("resource/third-party/chosen.css")
target.addExternalStylesheet("resource/overrides.css")
target.addExternalStylesheet("resource/basic.css")
target.addInternalStylesheet(".defaultfont, body { %s }" %
user.getPreference(db, "style.defaultFont"))
target.addInternalStylesheet(".sourcefont { %s }" %
user.getPreference(db, "style.sourceFont"))
target.addExternalScript("resource/third-party/jquery.js")
target.addExternalScript("resource/third-party/jquery-ui.js")
target.addExternalScript(
"resource/third-party/jquery-ui-autocomplete-html.js")
target.addExternalScript("resource/third-party/chosen.js")
target.addExternalScript("resource/basic.js")
target.noscript().h1("noscript").blink().text(
"Please enable scripting support!")
row = target.table("pageheader", width='100%').tr()
left = row.td("left", valign='bottom', align='left')
b = left.b()
opera_class = "opera"
if configuration.debug.IS_DEVELOPMENT:
opera_class += " development"
b.b(opera_class, onclick="location.href='/';").text("Opera")
b.b("critic", onclick="location.href='/';").text("Critic")
links = []
if not user.isAnonymous():
links.append(["home", "Home", None, None])
links.append(["dashboard", "Dashboard", None, None])
links.append(["branches", "Branches", None, None])
links.append(["search", "Search", None, None])
if user.hasRole(db, "administrator"):
links.append(["services", "Services", None, None])
if user.hasRole(db, "repositories"):
links.append(["repositories", "Repositories", None, None])
if profiler:
profiler.check("generateHeader (basic)")
if configuration.extensions.ENABLED:
from extensions.extension import Extension
updated = Extension.getUpdatedExtensions(db, user)
if updated:
link_title = "\n".join([
("%s by %s can be updated!" %
(extension_name, author_fullname))
for author_fullname, extension_name in updated
])
links.append([
"manageextensions",
"Extensions (%d)" % len(updated), "color: red", link_title
])
else:
links.append(["manageextensions", "Extensions", None, None])
if profiler:
profiler.check("generateHeader (updated extensions)")
links.append(["config", "Config", None, None])
links.append(["tutorial", "Tutorial", None, None])
if user.isAnonymous():
count = 0
else:
cursor = db.cursor()
cursor.execute(
"""SELECT COUNT(*)
FROM newsitems
LEFT OUTER JOIN newsread ON (item=id AND uid=%s)
WHERE uid IS NULL""", (user.id, ))
count = cursor.fetchone()[0]
if count:
links.append([
"news",
"News (%d)" % count, "color: red",
"There are %d unread news items!" % count
])
else:
links.append(["news", "News", None, None])
if profiler:
profiler.check("generateHeader (news)")
req = target.getRequest()
if configuration.base.AUTHENTICATION_MODE != "host" \
and configuration.base.SESSION_TYPE == "cookie":
if user.isAnonymous():
links.append([
"javascript:void(location.href='/login?target='+encodeURIComponent(location.href));",
"Sign in", None, None
])
elif not req or req.user == user.name:
links.append(["javascript:signOut();", "Sign out", None, None])
for url, label in extra_links:
links.append([url, label, None, None])
if req and configuration.extensions.ENABLED:
import extensions.role.inject
injected = {}
extensions.role.inject.execute(db,
req,
user,
target,
links,
injected,
profiler=profiler)
for url in injected.get("stylesheets", []):
target.addExternalStylesheet(url, use_static=False, order=1)
for url in injected.get("scripts", []):
target.addExternalScript(url, use_static=False, order=1)
else:
injected = None
ul = left.ul()
for index, (url, label, style, title) in enumerate(links):
if not re.match("[-.a-z]+:|/", url):
url = "/" + url
ul.li().a(href=url, style=style, title=title).text(label)
rel = LINK_RELS.get(label)
if rel: target.setLink(rel, url)
right = row.td("right", valign='bottom', align='right')
if generate_right:
generate_right(right)
else:
right.div("buttons").span("buttonscope buttonscope-global")
if profiler:
profiler.check("generateHeader (finish)")
return injected
def doInstallExtension(db, user, extension, version):
is_universal = user is None
extension_id = extension.getExtensionID(db, create=True)
manifest = extension.getManifest(version)
# Detect conflicting extension installs.
current_installs = Extension.getInstalls(db, user)
for current_extension_id, _, _, current_is_universal in current_installs:
# Two installs never conflict if one is universal and one is not.
if is_universal != current_is_universal:
continue
try:
current_extension = Extension.fromId(db, current_extension_id)
except ExtensionError as error:
# Invalid extension => no conflict.
#
# But if there would be a conflict, should the installed extension
# later become valid again, then delete the installation.
if extension.getName() == error.extension.getName():
doUninstallExtension(db, user, error.extension)
continue
# Same extension => conflict
#
# The web UI will typically not let you try to do this; if the extension
# is already installed the UI will only let you uninstall or upgrade it.
# But you never know. Also, there's a UNIQUE constraint in the database
# that would prevent this, but with a significantly worse error message,
# of course.
if extension_id == current_extension_id:
raise InstallationError(
title="Conflicting install",
message=("The extension <code>%s</code> is already "
"%sinstalled." %
(current_extension.getTitle(db),
"universally " if is_universal else "")),
is_html=True)
# Different extensions, same name => also conflict
#
# Two extensions with the same name are probably simply two forks of the
# same extension, and are very likely to have overlapping and
# conflicting functionality. Also, extension resource paths only
# contain the extension name as an identifier, and thus will conflict
# between the two extensions, even if they are actually completely
# unrelated.
if extension.getName() == current_extension.getName():
raise InstallationError(
title="Conflicting install",
message=("The extension <code>%s</code> is already "
"%sinstalled, and conflicts with the extension "
"<code>%s</code> since they have the same name." %
(current_extension.getTitle(db), "universally "
if is_universal else "", extension.getTitle(db))),
is_html=True)
cursor = db.cursor()
if is_universal:
user_id = None
else:
user_id = user.id
if version is not None:
sha1 = extension.prepareVersionSnapshot(version)
cursor.execute(
"""SELECT id
FROM extensionversions
WHERE extension=%s
AND name=%s
AND sha1=%s""", (extension_id, version, sha1))
row = cursor.fetchone()
if row:
(version_id, ) = row
else:
cursor.execute(
"""INSERT INTO extensionversions (extension, name, sha1)
VALUES (%s, %s, %s)
RETURNING id""", (extension_id, version, sha1))
(version_id, ) = cursor.fetchone()
for role in manifest.roles:
role.install(db, version_id)
else:
version_id = None
cursor.execute(
"""INSERT INTO extensioninstalls (uid, extension, version)
VALUES (%s, %s, %s)""",
(user_id, extension_id, version_id))
def __call__(self, value, context):
from extensions.extension import Extension
super(ExtensionId, self).__call__(value, context)
return Extension.fromId(context.db, value)
def renderManageExtensions(req, db, user):
cursor = db.cursor()
what = page.utils.getParameter(req, "what", "available")
selected_versions = page.utils.json_decode(page.utils.getParameter(req, "select", "{}"))
focused = page.utils.getParameter(req, "focus", None)
if what == "available":
title = "Available Extensions"
other = ("installed extensions", "/manageextensions?what=installed" + ("&user="******""))
listed_extensions = Extension.find()
elif what == "installed":
title = "Installed Extensions"
other = ("available extensions", "/manageextensions?what=available" + ("&user="******""))
cursor.execute("""SELECT DISTINCT users.name, extensions.name
FROM users
JOIN extensions ON (extensions.author=users.id)
JOIN extensionversions ON (extensionversions.extension=extensions.id)
LEFT OUTER JOIN extensionroles_page ON (extensionroles_page.version=extensionversions.id AND extensionroles_page.uid=%s)
LEFT OUTER JOIN extensionroles_processcommits ON (extensionroles_processcommits.version=extensionversions.id AND extensionroles_processcommits.uid=%s)
WHERE extensionroles_page.uid IS NOT NULL
OR extensionroles_processcommits.uid IS NOT NULL""",
(user.id, user.id))
listed_extensions = [Extension(*row) for row in cursor]
req.content_type = "text/html; charset=utf-8"
document = htmlutils.Document(req)
document.setTitle("Manage Extensions")
html = document.html()
head = html.head()
body = html.body()
def generateRight(target):
target.a("button", href="tutorial?item=extensions").text("Tutorial")
target.text(" ")
target.a("button", href="tutorial?item=extensions-api").text("API Documentation")
page.utils.generateHeader(body, db, user, current_page="extensions", generate_right=generateRight)
document.addExternalStylesheet("resource/manageextensions.css")
document.addExternalScript("resource/manageextensions.js")
document.addInternalScript(user.getJS())
table = page.utils.PaleYellowTable(body, title)
table.titleRight.a(href=other[1]).text("[" + other[0] + "]")
for extension in listed_extensions:
extension_path = extension.getPath()
author = dbutils.User.fromName(db, extension.getAuthorName())
if focused and extension.getKey() != focused:
continue
selected_version = selected_versions.get(extension.getKey(), False)
installed_sha1, installed_version = extension.getInstalledVersion(db, user)
if selected_version is False:
selected_version = installed_version
if selected_version is None: install_version = "live"
elif selected_version is not False: install_version = "version/%s" % selected_version
else: install_version = None
try:
if selected_version is False:
manifest = extension.readManifest()
else:
manifest = extension.getInstallationStatus(db, user, selected_version)
except ManifestError, error:
manifest = None
if installed_sha1:
current_sha1 = extension.getCurrentSHA1(installed_version)
if manifest:
if what == "available" and author != user and manifest.hidden: continue
else:
if author != user: continue
if manifest:
buttons = []
if installed_version is False:
if install_version:
buttons.append(("Install", "installExtension(%s, %s, %s)" % (htmlutils.jsify(extension.getAuthorName()), htmlutils.jsify(extension.getName()), htmlutils.jsify(install_version))))
else:
buttons.append(("Uninstall", "uninstallExtension(%s, %s)" % (htmlutils.jsify(extension.getAuthorName()), htmlutils.jsify(extension.getName()))))
if installed_sha1 and installed_sha1 != current_sha1: label = "Update"
elif manifest.status != "installed": label = "Reinstall"
else: label = None
if label: buttons.append((label, "reinstallExtension(%s, %s, %s)" % (htmlutils.jsify(extension.getAuthorName()), htmlutils.jsify(extension.getName()), htmlutils.jsify(install_version))))
else:
buttons = None
def renderItem(target):
span = target.span("name")
span.b().text(extension.getName())
span.text(" by %s" % author.fullname)
span = target.span("details")
span.b().text("Details: ")
select = span.select("details", critic_author=extension.getAuthorName(), critic_extension=extension.getName())
select.option(value='', selected="selected" if selected_version is False else None).text("Select version")
versions = extension.getVersions()
if versions:
optgroup = select.optgroup(label="Official Versions")
for version in extension.getVersions():
optgroup.option(value="version/%s" % version, selected="selected" if selected_version == version else None).text("%s" % version.upper())
optgroup = select.optgroup(label="Development")
optgroup.option(value='live', selected="selected" if selected_version is None else None).text("LIVE")
if manifest:
is_installed = manifest.status in ("partial", "installed") or installed_version is not False
if is_installed:
target.span("installed").text(" [installed]")
target.div("description").preformatted().text(manifest.description, linkify=True)
else:
is_installed = False
target.div("description broken").preformatted().a(href="loadmanifest?author=%s&name=%s" % (extension.getAuthorName(), extension.getName())).text("[This extension has an invalid MANIFEST file]")
if selected_version is False:
return
pages = []
injects = []
processcommits = []
processchanges = []
scheduled = []
if manifest:
for role in manifest.roles:
if isinstance(role, PageRole):
pages.append(role)
elif isinstance(role, InjectRole):
injects.append(role)
elif isinstance(role, ProcessCommitsRole):
processcommits.append(role)
elif isinstance(role, ProcessChangesRole):
processchanges.append(role)
elif isinstance(role, ScheduledRole):
scheduled.append(role)
role_table = target.table("roles")
if pages:
role_table.tr().th(colspan=2).text("Pages")
for role in pages:
row = role_table.tr()
url = "%s/%s" % (dbutils.getURLPrefix(db), role.pattern)
if is_installed and role.installed and "*" not in url:
row.td("pattern").a(href=url).text(url)
else:
row.td("pattern").text(url)
td = row.td("description")
td.text(role.description)
if is_installed and not role.installed:
td.text(" ")
td.span("inactive").text("[Not active!]")
if injects:
role_table.tr().th(colspan=2).text("Page Injections")
for role in injects:
row = role_table.tr()
row.td("pattern").text("%s/%s" % (dbutils.getURLPrefix(db), role.pattern))
td = row.td("description")
td.text(role.description)
if is_installed and not role.installed:
td.text(" ")
td.span("inactive").text("[Not active!]")
if processcommits:
role_table.tr().th(colspan=2).text("ProcessCommits hooks")
ul = role_table.tr().td(colspan=2).ul()
for role in processcommits:
li = ul.li()
li.text(role.description)
if is_installed and not role.installed:
li.text(" ")
li.span("inactive").text("[Not active!]")
if processchanges:
role_table.tr().th(colspan=2).text("ProcessChanges hooks")
ul = role_table.tr().td(colspan=2).ul()
for role in processchanges:
li = ul.li()
li.text(role.description)
if is_installed and not role.installed:
li.text(" ")
li.span("inactive").text("[Not active!]")
if scheduled:
role_table.tr().th(colspan=2).text("Scheduled hooks")
for role in scheduled:
row = role_table.tr()
row.td("pattern").text("%s @ %s" % (role.frequency, role.at))
td = row.td("description")
td.text(role.description)
if is_installed and not role.installed:
td.text(" ")
td.span("inactive").text("[Not active!]")
cursor.execute("""SELECT DISTINCT uid
FROM extensionroles
JOIN extensionversions ON (extensionversions.id=extensionroles.version)
JOIN extensions ON (extensions.id=extensionversions.extension)
WHERE extensions.author=%s
AND extensions.name=%s""",
(author.id, extension.getName()))
installed_count = len(cursor.fetchall())
if installed_count: installed = " (installed by %d user%s)" % (installed_count, "s" if installed_count > 1 else "")
else: installed = ""
table.addItem("Extension", renderItem, extension_path + "/" + installed, buttons)
def __call__(self, value, context):
from extensions.extension import Extension
super(ExtensionKey, self).__call__(value, context)
author_name, _, extension_name = value.rpartition("/")
return Extension(author_name, extension_name)
def generateHeader(target, db, user, generate_right=None, current_page=None, extra_links=[], profiler=None):
target.addExternalStylesheet("resource/third-party/jquery-ui.css")
target.addExternalStylesheet("resource/third-party/chosen.css")
target.addExternalStylesheet("resource/overrides.css")
target.addExternalStylesheet("resource/basic.css")
target.addInternalStylesheet(".defaultfont, body { %s }" % user.getPreference(db, "style.defaultFont"))
target.addInternalStylesheet(".sourcefont { %s }" % user.getPreference(db, "style.sourceFont"))
target.addExternalScript("resource/third-party/jquery.js")
target.addExternalScript("resource/third-party/jquery-ui.js")
target.addExternalScript("resource/third-party/jquery-ui-autocomplete-html.js")
target.addExternalScript("resource/third-party/chosen.js")
target.addExternalScript("resource/basic.js")
target.noscript().h1("noscript").blink().text("Please enable scripting support!")
row = target.table("pageheader", width='100%').tr()
left = row.td("left", valign='bottom', align='left')
b = left.b()
opera_class = "opera"
if configuration.debug.IS_DEVELOPMENT:
opera_class += " development"
b.b(opera_class, onclick="location.href='/';").text("Opera")
b.b("critic", onclick="location.href='/';").text("Critic")
links = []
if not user.isAnonymous():
links.append(["home", "Home", None, None])
links.append(["dashboard", "Dashboard", None, None])
links.append(["branches", "Branches", None, None])
links.append(["search", "Search", None, None])
if user.hasRole(db, "administrator"):
links.append(["services", "Services", None, None])
if user.hasRole(db, "repositories"):
links.append(["repositories", "Repositories", None, None])
if profiler:
profiler.check("generateHeader (basic)")
if configuration.extensions.ENABLED:
from extensions.extension import Extension
updated = Extension.getUpdatedExtensions(db, user)
if updated:
link_title = "\n".join([("%s by %s can be updated!" % (extension_name, author_fullname)) for author_fullname, extension_name in updated])
links.append(["manageextensions", "Extensions (%d)" % len(updated), "color: red", link_title])
else:
links.append(["manageextensions", "Extensions", None, None])
if profiler:
profiler.check("generateHeader (updated extensions)")
links.append(["config", "Config", None, None])
links.append(["tutorial", "Tutorial", None, None])
if user.isAnonymous():
count = 0
else:
cursor = db.cursor()
cursor.execute("""SELECT COUNT(*)
FROM newsitems
LEFT OUTER JOIN newsread ON (item=id AND uid=%s)
WHERE uid IS NULL""",
(user.id,))
count = cursor.fetchone()[0]
if count:
links.append(["news", "News (%d)" % count, "color: red", "There are %d unread news items!" % count])
else:
links.append(["news", "News", None, None])
if profiler:
profiler.check("generateHeader (news)")
req = target.getRequest()
if configuration.base.AUTHENTICATION_MODE != "host" \
and configuration.base.SESSION_TYPE == "cookie":
if user.isAnonymous():
links.append(["javascript:void(location.href='/login?target='+encodeURIComponent(location.href));", "Sign in", None, None])
elif not req or (req.user == user.name and req.session_type == "cookie"):
links.append(["javascript:signOut();", "Sign out", None, None])
for url, label in extra_links:
links.append([url, label, None, None])
if req and configuration.extensions.ENABLED:
import extensions.role.inject
injected = {}
extensions.role.inject.execute(db, req, user, target, links, injected, profiler=profiler)
for url in injected.get("stylesheets", []):
target.addExternalStylesheet(url, use_static=False, order=1)
for url in injected.get("scripts", []):
target.addExternalScript(url, use_static=False, order=1)
else:
injected = None
ul = left.ul()
for index, (url, label, style, title) in enumerate(links):
if not re.match("[-.a-z]+:|/", url):
url = "/" + url
ul.li().a(href=url, style=style, title=title).text(label)
rel = LINK_RELS.get(label)
if rel: target.setLink(rel, url)
right = row.td("right", valign='bottom', align='right')
if generate_right:
generate_right(right)
else:
right.div("buttons").span("buttonscope buttonscope-global")
if profiler:
profiler.check("generateHeader (finish)")
return injected
def execute(db, req, user, document, links, injected, profiler=None):
cursor = db.cursor()
installs = Extension.getInstalls(db, user)
def get_matching_path(path_regexp):
if re.match(path_regexp, req.path):
return (req.path, req.query)
elif re.match(path_regexp, req.original_path):
return (req.original_path, req.original_query)
else:
return None, None
query = None
for extension_id, version_id, version_sha1, is_universal in installs:
handlers = []
try:
if version_id is not None:
cursor.execute(
"""SELECT script, function, path
FROM extensionroles
JOIN extensioninjectroles ON (role=id)
WHERE version=%s
ORDER BY id ASC""", (version_id, ))
for script, function, path_regexp in cursor:
path, query = get_matching_path(path_regexp)
if path is not None:
handlers.append((path, query, script, function))
if not handlers:
continue
extension = Extension.fromId(db, extension_id)
manifest = Manifest.load(getExtensionInstallPath(version_sha1))
else:
extension = Extension.fromId(db, extension_id)
manifest = Manifest.load(extension.getPath())
for role in manifest.roles:
if isinstance(role, InjectRole):
path, query = get_matching_path(role.regexp)
if path is not None:
handlers.append(
(path, query, role.script, role.function))
if not handlers:
continue
def construct_query(query):
if not query:
return "null"
params = urlparse.parse_qs(query, keep_blank_values=True)
for key in params:
values = params[key]
if len(values) == 1:
if not values[0]:
params[key] = None
else:
params[key] = values[0]
return (
"Object.freeze({ raw: %s, params: Object.freeze(%s) })" %
(json_encode(query), json_encode(params)))
preferences = None
commands = []
for path, query, script, function in handlers:
argv = "[%s, %s]" % (jsify(path), construct_query(query))
try:
stdout_data = executeProcess(
manifest, "inject", script, function, extension_id,
user.id, argv, configuration.extensions.SHORT_TIMEOUT)
except ProcessTimeout:
raise InjectError("Timeout after %d seconds." %
configuration.extensions.SHORT_TIMEOUT)
except ProcessError as error:
if error.returncode < 0:
raise InjectError("Process terminated by signal %d." %
-error.returncode)
else:
raise InjectError("Process returned %d.\n%s" %
(error.returncode, error.stderr))
for line in stdout_data.splitlines():
if line.strip():
commands.append(processLine(path, line.strip()))
for command, value in commands:
if command == "script":
document.addExternalScript(value,
use_static=False,
order=1)
elif command == "stylesheet":
document.addExternalStylesheet(value,
use_static=False,
order=1)
elif command == "link":
for index, (_, label, _, _) in enumerate(links):
if label == value["label"]:
if value["url"] is None:
del links[index]
else:
links[index][0] = value["url"]
break
else:
if value["url"] is not None:
links.append(
[value["url"], value["label"], None, None])
elif command == "preference":
if not preferences:
preferences = []
injected.setdefault("preferences", []).append(
(extension.getName(), extension.getAuthor(db),
preferences))
preferences.append(value)
if profiler:
profiler.check("inject: %s" % extension.getKey())
except ExtensionError as error:
document.comment(
"\n\n[%s] Extension error:\nInvalid extension:\n%s\n\n" %
(error.extension.getKey(), error.message))
except ManifestError as error:
document.comment(
"\n\n[%s] Extension error:\nInvalid MANIFEST:\n%s\n\n" %
(extension.getKey(), error.message))
except InjectError as error:
document.comment("\n\n[%s] Extension error:\n%s\n\n" %
(extension.getKey(), error.message))