def execute(db, req, user):
cursor = db.cursor()
installs = Extension.getInstalls(db, user)
argv = None
stdin_data = None
for extension_id, version_id, version_sha1, is_universal in installs:
handlers = []
if version_id is not None:
cursor.execute("""SELECT script, function, path
FROM extensionroles
JOIN extensionpageroles ON (role=id)
WHERE version=%s
ORDER BY id ASC""",
(version_id,))
for script, function, path_regexp in cursor:
if re.match(path_regexp, req.path):
handlers.append((script, function))
if not handlers:
continue
extension_path = getExtensionInstallPath(version_sha1)
manifest = Manifest.load(extension_path)
else:
try:
extension = Extension.fromId(db, extension_id)
except ExtensionError:
# If the author/hosting user no longer exists, or the extension
# directory no longer exists or is inaccessible, ignore the
# extension.
continue
try:
manifest = Manifest.load(extension.getPath())
except ManifestError:
# If the MANIFEST is missing or invalid, we can't know whether
# the extension has a page role handling the path, so assume it
# doesn't and ignore it.
continue
for role in manifest.roles:
if isinstance(role, PageRole) and re.match(role.regexp, req.path):
handlers.append((role.script, role.function))
if not handlers:
continue
if argv is None:
def param(raw):
parts = raw.split("=", 1)
if len(parts) == 1:
return "%s: null" % jsify(decodeURIComponent(raw))
else:
return "%s: %s" % (jsify(decodeURIComponent(parts[0])),
jsify(decodeURIComponent(parts[1])))
if req.query:
query = ("Object.freeze({ raw: %s, params: Object.freeze({ %s }) })"
% (jsify(req.query),
", ".join(map(param, req.query.split("&")))))
else:
query = "null"
headers = ("Object.freeze({ %s })"
% ", ".join(("%s: %s" % (jsify(name), jsify(value)))
for name, value in req.getRequestHeaders().items()))
argv = ("[%(method)s, %(path)s, %(query)s, %(headers)s]"
% { 'method': jsify(req.method),
'path': jsify(req.path),
'query': query,
'headers': headers })
if req.method == "POST":
if stdin_data is None:
stdin_data = req.read()
for script, function in handlers:
before = time.time()
try:
stdout_data = executeProcess(
manifest, "page", script, function, extension_id, user.id,
argv, configuration.extensions.LONG_TIMEOUT, stdin=stdin_data)
except ProcessTimeout:
req.setStatus(500, "Extension Timeout")
return "Extension timed out!"
except ProcessError as error:
req.setStatus(500, "Extension Failure")
if error.returncode < 0:
return ("Extension failure: terminated by signal %d\n"
% -error.returncode)
else:
return ("Extension failure: returned %d\n%s"
% (error.returncode, error.stderr))
after = time.time()
status = None
headers = {}
if not stdout_data:
return False
while True:
try: line, stdout_data = stdout_data.split("\n", 1)
except:
req.setStatus(500, "Extension Error")
return "Extension error: output format error.\n%r\n" % stdout_data
if status is None:
try: status = int(line.strip())
except:
req.setStatus(500, "Extension Error")
return "Extension error: first line should contain only a numeric HTTP status code.\n%r\n" % line
elif not line:
break
else:
try: name, value = line.split(":", 1)
except:
req.setStatus(500, "Extension Error")
return "Extension error: header line should be on 'name: value' format.\n%r\n" % line
headers[name.strip()] = value.strip()
if status is None:
req.setStatus(500, "Extension Error")
return "Extension error: first line should contain only a numeric HTTP status code.\n"
content_type = "text/plain"
for name, value in headers.items():
if name.lower() == "content-type":
content_type = value
del headers[name]
else:
headers[name] = value
req.setStatus(status)
req.setContentType(content_type)
for name, value in headers.items():
req.addResponseHeader(name, value)
if content_type == "text/tutorial":
req.setContentType("text/html")
return renderTutorial(db, user, stdout_data)
if content_type.startswith("text/html"):
stdout_data += "\n\n<!-- extension execution time: %.2f seconds -->\n" % (after - before)
return stdout_data
return False
def execute(db, user, review, all_commits, old_head, new_head, output):
cursor = db.cursor()
installs = Extension.getInstalls(db, user)
data = None
for extension_id, version_id, version_sha1, is_universal in installs:
handlers = []
extension = Extension.fromId(db, extension_id)
if version_id is not None:
cursor.execute("""SELECT script, function
FROM extensionroles
JOIN extensionprocesscommitsroles ON (role=id)
WHERE version=%s
ORDER BY id ASC""",
(version_id,))
handlers.extend(cursor)
if not handlers:
continue
extension_path = getExtensionInstallPath(version_sha1)
manifest = Manifest.load(extension_path)
else:
manifest = Manifest.load(extension.getPath())
for role in manifest.roles:
if isinstance(role, ProcessCommitsRole):
handlers.append((role.script, role.function))
if not handlers:
continue
if data is None:
commitset = log.commitset.CommitSet(all_commits)
assert old_head is None or old_head in commitset.getTails()
assert new_head in commitset.getHeads()
assert len(commitset.getHeads()) == 1
tails = commitset.getFilteredTails(review.repository)
if len(tails) == 1:
tail = gitutils.Commit.fromSHA1(db, review.repository, tails.pop())
changeset_id = changeset.utils.createChangeset(
db, user, review.repository, from_commit=tail, to_commit=new_head)[0].id
changeset_arg = "repository.getChangeset(%d)" % changeset_id
else:
changeset_arg = "null"
commits_arg = "[%s]" % ",".join(
[("repository.getCommit(%d)" % commit.getId(db))
for commit in all_commits])
data = { "review_id": review.id,
"changeset": changeset_arg,
"commits": commits_arg }
for script, function in handlers:
class Error(Exception):
pass
def print_header():
header = "%s::%s()" % (script, function)
print >>output, ("\n[%s] %s\n[%s] %s"
% (extension.getName(), header,
extension.getName(), "=" * len(header)))
try:
argv = """
(function ()
{
var review = new critic.Review(%(review_id)d);
var repository = review.repository;
var changeset = %(changeset)s;
var commitset = new critic.CommitSet(%(commits)s);
return [review, changeset, commitset];
})()
""" % data
argv = re.sub("[ \n]+", " ", argv.strip())
try:
stdout_data = executeProcess(
manifest, "processcommits", script, function, extension_id, user.id,
argv, configuration.extensions.SHORT_TIMEOUT)
except ProcessTimeout:
raise Error("Timeout after %d seconds." % configuration.extensions.SHORT_TIMEOUT)
except ProcessError as error:
if error.returncode < 0:
raise Error("Process terminated by signal %d." % -error.returncode)
else:
raise Error("Process returned %d.\n%s" % (error.returncode, error.stderr))
if stdout_data.strip():
print_header()
for line in stdout_data.splitlines():
print >>output, "[%s] %s" % (extension.getName(), line)
except Error as error:
print_header()
print >>output, "[%s] Extension error: %s" % (extension.getName(), error.message)
def execute(db, user_id, batch_id, output):
cursor = db.cursor()
cursor.execute("SELECT review, uid FROM batches WHERE id=%s", (batch_id,))
review_id, batch_user_id = cursor.fetchone()
cursor.execute("""SELECT extensions.id, extensions.author, extensions.name, extensionversions.sha1, roles.id, roles.script, roles.function
FROM extensions
JOIN extensionversions ON (extensionversions.extension=extensions.id)
JOIN extensionroles_processchanges AS roles ON (roles.version=extensionversions.id)
WHERE uid=%s AND roles.skip < %s""", (user_id, batch_id))
rows = cursor.fetchall()
if rows:
for extension_id, author_id, extension_name, sha1, role_id, script, function in rows:
cursor.execute("INSERT INTO extensionprocessedbatches (batch, role) VALUES (%s, %s)", (batch_id, role_id))
# Don't do further processing of own batches.
if batch_user_id == user_id: continue
author = dbutils.User.fromId(db, author_id)
if sha1 is None:
extension_path = getExtensionPath(author.name, extension_name)
else:
extension_path = getExtensionInstallPath(sha1)
class Error(Exception):
pass
def print_header():
header = "%s::%s()" % (script, function)
print >>output, "\n[%s] %s\n[%s] %s" % (extension_name, header, extension_name, "=" * len(header))
try:
try:
manifest = Manifest.load(extension_path)
except ManifestError, error:
raise Error("Invalid MANIFEST:\n%s" % error.message)
for role in manifest.roles:
if isinstance(role, ProcessChangesRole) and role.script == script and role.function == function:
break
else:
continue
argv = "[(new critic.Review(%d)).getBatch(%d)]" % (review_id, batch_id)
try:
stdout_data = executeProcess(manifest, role, extension_id, user_id, argv, configuration.extensions.SHORT_TIMEOUT)
except ProcessTimeout:
raise Error("Timeout after %d seconds." % configuration.extensions.SHORT_TIMEOUT)
except ProcessError, error:
if error.returncode < 0:
if -error.returncode == signal.SIGXCPU:
raise Error("CPU time limit (5 seconds) exceeded.")
else:
raise Error("Process terminated by signal %d." % -error.returncode)
else:
raise Error("Process returned %d.\n%s" % (error.returncode, error.stderr))
if stdout_data.strip():
print_header()
for line in stdout_data.splitlines():
print >>output, "[%s] %s" % (extension_name, line)
except Error, error:
print_header()
print >>output, "[%s] Extension error: %s" % (extension_name, error.message)
def processFilterHookEvent(db, event_id, logfn):
cursor = db.cursor()
cursor.execute(
"""SELECT filters.extension, filters.uid, filters.path,
filters.name, events.review, events.uid, events.data
FROM extensionfilterhookevents AS events
JOIN extensionhookfilters AS filters ON (filters.id=events.filter)
WHERE events.id=%s""", (event_id, ))
# Note:
# - filter_user_id / filter_user represent the user whose filter was
# triggered.
# - user_id /user represent the user that added commits and thereby
# triggered the filter.
(extension_id, filter_user_id, filter_path, filterhook_name, review_id,
user_id, filter_data) = cursor.fetchone()
extension = Extension.fromId(db, extension_id)
filter_user = dbutils.User.fromId(db, filter_user_id)
installed_sha1, _ = extension.getInstalledVersion(db, filter_user)
if installed_sha1 is False:
# Invalid event (user doesn't have extension installed); do nothing.
# The event will be deleted by the caller.
return
manifest = extension.getManifest(sha1=installed_sha1)
for role in manifest.roles:
if isinstance(role, FilterHookRole) and role.name == filterhook_name:
break
else:
# Invalid event (installed version of extension doesn't have the named
# filter hook role); do nothing. The event will be deleted by the
# caller.
return
cursor.execute(
"""SELECT commit
FROM extensionfilterhookcommits
WHERE event=%s""", (event_id, ))
commit_ids = [commit_id for (commit_id, ) in cursor]
cursor.execute(
"""SELECT file
FROM extensionfilterhookfiles
WHERE event=%s""", (event_id, ))
file_ids = [file_id for (file_id, ) in cursor]
argv = """
(function () {
var review = new critic.Review(%(review_id)d);
var user = new critic.User(%(user_id)d);
var repository = review.repository;
var commits = new critic.CommitSet(
%(commit_ids)r.map(
function (commit_id) {
return repository.getCommit(commit_id);
}));
var files = %(file_ids)r.map(
function (file_id) {
return critic.File.find(file_id);
});
return [%(filter_data)s, review, user, commits, files];
})()
""" % {
"filter_data": htmlutils.jsify(filter_data),
"review_id": review_id,
"user_id": user_id,
"commit_ids": commit_ids,
"file_ids": file_ids
}
argv = re.sub("[ \n]+", " ", argv.strip())
logfn("argv=%r" % argv)
logfn("script=%r" % role.script)
logfn("function=%r" % role.function)
try:
executeProcess(manifest, "filterhook", role.script, role.function,
extension_id, filter_user_id, argv,
configuration.extensions.LONG_TIMEOUT)
except (ProcessTimeout, ProcessError) as error:
review = dbutils.Review.fromId(db, review_id)
recipients = set([filter_user])
author = extension.getAuthor(db)
if author is None:
recipients.update(dbutils.User.withRole(db, "administrator"))
else:
recipients.add(author)
body = """\
An error occurred while processing an extension hook filter event!
Filter details:
Extension: %(extension.title)s
Filter hook: %(role.title)s
Repository: %(repository.name)s
Path: %(filter.path)s
Data: %(filter.data)s
Event details:
Review: r/%(review.id)d "%(review.summary)s"
Commits: %(commits)s
Error details:
Error: %(error.message)s
Output:%(error.output)s
-- critic"""
commits = (gitutils.Commit.fromId(db, review.repository, commit_id)
for commit_id in commit_ids)
commits_text = "\n ".join(
('%s "%s"' % (commit.sha1[:8], commit.niceSummary())
for commit in commits))
if isinstance(error, ProcessTimeout):
error_output = " N/A"
else:
error_output = "\n\n " + "\n ".join(
error.stderr.splitlines())
body = body % {
"extension.title": extension.getTitle(db),
"role.title": role.title,
"repository.name": review.repository.name,
"filter.path": filter_path,
"filter.data": htmlutils.jsify(filter_data),
"review.id": review.id,
"review.summary": review.summary,
"commits": commits_text,
"error.message": error.message,
"error.output": error_output
}
mailutils.sendMessage(recipients=list(recipients),
subject="Failed: " + role.title,
body=body)
def execute(db, req, user):
cursor = db.cursor()
installs = Extension.getInstalls(db, user)
argv = None
stdin_data = None
for extension_id, version_id, version_sha1, is_universal in installs:
handlers = []
if version_id is not None:
cursor.execute(
"""SELECT script, function, path
FROM extensionroles
JOIN extensionpageroles ON (role=id)
WHERE version=%s
ORDER BY id ASC""", (version_id, ))
for script, function, path_regexp in cursor:
if re.match(path_regexp, req.path):
handlers.append((script, function))
if not handlers:
continue
extension_path = getExtensionInstallPath(version_sha1)
manifest = Manifest.load(extension_path)
else:
try:
extension = Extension.fromId(db, extension_id)
except ExtensionError:
# If the author/hosting user no longer exists, or the extension
# directory no longer exists or is inaccessible, ignore the
# extension.
continue
try:
manifest = Manifest.load(extension.getPath())
except ManifestError:
# If the MANIFEST is missing or invalid, we can't know whether
# the extension has a page role handling the path, so assume it
# doesn't and ignore it.
continue
for role in manifest.roles:
if isinstance(role, PageRole) and re.match(
role.regexp, req.path):
handlers.append((role.script, role.function))
if not handlers:
continue
if argv is None:
def param(raw):
parts = raw.split("=", 1)
if len(parts) == 1:
return "%s: null" % jsify(decodeURIComponent(raw))
else:
return "%s: %s" % (jsify(decodeURIComponent(
parts[0])), jsify(decodeURIComponent(parts[1])))
if req.query:
query = (
"Object.freeze({ raw: %s, params: Object.freeze({ %s }) })"
% (jsify(req.query), ", ".join(
map(param, req.query.split("&")))))
else:
query = "null"
headers = ("Object.freeze({ %s })" % ", ".join(
("%s: %s" % (jsify(name), jsify(value)))
for name, value in req.getRequestHeaders().items()))
argv = ("[%(method)s, %(path)s, %(query)s, %(headers)s]" % {
'method': jsify(req.method),
'path': jsify(req.path),
'query': query,
'headers': headers
})
if req.method == "POST":
if stdin_data is None:
stdin_data = req.read()
for script, function in handlers:
before = time.time()
try:
stdout_data = executeProcess(
manifest,
"page",
script,
function,
extension_id,
user.id,
argv,
configuration.extensions.LONG_TIMEOUT,
stdin=stdin_data)
except ProcessTimeout:
req.setStatus(500, "Extension Timeout")
return "Extension timed out!"
except ProcessError as error:
req.setStatus(500, "Extension Failure")
if error.returncode < 0:
return ("Extension failure: terminated by signal %d\n" %
-error.returncode)
else:
return ("Extension failure: returned %d\n%s" %
(error.returncode, error.stderr))
after = time.time()
status = None
headers = {}
if not stdout_data:
return False
while True:
try:
line, stdout_data = stdout_data.split("\n", 1)
except:
req.setStatus(500, "Extension Error")
return "Extension error: output format error.\n%r\n" % stdout_data
if status is None:
try:
status = int(line.strip())
except:
req.setStatus(500, "Extension Error")
return "Extension error: first line should contain only a numeric HTTP status code.\n%r\n" % line
elif not line:
break
else:
try:
name, value = line.split(":", 1)
except:
req.setStatus(500, "Extension Error")
return "Extension error: header line should be on 'name: value' format.\n%r\n" % line
headers[name.strip()] = value.strip()
if status is None:
req.setStatus(500, "Extension Error")
return "Extension error: first line should contain only a numeric HTTP status code.\n"
content_type = "text/plain"
for name, value in headers.items():
if name.lower() == "content-type":
content_type = value
del headers[name]
else:
headers[name] = value
req.setStatus(status)
req.setContentType(content_type)
for name, value in headers.items():
req.addResponseHeader(name, value)
if content_type == "text/tutorial":
req.setContentType("text/html")
return renderTutorial(db, user, stdout_data)
if content_type.startswith("text/html"):
stdout_data += "\n\n<!-- extension execution time: %.2f seconds -->\n" % (
after - before)
return stdout_data
return False
def execute(db, user, review, all_commits, old_head, new_head, output):
cursor = db.cursor()
installs = Extension.getInstalls(db, user)
data = None
for extension_id, version_id, version_sha1, is_universal in installs:
handlers = []
extension = Extension.fromId(db, extension_id)
if version_id is not None:
cursor.execute(
"""SELECT script, function
FROM extensionroles
JOIN extensionprocesscommitsroles ON (role=id)
WHERE version=%s
ORDER BY id ASC""", (version_id, ))
handlers.extend(cursor)
if not handlers:
continue
extension_path = getExtensionInstallPath(version_sha1)
manifest = Manifest.load(extension_path)
else:
manifest = Manifest.load(extension.getPath())
for role in manifest.roles:
if isinstance(role, ProcessCommitsRole):
handlers.append((role.script, role.function))
if not handlers:
continue
if data is None:
commitset = log.commitset.CommitSet(all_commits)
assert old_head is None or old_head in commitset.getTails()
assert new_head in commitset.getHeads()
assert len(commitset.getHeads()) == 1
tails = commitset.getFilteredTails(review.repository)
if len(tails) == 1:
tail = gitutils.Commit.fromSHA1(db, review.repository,
tails.pop())
changeset_id = changeset.utils.createChangeset(
db,
user,
review.repository,
from_commit=tail,
to_commit=new_head)[0].id
changeset_arg = "repository.getChangeset(%d)" % changeset_id
else:
changeset_arg = "null"
commits_arg = "[%s]" % ",".join(
[("repository.getCommit(%d)" % commit.getId(db))
for commit in all_commits])
data = {
"review_id": review.id,
"changeset": changeset_arg,
"commits": commits_arg
}
for script, function in handlers:
class Error(Exception):
pass
def print_header():
header = "%s::%s()" % (script, function)
print >> output, ("\n[%s] %s\n[%s] %s" %
(extension.getName(), header,
extension.getName(), "=" * len(header)))
try:
argv = """
(function ()
{
var review = new critic.Review(%(review_id)d);
var repository = review.repository;
var changeset = %(changeset)s;
var commitset = new critic.CommitSet(%(commits)s);
return [review, changeset, commitset];
})()
""" % data
argv = re.sub("[ \n]+", " ", argv.strip())
try:
stdout_data = executeProcess(
manifest, "processcommits", script, function,
extension_id, user.id, argv,
configuration.extensions.SHORT_TIMEOUT)
except ProcessTimeout:
raise Error("Timeout after %d seconds." %
configuration.extensions.SHORT_TIMEOUT)
except ProcessError as error:
if error.returncode < 0:
raise Error("Process terminated by signal %d." %
-error.returncode)
else:
raise Error("Process returned %d.\n%s" %
(error.returncode, error.stderr))
if stdout_data.strip():
print_header()
for line in stdout_data.splitlines():
print >> output, "[%s] %s" % (extension.getName(),
line)
except Error as error:
print_header()
print >> output, "[%s] Extension error: %s" % (
extension.getName(), error.message)
def execute(db, req, user, document, links, injected, profiler=None):
cursor = db.cursor()
installs = Extension.getInstalls(db, user)
def get_matching_path(path_regexp):
if re.match(path_regexp, req.path):
return (req.path, req.query)
elif re.match(path_regexp, req.original_path):
return (req.original_path, req.original_query)
else:
return None, None
query = None
for extension_id, version_id, version_sha1, is_universal in installs:
handlers = []
try:
if version_id is not None:
cursor.execute("""SELECT script, function, path
FROM extensionroles
JOIN extensioninjectroles ON (role=id)
WHERE version=%s
ORDER BY id ASC""",
(version_id,))
for script, function, path_regexp in cursor:
path, query = get_matching_path(path_regexp)
if path is not None:
handlers.append((path, query, script, function))
if not handlers:
continue
extension = Extension.fromId(db, extension_id)
manifest = Manifest.load(getExtensionInstallPath(version_sha1))
else:
extension = Extension.fromId(db, extension_id)
manifest = Manifest.load(extension.getPath())
for role in manifest.roles:
if isinstance(role, InjectRole):
path, query = get_matching_path(role.regexp)
if path is not None:
handlers.append((path, query, role.script, role.function))
if not handlers:
continue
def construct_query(query):
if not query:
return "null"
params = urlparse.parse_qs(query, keep_blank_values=True)
for key in params:
values = params[key]
if len(values) == 1:
if not values[0]:
params[key] = None
else:
params[key] = values[0]
return ("Object.freeze({ raw: %s, params: Object.freeze(%s) })"
% (json_encode(query), json_encode(params)))
preferences = None
commands = []
for path, query, script, function in handlers:
argv = "[%s, %s]" % (jsify(path), construct_query(query))
try:
stdout_data = executeProcess(
manifest, "inject", script, function, extension_id, user.id, argv,
configuration.extensions.SHORT_TIMEOUT)
except ProcessTimeout:
raise InjectError("Timeout after %d seconds." % configuration.extensions.SHORT_TIMEOUT)
except ProcessError as error:
if error.returncode < 0:
raise InjectError("Process terminated by signal %d." % -error.returncode)
else:
raise InjectError("Process returned %d.\n%s" % (error.returncode, error.stderr))
for line in stdout_data.splitlines():
if line.strip():
commands.append(processLine(path, line.strip()))
for command, value in commands:
if command == "script":
document.addExternalScript(value, use_static=False, order=1)
elif command == "stylesheet":
document.addExternalStylesheet(value, use_static=False, order=1)
elif command == "link":
for index, (_, label, _, _) in enumerate(links):
if label == value["label"]:
if value["url"] is None:
del links[index]
else:
links[index][0] = value["url"]
break
else:
if value["url"] is not None:
links.append([value["url"], value["label"], None, None])
elif command == "preference":
if not preferences:
preferences = []
injected.setdefault("preferences", []).append(
(extension.getName(), extension.getAuthor(db), preferences))
preferences.append(value)
if profiler:
profiler.check("inject: %s" % extension.getKey())
except ExtensionError as error:
document.comment("\n\n[%s] Extension error:\nInvalid extension:\n%s\n\n"
% (error.extension.getKey(), error.message))
except ManifestError as error:
document.comment("\n\n[%s] Extension error:\nInvalid MANIFEST:\n%s\n\n"
% (extension.getKey(), error.message))
except InjectError as error:
document.comment("\n\n[%s] Extension error:\n%s\n\n"
% (extension.getKey(), error.message))
def execute(db, paths, args, user, document, links, injected, profiler=None):
cursor = db.cursor()
cursor.execute("""SELECT extensions.id, extensions.author, extensions.name, extensionversions.sha1, roles.path, roles.script, roles.function
FROM extensions
JOIN extensionversions ON (extensionversions.extension=extensions.id)
JOIN extensionroles_inject AS roles ON (roles.version=extensionversions.id)
WHERE uid=%s""", (user.id,))
for extension_id, author_id, extension_name, sha1, regexp, script, function in cursor:
for path in paths:
match = re.match(regexp, path)
if match: break
if match:
def param(raw):
parts = raw.split("=", 1)
if len(parts) == 1: return "%s: null" % jsify(decodeURIComponent(raw))
else: return "%s: %s" % (jsify(decodeURIComponent(parts[0])), jsify(decodeURIComponent(parts[1])))
if args:
query = "Object.freeze({ raw: %s, params: Object.freeze({ %s }) })" % (jsify(args), ", ".join(map(param, args.split("&"))))
else:
query = "null"
author = dbutils.User.fromId(db, author_id)
if sha1 is None:
extension_path = os.path.join(configuration.extensions.SEARCH_ROOT, author.name, "CriticExtensions", extension_name)
else:
extension_path = os.path.join(configuration.extensions.INSTALL_DIR, sha1)
class Error(Exception):
pass
try:
try:
manifest = Manifest.load(extension_path)
except ManifestError, error:
raise Error("Invalid MANIFEST:\n%s" % error.message)
for role in manifest.roles:
if isinstance(role, InjectRole) and role.regexp == regexp and role.script == script and role.function == function:
break
else:
continue
argv = "[%(path)s, %(query)s]" % { 'path': jsify(path), 'query': query }
try:
stdout_data = executeProcess(manifest, role, extension_id, user.id, argv, configuration.extensions.SHORT_TIMEOUT)
except ProcessTimeout:
raise Error("Timeout after %d seconds." % configuration.extensions.SHORT_TIMEOUT)
except ProcessError, error:
if error.returncode < 0:
if -error.returncode == signal.SIGXCPU:
raise Error("CPU time limit (5 seconds) exceeded.")
else:
raise Error("Process terminated by signal %d." % -error.returncode)
else:
raise Error("Process returned %d.\n%s" % (error.returncode, error.stderr))
commands = []
def processLine(line):
try:
command, value = line.split(" ", 1)
except ValueError:
raise Error("Invalid line in output: %r" % line)
if command not in ("link", "script", "stylesheet", "preference"):
raise Error("Invalid command: %r" % command)
try:
value = json_decode(value.strip())
except ValueError:
raise Error("Invalid JSON: %r" % value.strip())
def is_string(value):
return isinstance(value, basestring)
if command in ("script", "stylesheet") and not is_string(value):
raise Error("Invalid value for %r: %r (expected string)" % (command, value))
elif command == "link":
if not isinstance(value, list) or len(value) != 2:
raise Error("Invalid value for %r: %r (expected array of length two)" % (command, value))
elif not is_string(value[0]):
raise Error("Invalid value for %r: %r (expected string at array[0])" % (command, value))
elif not (is_string(value[1]) or value[1] is None):
raise Error("Invalid value for %r: %r (expected string or null at array[1])" % (command, value))
elif command == "preference":
if path != "config":
raise Error("Invalid command: %r only valid on /config page" % command)
elif not isinstance(value, dict):
raise Error("Invalid value for %r: %r (expected object)" % (command, value))
for name in ("url", "name", "type", "value", "default", "description"):
if name not in value:
raise Error("Invalid value for %r: %r (missing property: %r)" % (command, value, name))
preference_url = value["url"]
preference_name = value["name"]
preference_type = value["type"]
preference_value = value["value"]
preference_default = value["default"]
preference_description = value["description"]
if not is_string(preference_url):
raise Error("Invalid value for %r: %r (expected string as %r)" % (command, value, "url"))
elif not is_string(preference_name):
raise Error("Invalid value for %r: %r (expected string as %r)" % (command, value, "name"))
elif not is_string(preference_description):
raise Error("Invalid value for %r: %r (expected string as %r)" % (command, value, "description"))
if is_string(preference_type):
if preference_type not in ("boolean", "integer", "string"):
raise Error("Invalid value for %r: %r (unsupported preference type)" % (command, value))
if preference_type == "boolean": type_check = lambda value: isinstance(value, bool)
elif preference_type == "integer": type_check = lambda value: isinstance(value, int)
else: type_check = is_string
if not type_check(preference_value):
raise Error("Invalid value for %r: %r (type mismatch between %r and %r)" % (command, value, "value", "type"))
if not type_check(preference_default):
raise Error("Invalid value for %r: %r (type mismatch between %r and %r)" % (command, value, "default", "type"))
else:
if not isinstance(preference_type, list):
raise Error("Invalid value for %r: %r (invalid %r, expected string or array)" % (command, value, "type"))
for index, choice in enumerate(preference_type):
if not isinstance(choice, dict) \
or not isinstance(choice.get("value"), basestring) \
or not isinstance(choice.get("title"), basestring):
raise Error("Invalid value for %r: %r (invalid preference choice: %r)" % (command, value, choice))
choices = set([choice["value"] for choice in preference_type])
if not is_string(preference_value) or preference_value not in choices:
raise Error("Invalid value for %r: %r (%r not among valid choices)" % (command, value, "value"))
if not is_string(preference_default) or preference_default not in choices:
raise Error("Invalid value for %r: %r (%r not among valid choices)" % (command, value, "default"))
commands.append((command, value))
return True
failed = False
for line in stdout_data.splitlines():
if line.strip():
if not processLine(line.strip()):
failed = True
break
if not failed:
preferences = None
for command, value in commands:
if command == "script":
document.addExternalScript(value, use_static=False, order=1)
elif command == "stylesheet":
document.addExternalStylesheet(value, use_static=False, order=1)
elif command == "link":
for index, (url, label, not_current, style, title) in enumerate(links):
if label == value[0]:
if value[1] is None: del links[index]
else: links[index][0] = value[0]
break
else:
if value[1] is not None:
links.append([value[1], value[0], True, None, None])
elif command == "preference":
if not preferences:
preferences = []
injected.setdefault("preferences", []).append((extension_name, author, preferences))
preferences.append(value)
except Error, error:
document.comment("\n\n[%s/%s] Extension error:\n%s\n\n" % (author.name, extension_name, error.message))
def processFilterHookEvent(db, event_id, logfn):
cursor = db.cursor()
cursor.execute("""SELECT filters.extension, filters.uid, filters.path,
filters.name, events.review, events.uid, events.data
FROM extensionfilterhookevents AS events
JOIN extensionhookfilters AS filters ON (filters.id=events.filter)
WHERE events.id=%s""",
(event_id,))
# Note:
# - filter_user_id / filter_user represent the user whose filter was
# triggered.
# - user_id /user represent the user that added commits and thereby
# triggered the filter.
(extension_id, filter_user_id, filter_path,
filterhook_name, review_id, user_id, filter_data) = cursor.fetchone()
extension = Extension.fromId(db, extension_id)
filter_user = dbutils.User.fromId(db, filter_user_id)
installed_sha1, _ = extension.getInstalledVersion(db, filter_user)
if installed_sha1 is False:
# Invalid event (user doesn't have extension installed); do nothing.
# The event will be deleted by the caller.
return
manifest = extension.getManifest(sha1=installed_sha1)
for role in manifest.roles:
if isinstance(role, FilterHookRole) and role.name == filterhook_name:
break
else:
# Invalid event (installed version of extension doesn't have the named
# filter hook role); do nothing. The event will be deleted by the
# caller.
return
cursor.execute("""SELECT commit
FROM extensionfilterhookcommits
WHERE event=%s""",
(event_id,))
commit_ids = [commit_id for (commit_id,) in cursor]
cursor.execute("""SELECT file
FROM extensionfilterhookfiles
WHERE event=%s""",
(event_id,))
file_ids = [file_id for (file_id,) in cursor]
argv = """
(function () {
var review = new critic.Review(%(review_id)d);
var user = new critic.User(%(user_id)d);
var repository = review.repository;
var commits = new critic.CommitSet(
%(commit_ids)r.map(
function (commit_id) {
return repository.getCommit(commit_id);
}));
var files = %(file_ids)r.map(
function (file_id) {
return critic.File.find(file_id);
});
return [%(filter_data)s, review, user, commits, files];
})()
""" % { "filter_data": htmlutils.jsify(filter_data),
"review_id": review_id,
"user_id": user_id,
"commit_ids": commit_ids,
"file_ids": file_ids }
argv = re.sub("[ \n]+", " ", argv.strip())
logfn("argv=%r" % argv)
logfn("script=%r" % role.script)
logfn("function=%r" % role.function)
try:
executeProcess(
manifest, "filterhook", role.script, role.function, extension_id,
filter_user_id, argv, configuration.extensions.LONG_TIMEOUT)
except (ProcessTimeout, ProcessError) as error:
review = dbutils.Review.fromId(db, review_id)
recipients = set([filter_user])
author = extension.getAuthor(db)
if author is None:
recipients.update(dbutils.User.withRole(db, "administrator"))
else:
recipients.add(author)
body = """\
An error occurred while processing an extension hook filter event!
Filter details:
Extension: %(extension.title)s
Filter hook: %(role.title)s
Repository: %(repository.name)s
Path: %(filter.path)s
Data: %(filter.data)s
Event details:
Review: r/%(review.id)d "%(review.summary)s"
Commits: %(commits)s
Error details:
Error: %(error.message)s
Output:%(error.output)s
-- critic"""
commits = (gitutils.Commit.fromId(db, review.repository, commit_id)
for commit_id in commit_ids)
commits_text = "\n ".join(
('%s "%s"' % (commit.sha1[:8], commit.niceSummary())
for commit in commits))
if isinstance(error, ProcessTimeout):
error_output = " N/A"
else:
error_output = "\n\n " + "\n ".join(error.stderr.splitlines())
body = body % { "extension.title": extension.getTitle(db),
"role.title": role.title,
"repository.name": review.repository.name,
"filter.path": filter_path,
"filter.data": htmlutils.jsify(filter_data),
"review.id": review.id,
"review.summary": review.summary,
"commits": commits_text,
"error.message": error.message,
"error.output": error_output }
mailutils.sendMessage(
recipients=list(recipients),
subject="Failed: " + role.title,
body=body)
def execute(db, req, user, document, links, injected, profiler=None):
cursor = db.cursor()
installs = Extension.getInstalls(db, user)
def get_matching_path(path_regexp):
if re.match(path_regexp, req.path):
return (req.path, req.query)
elif re.match(path_regexp, req.original_path):
return (req.original_path, req.original_query)
else:
return None, None
query = None
for extension_id, version_id, version_sha1, is_universal in installs:
handlers = []
try:
if version_id is not None:
cursor.execute(
"""SELECT script, function, path
FROM extensionroles
JOIN extensioninjectroles ON (role=id)
WHERE version=%s
ORDER BY id ASC""", (version_id, ))
for script, function, path_regexp in cursor:
path, query = get_matching_path(path_regexp)
if path is not None:
handlers.append((path, query, script, function))
if not handlers:
continue
extension = Extension.fromId(db, extension_id)
manifest = Manifest.load(getExtensionInstallPath(version_sha1))
else:
extension = Extension.fromId(db, extension_id)
manifest = Manifest.load(extension.getPath())
for role in manifest.roles:
if isinstance(role, InjectRole):
path, query = get_matching_path(role.regexp)
if path is not None:
handlers.append(
(path, query, role.script, role.function))
if not handlers:
continue
def construct_query(query):
if not query:
return "null"
params = urlparse.parse_qs(query, keep_blank_values=True)
for key in params:
values = params[key]
if len(values) == 1:
if not values[0]:
params[key] = None
else:
params[key] = values[0]
return (
"Object.freeze({ raw: %s, params: Object.freeze(%s) })" %
(json_encode(query), json_encode(params)))
preferences = None
commands = []
for path, query, script, function in handlers:
argv = "[%s, %s]" % (jsify(path), construct_query(query))
try:
stdout_data = executeProcess(
manifest, "inject", script, function, extension_id,
user.id, argv, configuration.extensions.SHORT_TIMEOUT)
except ProcessTimeout:
raise InjectError("Timeout after %d seconds." %
configuration.extensions.SHORT_TIMEOUT)
except ProcessError as error:
if error.returncode < 0:
raise InjectError("Process terminated by signal %d." %
-error.returncode)
else:
raise InjectError("Process returned %d.\n%s" %
(error.returncode, error.stderr))
for line in stdout_data.splitlines():
if line.strip():
commands.append(processLine(path, line.strip()))
for command, value in commands:
if command == "script":
document.addExternalScript(value,
use_static=False,
order=1)
elif command == "stylesheet":
document.addExternalStylesheet(value,
use_static=False,
order=1)
elif command == "link":
for index, (_, label, _, _) in enumerate(links):
if label == value["label"]:
if value["url"] is None:
del links[index]
else:
links[index][0] = value["url"]
break
else:
if value["url"] is not None:
links.append(
[value["url"], value["label"], None, None])
elif command == "preference":
if not preferences:
preferences = []
injected.setdefault("preferences", []).append(
(extension.getName(), extension.getAuthor(db),
preferences))
preferences.append(value)
if profiler:
profiler.check("inject: %s" % extension.getKey())
except ExtensionError as error:
document.comment(
"\n\n[%s] Extension error:\nInvalid extension:\n%s\n\n" %
(error.extension.getKey(), error.message))
except ManifestError as error:
document.comment(
"\n\n[%s] Extension error:\nInvalid MANIFEST:\n%s\n\n" %
(extension.getKey(), error.message))
except InjectError as error:
document.comment("\n\n[%s] Extension error:\n%s\n\n" %
(extension.getKey(), error.message))
def execute(db, user, review, all_commits, old_head, new_head, output):
cursor = db.cursor()
cursor.execute("""SELECT extensions.id, extensions.author, extensions.name, extensionversions.sha1, roles.script, roles.function
FROM extensions
JOIN extensionversions ON (extensionversions.extension=extensions.id)
JOIN extensionroles_processcommits AS roles ON (roles.version=extensionversions.id)
WHERE uid=%s
AND filter IS NULL""", (user.id,))
rows = cursor.fetchall()
if rows:
commitset = log.commitset.CommitSet(all_commits)
assert old_head is None or old_head in commitset.getTails()
assert new_head in commitset.getHeads()
assert len(commitset.getHeads()) == 1
tails = commitset.getFilteredTails(review.repository)
if len(tails) == 1:
tail = gitutils.Commit.fromSHA1(db, review.repository, tails.pop())
changeset_id = changeset.utils.createChangeset(db, user, review.repository, from_commit=tail, to_commit=new_head)[0].id
changeset_arg = "repository.getChangeset(%d)" % changeset_id
db.commit()
else:
changeset_arg = "null"
commits = "[%s]" % ",".join([("repository.getCommit(%d)" % commit.getId(db)) for commit in all_commits])
data = { "review_id": review.id,
"changeset": changeset_arg,
"commits": commits }
for extension_id, author_id, extension_name, sha1, script, function in rows:
author = dbutils.User.fromId(db, author_id)
if sha1 is None: extension_path = getExtensionPath(author.name, extension_name)
else: extension_path = getExtensionInstallPath(sha1)
class Error(Exception):
pass
def print_header():
header = "%s::%s()" % (script, function)
print >>output, "\n[%s] %s\n[%s] %s" % (extension_name, header, extension_name, "=" * len(header))
try:
try:
manifest = Manifest.load(extension_path)
except ManifestError, error:
raise Error("Invalid MANIFEST:\n%s" % error.message)
for role in manifest.roles:
if isinstance(role, ProcessCommitsRole) and role.script == script and role.function == function:
break
else:
continue
argv = """
(function ()
{
var review = new critic.Review(%(review_id)d);
var repository = review.repository;
var changeset = %(changeset)s;
var commitset = new critic.CommitSet(%(commits)s);
return [review, changeset, commitset];
})()
""" % data
argv = re.sub("[ \n]+", " ", argv.strip())
try:
stdout_data = executeProcess(manifest, role, extension_id, user.id, argv, configuration.extensions.SHORT_TIMEOUT)
except ProcessTimeout:
raise Error("Timeout after %d seconds." % configuration.extensions.SHORT_TIMEOUT)
except ProcessError, error:
if error.returncode < 0:
if -error.returncode == signal.SIGXCPU:
raise Error("CPU time limit (5 seconds) exceeded.")
else:
raise Error("Process terminated by signal %d." % -error.returncode)
else:
raise Error("Process returned %d.\n%s" % (error.returncode, error.stderr))
if stdout_data.strip():
print_header()
for line in stdout_data.splitlines():
print >>output, "[%s] %s" % (extension_name, line)
except Error, error:
print_header()
print >>output, "[%s] Extension error: %s" % (extension_name, error.message)
def execute(db, req, user):
cursor = db.cursor()
cursor.execute("""SELECT extensions.id, extensions.author, extensions.name, extensionversions.sha1, roles.path, roles.script, roles.function
FROM extensions
JOIN extensionversions ON (extensionversions.extension=extensions.id)
JOIN extensionroles_page AS roles ON (roles.version=extensionversions.id)
WHERE uid=%s""", (user.id,))
for extension_id, author_id, extension_name, sha1, regexp, script, function in cursor:
if re.match(regexp, req.path):
def param(raw):
parts = raw.split("=", 1)
if len(parts) == 1: return "%s: null" % jsify(decodeURIComponent(raw))
else: return "%s: %s" % (jsify(decodeURIComponent(parts[0])), jsify(decodeURIComponent(parts[1])))
if req.query:
query = "Object.freeze({ raw: %s, params: Object.freeze({ %s }) })" % (jsify(req.query), ", ".join(map(param, req.query.split("&"))))
else:
query = "null"
headers = "Object.create(null, { %s })" % ", ".join(["%s: { value: %s, enumerable: true }" % (jsify(name), jsify(value)) for name, value in req.getRequestHeaders().items()])
author = dbutils.User.fromId(db, author_id)
if sha1 is None: extension_path = os.path.join(configuration.extensions.SEARCH_ROOT, author.name, "CriticExtensions", extension_name)
else: extension_path = os.path.join(configuration.extensions.INSTALL_DIR, sha1)
manifest = Manifest.load(extension_path)
for role in manifest.roles:
if isinstance(role, PageRole) and role.regexp == regexp and role.script == script and role.function == function:
break
else:
continue
argv = ("[%(method)s, %(path)s, %(query)s, %(headers)s]"
% { 'method': jsify(req.method),
'path': jsify(req.path),
'query': query,
'headers': headers })
if req.method == "POST":
stdin_data = req.read()
else:
stdin_data = None
before = time.time()
try:
stdout_data = executeProcess(manifest, role, extension_id, user.id, argv, configuration.extensions.LONG_TIMEOUT,
stdin=stdin_data, rlimit_cpu=60)
except ProcessTimeout:
req.setStatus(500, "Extension Timeout")
return "Extension timed out!"
except ProcessError, error:
req.setStatus(500, "Extension Failure")
if error.returncode < 0:
if -error.returncode == signal.SIGXCPU:
return "Extension failure: time limit (5 CPU seconds) exceeded\n"
else:
return "Extension failure: terminated by signal %d\n" % -error.returncode
else:
return "Extension failure: returned %d\n%s" % (error.returncode, error.stderr)
after = time.time()
status = None
headers = {}
if not stdout_data:
return False
while True:
try: line, stdout_data = stdout_data.split("\n", 1)
except:
req.setStatus(500, "Extension Error")
return "Extension error: output format error.\n%r\n" % stdout_data
if status is None:
try: status = int(line.strip())
except:
req.setStatus(500, "Extension Error")
return "Extension error: first line should contain only a numeric HTTP status code.\n%r\n" % line
elif not line:
break
else:
try: name, value = line.split(":", 1)
except:
req.setStatus(500, "Extension Error")
return "Extension error: header line should be on 'name: value' format.\n%r\n" % line
headers[name.strip()] = value.strip()
if status is None:
req.setStatus(500, "Extension Error")
return "Extension error: first line should contain only a numeric HTTP status code.\n"
content_type = "text/plain"
for name, value in headers.items():
if name.lower() == "content-type":
content_type = value
del headers[name]
else:
headers[name] = value
req.setStatus(status)
req.setContentType(content_type)
for name, value in headers.items():
req.addResponseHeader(name, value)
if content_type == "text/tutorial":
req.setContentType("text/html")
return renderTutorial(db, user, stdout_data)
if content_type.startswith("text/html"):
stdout_data += "\n\n<!-- extension execution time: %.2f seconds -->\n" % (after - before)
return stdout_data