def getManifest(self, version=None, sha1=None): import configuration path = self.__path source = None if sha1 is not None: if sha1 in self.__manifest: return self.__manifest[sha1] install_path = getExtensionInstallPath(sha1) with open(os.path.join(install_path, "MANIFEST")) as manifest_file: source = manifest_file.read() path = "<snapshot of commit %s>" % sha1[:8] elif version in self.__manifest: return self.__manifest[version] if source is None and version is not None: source = subprocess.check_output([ configuration.executables.GIT, "cat-file", "blob", "version/%s:MANIFEST" % version ], cwd=self.__path) manifest = Manifest(path, source) manifest.read() if sha1 is not None: self.__manifest[sha1] = manifest else: self.__manifest[version] = manifest return manifest
def getManifest(self, version=None, sha1=None): import configuration path = self.__path source = None if sha1 is not None: if sha1 in self.__manifest: return self.__manifest[sha1] install_path = getExtensionInstallPath(sha1) with open(os.path.join(install_path, "MANIFEST")) as manifest_file: source = manifest_file.read() path = "<snapshot of commit %s>" % sha1[:8] elif version in self.__manifest: return self.__manifest[version] if source is None and version is not None: source = subprocess.check_output( [configuration.executables.GIT, "cat-file", "blob", "version/%s:MANIFEST" % version], cwd=self.__path) manifest = Manifest(path, source) manifest.read() if sha1 is not None: self.__manifest[sha1] = manifest else: self.__manifest[version] = manifest return manifest
def readManifest(self, version=None): if version is None: source = None else: source = subprocess.check_output([configuration.executables.GIT, "cat-file", "blob", "version/%s:MANIFEST" % version], cwd=self.__path) manifest = Manifest(self.__path, source) manifest.read() return manifest
def listFilterHooks(db, user): cursor = db.cursor() installs = Extension.getInstalls(db, user) filterhooks = [] for extension_id, version_id, version_sha1, is_universal in installs: if version_id is not None: cursor.execute( """SELECT 1 FROM extensionroles JOIN extensionfilterhookroles ON (role=id) WHERE version=%s""", (version_id, )) if not cursor.fetchone(): continue extension = Extension.fromId(db, extension_id) manifest = extension.getManifest(sha1=version_sha1) else: try: extension = Extension.fromId(db, extension_id) except ExtensionError: # If the author/hosting user no longer exists, or the extension # directory no longer exists or is inaccessible, ignore the # extension. continue try: manifest = Manifest.load(extension.getPath()) except ManifestError: # If the MANIFEST is missing or invalid, we can't know whether # the extension has any filter hook roles, so assume it doesn't # and ignore it. continue if not any( isinstance(role, FilterHookRole) for role in manifest.roles): continue filterhooks.append((extension, manifest, sorted((role for role in manifest.roles if isinstance(role, FilterHookRole)), key=lambda role: role.title))) return sorted(filterhooks, key=lambda (extension, manifest, roles): extension.getKey())
def listFilterHooks(db, user): cursor = db.cursor() installs = Extension.getInstalls(db, user) filterhooks = [] for extension_id, version_id, version_sha1, is_universal in installs: if version_id is not None: cursor.execute("""SELECT 1 FROM extensionroles JOIN extensionfilterhookroles ON (role=id) WHERE version=%s""", (version_id,)) if not cursor.fetchone(): continue extension = Extension.fromId(db, extension_id) manifest = extension.getManifest(sha1=version_sha1) else: try: extension = Extension.fromId(db, extension_id) except ExtensionError: # If the author/hosting user no longer exists, or the extension # directory no longer exists or is inaccessible, ignore the # extension. continue try: manifest = Manifest.load(extension.getPath()) except ManifestError: # If the MANIFEST is missing or invalid, we can't know whether # the extension has any filter hook roles, so assume it doesn't # and ignore it. continue if not any(isinstance(role, FilterHookRole) for role in manifest.roles): continue filterhooks.append((extension, manifest, sorted( (role for role in manifest.roles if isinstance(role, FilterHookRole)), key=lambda role: role.title))) return sorted(filterhooks, key=lambda (extension, manifest, roles): extension.getKey())
def execute(db, req, user): cursor = db.cursor() installs = Extension.getInstalls(db, user) argv = None stdin_data = None for extension_id, version_id, version_sha1, is_universal in installs: handlers = [] if version_id is not None: cursor.execute("""SELECT script, function, path FROM extensionroles JOIN extensionpageroles ON (role=id) WHERE version=%s ORDER BY id ASC""", (version_id,)) for script, function, path_regexp in cursor: if re.match(path_regexp, req.path): handlers.append((script, function)) if not handlers: continue extension_path = getExtensionInstallPath(version_sha1) manifest = Manifest.load(extension_path) else: try: extension = Extension.fromId(db, extension_id) except ExtensionError: # If the author/hosting user no longer exists, or the extension # directory no longer exists or is inaccessible, ignore the # extension. continue try: manifest = Manifest.load(extension.getPath()) except ManifestError: # If the MANIFEST is missing or invalid, we can't know whether # the extension has a page role handling the path, so assume it # doesn't and ignore it. continue for role in manifest.roles: if isinstance(role, PageRole) and re.match(role.regexp, req.path): handlers.append((role.script, role.function)) if not handlers: continue if argv is None: def param(raw): parts = raw.split("=", 1) if len(parts) == 1: return "%s: null" % jsify(decodeURIComponent(raw)) else: return "%s: %s" % (jsify(decodeURIComponent(parts[0])), jsify(decodeURIComponent(parts[1]))) if req.query: query = ("Object.freeze({ raw: %s, params: Object.freeze({ %s }) })" % (jsify(req.query), ", ".join(map(param, req.query.split("&"))))) else: query = "null" headers = ("Object.freeze({ %s })" % ", ".join(("%s: %s" % (jsify(name), jsify(value))) for name, value in req.getRequestHeaders().items())) argv = ("[%(method)s, %(path)s, %(query)s, %(headers)s]" % { 'method': jsify(req.method), 'path': jsify(req.path), 'query': query, 'headers': headers }) if req.method == "POST": if stdin_data is None: stdin_data = req.read() for script, function in handlers: before = time.time() try: stdout_data = executeProcess( manifest, "page", script, function, extension_id, user.id, argv, configuration.extensions.LONG_TIMEOUT, stdin=stdin_data) except ProcessTimeout: req.setStatus(500, "Extension Timeout") return "Extension timed out!" except ProcessError as error: req.setStatus(500, "Extension Failure") if error.returncode < 0: return ("Extension failure: terminated by signal %d\n" % -error.returncode) else: return ("Extension failure: returned %d\n%s" % (error.returncode, error.stderr)) after = time.time() status = None headers = {} if not stdout_data: return False while True: try: line, stdout_data = stdout_data.split("\n", 1) except: req.setStatus(500, "Extension Error") return "Extension error: output format error.\n%r\n" % stdout_data if status is None: try: status = int(line.strip()) except: req.setStatus(500, "Extension Error") return "Extension error: first line should contain only a numeric HTTP status code.\n%r\n" % line elif not line: break else: try: name, value = line.split(":", 1) except: req.setStatus(500, "Extension Error") return "Extension error: header line should be on 'name: value' format.\n%r\n" % line headers[name.strip()] = value.strip() if status is None: req.setStatus(500, "Extension Error") return "Extension error: first line should contain only a numeric HTTP status code.\n" content_type = "text/plain" for name, value in headers.items(): if name.lower() == "content-type": content_type = value del headers[name] else: headers[name] = value req.setStatus(status) req.setContentType(content_type) for name, value in headers.items(): req.addResponseHeader(name, value) if content_type == "text/tutorial": req.setContentType("text/html") return renderTutorial(db, user, stdout_data) if content_type.startswith("text/html"): stdout_data += "\n\n<!-- extension execution time: %.2f seconds -->\n" % (after - before) return stdout_data return False
def execute(db, user_id, batch_id, output): cursor = db.cursor() cursor.execute("SELECT review, uid FROM batches WHERE id=%s", (batch_id,)) review_id, batch_user_id = cursor.fetchone() cursor.execute("""SELECT extensions.id, extensions.author, extensions.name, extensionversions.sha1, roles.id, roles.script, roles.function FROM extensions JOIN extensionversions ON (extensionversions.extension=extensions.id) JOIN extensionroles_processchanges AS roles ON (roles.version=extensionversions.id) WHERE uid=%s AND roles.skip < %s""", (user_id, batch_id)) rows = cursor.fetchall() if rows: for extension_id, author_id, extension_name, sha1, role_id, script, function in rows: cursor.execute("INSERT INTO extensionprocessedbatches (batch, role) VALUES (%s, %s)", (batch_id, role_id)) # Don't do further processing of own batches. if batch_user_id == user_id: continue author = dbutils.User.fromId(db, author_id) if sha1 is None: extension_path = getExtensionPath(author.name, extension_name) else: extension_path = getExtensionInstallPath(sha1) class Error(Exception): pass def print_header(): header = "%s::%s()" % (script, function) print >>output, "\n[%s] %s\n[%s] %s" % (extension_name, header, extension_name, "=" * len(header)) try: try: manifest = Manifest.load(extension_path) except ManifestError, error: raise Error("Invalid MANIFEST:\n%s" % error.message) for role in manifest.roles: if isinstance(role, ProcessChangesRole) and role.script == script and role.function == function: break else: continue argv = "[(new critic.Review(%d)).getBatch(%d)]" % (review_id, batch_id) try: stdout_data = executeProcess(manifest, role, extension_id, user_id, argv, configuration.extensions.SHORT_TIMEOUT) except ProcessTimeout: raise Error("Timeout after %d seconds." % configuration.extensions.SHORT_TIMEOUT) except ProcessError, error: if error.returncode < 0: if -error.returncode == signal.SIGXCPU: raise Error("CPU time limit (5 seconds) exceeded.") else: raise Error("Process terminated by signal %d." % -error.returncode) else: raise Error("Process returned %d.\n%s" % (error.returncode, error.stderr)) if stdout_data.strip(): print_header() for line in stdout_data.splitlines(): print >>output, "[%s] %s" % (extension_name, line) except Error, error: print_header() print >>output, "[%s] Extension error: %s" % (extension_name, error.message)
def execute(db, user, review, all_commits, old_head, new_head, output): cursor = db.cursor() installs = Extension.getInstalls(db, user) data = None for extension_id, version_id, version_sha1, is_universal in installs: handlers = [] extension = Extension.fromId(db, extension_id) if version_id is not None: cursor.execute("""SELECT script, function FROM extensionroles JOIN extensionprocesscommitsroles ON (role=id) WHERE version=%s ORDER BY id ASC""", (version_id,)) handlers.extend(cursor) if not handlers: continue extension_path = getExtensionInstallPath(version_sha1) manifest = Manifest.load(extension_path) else: manifest = Manifest.load(extension.getPath()) for role in manifest.roles: if isinstance(role, ProcessCommitsRole): handlers.append((role.script, role.function)) if not handlers: continue if data is None: commitset = log.commitset.CommitSet(all_commits) assert old_head is None or old_head in commitset.getTails() assert new_head in commitset.getHeads() assert len(commitset.getHeads()) == 1 tails = commitset.getFilteredTails(review.repository) if len(tails) == 1: tail = gitutils.Commit.fromSHA1(db, review.repository, tails.pop()) changeset_id = changeset.utils.createChangeset( db, user, review.repository, from_commit=tail, to_commit=new_head)[0].id changeset_arg = "repository.getChangeset(%d)" % changeset_id else: changeset_arg = "null" commits_arg = "[%s]" % ",".join( [("repository.getCommit(%d)" % commit.getId(db)) for commit in all_commits]) data = { "review_id": review.id, "changeset": changeset_arg, "commits": commits_arg } for script, function in handlers: class Error(Exception): pass def print_header(): header = "%s::%s()" % (script, function) print >>output, ("\n[%s] %s\n[%s] %s" % (extension.getName(), header, extension.getName(), "=" * len(header))) try: argv = """ (function () { var review = new critic.Review(%(review_id)d); var repository = review.repository; var changeset = %(changeset)s; var commitset = new critic.CommitSet(%(commits)s); return [review, changeset, commitset]; })() """ % data argv = re.sub("[ \n]+", " ", argv.strip()) try: stdout_data = executeProcess( manifest, "processcommits", script, function, extension_id, user.id, argv, configuration.extensions.SHORT_TIMEOUT) except ProcessTimeout: raise Error("Timeout after %d seconds." % configuration.extensions.SHORT_TIMEOUT) except ProcessError as error: if error.returncode < 0: raise Error("Process terminated by signal %d." % -error.returncode) else: raise Error("Process returned %d.\n%s" % (error.returncode, error.stderr)) if stdout_data.strip(): print_header() for line in stdout_data.splitlines(): print >>output, "[%s] %s" % (extension.getName(), line) except Error as error: print_header() print >>output, "[%s] Extension error: %s" % (extension.getName(), error.message)
def execute(db, req, user): cursor = db.cursor() installs = Extension.getInstalls(db, user) argv = None stdin_data = None for extension_id, version_id, version_sha1, is_universal in installs: handlers = [] if version_id is not None: cursor.execute( """SELECT script, function, path FROM extensionroles JOIN extensionpageroles ON (role=id) WHERE version=%s ORDER BY id ASC""", (version_id, )) for script, function, path_regexp in cursor: if re.match(path_regexp, req.path): handlers.append((script, function)) if not handlers: continue extension_path = getExtensionInstallPath(version_sha1) manifest = Manifest.load(extension_path) else: try: extension = Extension.fromId(db, extension_id) except ExtensionError: # If the author/hosting user no longer exists, or the extension # directory no longer exists or is inaccessible, ignore the # extension. continue try: manifest = Manifest.load(extension.getPath()) except ManifestError: # If the MANIFEST is missing or invalid, we can't know whether # the extension has a page role handling the path, so assume it # doesn't and ignore it. continue for role in manifest.roles: if isinstance(role, PageRole) and re.match( role.regexp, req.path): handlers.append((role.script, role.function)) if not handlers: continue if argv is None: def param(raw): parts = raw.split("=", 1) if len(parts) == 1: return "%s: null" % jsify(decodeURIComponent(raw)) else: return "%s: %s" % (jsify(decodeURIComponent( parts[0])), jsify(decodeURIComponent(parts[1]))) if req.query: query = ( "Object.freeze({ raw: %s, params: Object.freeze({ %s }) })" % (jsify(req.query), ", ".join( map(param, req.query.split("&"))))) else: query = "null" headers = ("Object.freeze({ %s })" % ", ".join( ("%s: %s" % (jsify(name), jsify(value))) for name, value in req.getRequestHeaders().items())) argv = ("[%(method)s, %(path)s, %(query)s, %(headers)s]" % { 'method': jsify(req.method), 'path': jsify(req.path), 'query': query, 'headers': headers }) if req.method == "POST": if stdin_data is None: stdin_data = req.read() for script, function in handlers: before = time.time() try: stdout_data = executeProcess( manifest, "page", script, function, extension_id, user.id, argv, configuration.extensions.LONG_TIMEOUT, stdin=stdin_data) except ProcessTimeout: req.setStatus(500, "Extension Timeout") return "Extension timed out!" except ProcessError as error: req.setStatus(500, "Extension Failure") if error.returncode < 0: return ("Extension failure: terminated by signal %d\n" % -error.returncode) else: return ("Extension failure: returned %d\n%s" % (error.returncode, error.stderr)) after = time.time() status = None headers = {} if not stdout_data: return False while True: try: line, stdout_data = stdout_data.split("\n", 1) except: req.setStatus(500, "Extension Error") return "Extension error: output format error.\n%r\n" % stdout_data if status is None: try: status = int(line.strip()) except: req.setStatus(500, "Extension Error") return "Extension error: first line should contain only a numeric HTTP status code.\n%r\n" % line elif not line: break else: try: name, value = line.split(":", 1) except: req.setStatus(500, "Extension Error") return "Extension error: header line should be on 'name: value' format.\n%r\n" % line headers[name.strip()] = value.strip() if status is None: req.setStatus(500, "Extension Error") return "Extension error: first line should contain only a numeric HTTP status code.\n" content_type = "text/plain" for name, value in headers.items(): if name.lower() == "content-type": content_type = value del headers[name] else: headers[name] = value req.setStatus(status) req.setContentType(content_type) for name, value in headers.items(): req.addResponseHeader(name, value) if content_type == "text/tutorial": req.setContentType("text/html") return renderTutorial(db, user, stdout_data) if content_type.startswith("text/html"): stdout_data += "\n\n<!-- extension execution time: %.2f seconds -->\n" % ( after - before) return stdout_data return False
def execute(db, req, user, document, links, injected, profiler=None): cursor = db.cursor() installs = Extension.getInstalls(db, user) def get_matching_path(path_regexp): if re.match(path_regexp, req.path): return (req.path, req.query) elif re.match(path_regexp, req.original_path): return (req.original_path, req.original_query) else: return None, None query = None for extension_id, version_id, version_sha1, is_universal in installs: handlers = [] try: if version_id is not None: cursor.execute("""SELECT script, function, path FROM extensionroles JOIN extensioninjectroles ON (role=id) WHERE version=%s ORDER BY id ASC""", (version_id,)) for script, function, path_regexp in cursor: path, query = get_matching_path(path_regexp) if path is not None: handlers.append((path, query, script, function)) if not handlers: continue extension = Extension.fromId(db, extension_id) manifest = Manifest.load(getExtensionInstallPath(version_sha1)) else: extension = Extension.fromId(db, extension_id) manifest = Manifest.load(extension.getPath()) for role in manifest.roles: if isinstance(role, InjectRole): path, query = get_matching_path(role.regexp) if path is not None: handlers.append((path, query, role.script, role.function)) if not handlers: continue def construct_query(query): if not query: return "null" params = urlparse.parse_qs(query, keep_blank_values=True) for key in params: values = params[key] if len(values) == 1: if not values[0]: params[key] = None else: params[key] = values[0] return ("Object.freeze({ raw: %s, params: Object.freeze(%s) })" % (json_encode(query), json_encode(params))) preferences = None commands = [] for path, query, script, function in handlers: argv = "[%s, %s]" % (jsify(path), construct_query(query)) try: stdout_data = executeProcess( manifest, "inject", script, function, extension_id, user.id, argv, configuration.extensions.SHORT_TIMEOUT) except ProcessTimeout: raise InjectError("Timeout after %d seconds." % configuration.extensions.SHORT_TIMEOUT) except ProcessError as error: if error.returncode < 0: raise InjectError("Process terminated by signal %d." % -error.returncode) else: raise InjectError("Process returned %d.\n%s" % (error.returncode, error.stderr)) for line in stdout_data.splitlines(): if line.strip(): commands.append(processLine(path, line.strip())) for command, value in commands: if command == "script": document.addExternalScript(value, use_static=False, order=1) elif command == "stylesheet": document.addExternalStylesheet(value, use_static=False, order=1) elif command == "link": for index, (_, label, _, _) in enumerate(links): if label == value["label"]: if value["url"] is None: del links[index] else: links[index][0] = value["url"] break else: if value["url"] is not None: links.append([value["url"], value["label"], None, None]) elif command == "preference": if not preferences: preferences = [] injected.setdefault("preferences", []).append( (extension.getName(), extension.getAuthor(db), preferences)) preferences.append(value) if profiler: profiler.check("inject: %s" % extension.getKey()) except ExtensionError as error: document.comment("\n\n[%s] Extension error:\nInvalid extension:\n%s\n\n" % (error.extension.getKey(), error.message)) except ManifestError as error: document.comment("\n\n[%s] Extension error:\nInvalid MANIFEST:\n%s\n\n" % (extension.getKey(), error.message)) except InjectError as error: document.comment("\n\n[%s] Extension error:\n%s\n\n" % (extension.getKey(), error.message))
def execute(db, user, review, all_commits, old_head, new_head, output): cursor = db.cursor() installs = Extension.getInstalls(db, user) data = None for extension_id, version_id, version_sha1, is_universal in installs: handlers = [] extension = Extension.fromId(db, extension_id) if version_id is not None: cursor.execute( """SELECT script, function FROM extensionroles JOIN extensionprocesscommitsroles ON (role=id) WHERE version=%s ORDER BY id ASC""", (version_id, )) handlers.extend(cursor) if not handlers: continue extension_path = getExtensionInstallPath(version_sha1) manifest = Manifest.load(extension_path) else: manifest = Manifest.load(extension.getPath()) for role in manifest.roles: if isinstance(role, ProcessCommitsRole): handlers.append((role.script, role.function)) if not handlers: continue if data is None: commitset = log.commitset.CommitSet(all_commits) assert old_head is None or old_head in commitset.getTails() assert new_head in commitset.getHeads() assert len(commitset.getHeads()) == 1 tails = commitset.getFilteredTails(review.repository) if len(tails) == 1: tail = gitutils.Commit.fromSHA1(db, review.repository, tails.pop()) changeset_id = changeset.utils.createChangeset( db, user, review.repository, from_commit=tail, to_commit=new_head)[0].id changeset_arg = "repository.getChangeset(%d)" % changeset_id else: changeset_arg = "null" commits_arg = "[%s]" % ",".join( [("repository.getCommit(%d)" % commit.getId(db)) for commit in all_commits]) data = { "review_id": review.id, "changeset": changeset_arg, "commits": commits_arg } for script, function in handlers: class Error(Exception): pass def print_header(): header = "%s::%s()" % (script, function) print >> output, ("\n[%s] %s\n[%s] %s" % (extension.getName(), header, extension.getName(), "=" * len(header))) try: argv = """ (function () { var review = new critic.Review(%(review_id)d); var repository = review.repository; var changeset = %(changeset)s; var commitset = new critic.CommitSet(%(commits)s); return [review, changeset, commitset]; })() """ % data argv = re.sub("[ \n]+", " ", argv.strip()) try: stdout_data = executeProcess( manifest, "processcommits", script, function, extension_id, user.id, argv, configuration.extensions.SHORT_TIMEOUT) except ProcessTimeout: raise Error("Timeout after %d seconds." % configuration.extensions.SHORT_TIMEOUT) except ProcessError as error: if error.returncode < 0: raise Error("Process terminated by signal %d." % -error.returncode) else: raise Error("Process returned %d.\n%s" % (error.returncode, error.stderr)) if stdout_data.strip(): print_header() for line in stdout_data.splitlines(): print >> output, "[%s] %s" % (extension.getName(), line) except Error as error: print_header() print >> output, "[%s] Extension error: %s" % ( extension.getName(), error.message)
def execute(db, paths, args, user, document, links, injected, profiler=None): cursor = db.cursor() cursor.execute("""SELECT extensions.id, extensions.author, extensions.name, extensionversions.sha1, roles.path, roles.script, roles.function FROM extensions JOIN extensionversions ON (extensionversions.extension=extensions.id) JOIN extensionroles_inject AS roles ON (roles.version=extensionversions.id) WHERE uid=%s""", (user.id,)) for extension_id, author_id, extension_name, sha1, regexp, script, function in cursor: for path in paths: match = re.match(regexp, path) if match: break if match: def param(raw): parts = raw.split("=", 1) if len(parts) == 1: return "%s: null" % jsify(decodeURIComponent(raw)) else: return "%s: %s" % (jsify(decodeURIComponent(parts[0])), jsify(decodeURIComponent(parts[1]))) if args: query = "Object.freeze({ raw: %s, params: Object.freeze({ %s }) })" % (jsify(args), ", ".join(map(param, args.split("&")))) else: query = "null" author = dbutils.User.fromId(db, author_id) if sha1 is None: extension_path = os.path.join(configuration.extensions.SEARCH_ROOT, author.name, "CriticExtensions", extension_name) else: extension_path = os.path.join(configuration.extensions.INSTALL_DIR, sha1) class Error(Exception): pass try: try: manifest = Manifest.load(extension_path) except ManifestError, error: raise Error("Invalid MANIFEST:\n%s" % error.message) for role in manifest.roles: if isinstance(role, InjectRole) and role.regexp == regexp and role.script == script and role.function == function: break else: continue argv = "[%(path)s, %(query)s]" % { 'path': jsify(path), 'query': query } try: stdout_data = executeProcess(manifest, role, extension_id, user.id, argv, configuration.extensions.SHORT_TIMEOUT) except ProcessTimeout: raise Error("Timeout after %d seconds." % configuration.extensions.SHORT_TIMEOUT) except ProcessError, error: if error.returncode < 0: if -error.returncode == signal.SIGXCPU: raise Error("CPU time limit (5 seconds) exceeded.") else: raise Error("Process terminated by signal %d." % -error.returncode) else: raise Error("Process returned %d.\n%s" % (error.returncode, error.stderr)) commands = [] def processLine(line): try: command, value = line.split(" ", 1) except ValueError: raise Error("Invalid line in output: %r" % line) if command not in ("link", "script", "stylesheet", "preference"): raise Error("Invalid command: %r" % command) try: value = json_decode(value.strip()) except ValueError: raise Error("Invalid JSON: %r" % value.strip()) def is_string(value): return isinstance(value, basestring) if command in ("script", "stylesheet") and not is_string(value): raise Error("Invalid value for %r: %r (expected string)" % (command, value)) elif command == "link": if not isinstance(value, list) or len(value) != 2: raise Error("Invalid value for %r: %r (expected array of length two)" % (command, value)) elif not is_string(value[0]): raise Error("Invalid value for %r: %r (expected string at array[0])" % (command, value)) elif not (is_string(value[1]) or value[1] is None): raise Error("Invalid value for %r: %r (expected string or null at array[1])" % (command, value)) elif command == "preference": if path != "config": raise Error("Invalid command: %r only valid on /config page" % command) elif not isinstance(value, dict): raise Error("Invalid value for %r: %r (expected object)" % (command, value)) for name in ("url", "name", "type", "value", "default", "description"): if name not in value: raise Error("Invalid value for %r: %r (missing property: %r)" % (command, value, name)) preference_url = value["url"] preference_name = value["name"] preference_type = value["type"] preference_value = value["value"] preference_default = value["default"] preference_description = value["description"] if not is_string(preference_url): raise Error("Invalid value for %r: %r (expected string as %r)" % (command, value, "url")) elif not is_string(preference_name): raise Error("Invalid value for %r: %r (expected string as %r)" % (command, value, "name")) elif not is_string(preference_description): raise Error("Invalid value for %r: %r (expected string as %r)" % (command, value, "description")) if is_string(preference_type): if preference_type not in ("boolean", "integer", "string"): raise Error("Invalid value for %r: %r (unsupported preference type)" % (command, value)) if preference_type == "boolean": type_check = lambda value: isinstance(value, bool) elif preference_type == "integer": type_check = lambda value: isinstance(value, int) else: type_check = is_string if not type_check(preference_value): raise Error("Invalid value for %r: %r (type mismatch between %r and %r)" % (command, value, "value", "type")) if not type_check(preference_default): raise Error("Invalid value for %r: %r (type mismatch between %r and %r)" % (command, value, "default", "type")) else: if not isinstance(preference_type, list): raise Error("Invalid value for %r: %r (invalid %r, expected string or array)" % (command, value, "type")) for index, choice in enumerate(preference_type): if not isinstance(choice, dict) \ or not isinstance(choice.get("value"), basestring) \ or not isinstance(choice.get("title"), basestring): raise Error("Invalid value for %r: %r (invalid preference choice: %r)" % (command, value, choice)) choices = set([choice["value"] for choice in preference_type]) if not is_string(preference_value) or preference_value not in choices: raise Error("Invalid value for %r: %r (%r not among valid choices)" % (command, value, "value")) if not is_string(preference_default) or preference_default not in choices: raise Error("Invalid value for %r: %r (%r not among valid choices)" % (command, value, "default")) commands.append((command, value)) return True failed = False for line in stdout_data.splitlines(): if line.strip(): if not processLine(line.strip()): failed = True break if not failed: preferences = None for command, value in commands: if command == "script": document.addExternalScript(value, use_static=False, order=1) elif command == "stylesheet": document.addExternalStylesheet(value, use_static=False, order=1) elif command == "link": for index, (url, label, not_current, style, title) in enumerate(links): if label == value[0]: if value[1] is None: del links[index] else: links[index][0] = value[0] break else: if value[1] is not None: links.append([value[1], value[0], True, None, None]) elif command == "preference": if not preferences: preferences = [] injected.setdefault("preferences", []).append((extension_name, author, preferences)) preferences.append(value) except Error, error: document.comment("\n\n[%s/%s] Extension error:\n%s\n\n" % (author.name, extension_name, error.message))
def execute(db, req, user, document, links, injected, profiler=None): cursor = db.cursor() installs = Extension.getInstalls(db, user) def get_matching_path(path_regexp): if re.match(path_regexp, req.path): return (req.path, req.query) elif re.match(path_regexp, req.original_path): return (req.original_path, req.original_query) else: return None, None query = None for extension_id, version_id, version_sha1, is_universal in installs: handlers = [] try: if version_id is not None: cursor.execute( """SELECT script, function, path FROM extensionroles JOIN extensioninjectroles ON (role=id) WHERE version=%s ORDER BY id ASC""", (version_id, )) for script, function, path_regexp in cursor: path, query = get_matching_path(path_regexp) if path is not None: handlers.append((path, query, script, function)) if not handlers: continue extension = Extension.fromId(db, extension_id) manifest = Manifest.load(getExtensionInstallPath(version_sha1)) else: extension = Extension.fromId(db, extension_id) manifest = Manifest.load(extension.getPath()) for role in manifest.roles: if isinstance(role, InjectRole): path, query = get_matching_path(role.regexp) if path is not None: handlers.append( (path, query, role.script, role.function)) if not handlers: continue def construct_query(query): if not query: return "null" params = urlparse.parse_qs(query, keep_blank_values=True) for key in params: values = params[key] if len(values) == 1: if not values[0]: params[key] = None else: params[key] = values[0] return ( "Object.freeze({ raw: %s, params: Object.freeze(%s) })" % (json_encode(query), json_encode(params))) preferences = None commands = [] for path, query, script, function in handlers: argv = "[%s, %s]" % (jsify(path), construct_query(query)) try: stdout_data = executeProcess( manifest, "inject", script, function, extension_id, user.id, argv, configuration.extensions.SHORT_TIMEOUT) except ProcessTimeout: raise InjectError("Timeout after %d seconds." % configuration.extensions.SHORT_TIMEOUT) except ProcessError as error: if error.returncode < 0: raise InjectError("Process terminated by signal %d." % -error.returncode) else: raise InjectError("Process returned %d.\n%s" % (error.returncode, error.stderr)) for line in stdout_data.splitlines(): if line.strip(): commands.append(processLine(path, line.strip())) for command, value in commands: if command == "script": document.addExternalScript(value, use_static=False, order=1) elif command == "stylesheet": document.addExternalStylesheet(value, use_static=False, order=1) elif command == "link": for index, (_, label, _, _) in enumerate(links): if label == value["label"]: if value["url"] is None: del links[index] else: links[index][0] = value["url"] break else: if value["url"] is not None: links.append( [value["url"], value["label"], None, None]) elif command == "preference": if not preferences: preferences = [] injected.setdefault("preferences", []).append( (extension.getName(), extension.getAuthor(db), preferences)) preferences.append(value) if profiler: profiler.check("inject: %s" % extension.getKey()) except ExtensionError as error: document.comment( "\n\n[%s] Extension error:\nInvalid extension:\n%s\n\n" % (error.extension.getKey(), error.message)) except ManifestError as error: document.comment( "\n\n[%s] Extension error:\nInvalid MANIFEST:\n%s\n\n" % (extension.getKey(), error.message)) except InjectError as error: document.comment("\n\n[%s] Extension error:\n%s\n\n" % (extension.getKey(), error.message))
def execute(db, user, review, all_commits, old_head, new_head, output): cursor = db.cursor() cursor.execute("""SELECT extensions.id, extensions.author, extensions.name, extensionversions.sha1, roles.script, roles.function FROM extensions JOIN extensionversions ON (extensionversions.extension=extensions.id) JOIN extensionroles_processcommits AS roles ON (roles.version=extensionversions.id) WHERE uid=%s AND filter IS NULL""", (user.id,)) rows = cursor.fetchall() if rows: commitset = log.commitset.CommitSet(all_commits) assert old_head is None or old_head in commitset.getTails() assert new_head in commitset.getHeads() assert len(commitset.getHeads()) == 1 tails = commitset.getFilteredTails(review.repository) if len(tails) == 1: tail = gitutils.Commit.fromSHA1(db, review.repository, tails.pop()) changeset_id = changeset.utils.createChangeset(db, user, review.repository, from_commit=tail, to_commit=new_head)[0].id changeset_arg = "repository.getChangeset(%d)" % changeset_id db.commit() else: changeset_arg = "null" commits = "[%s]" % ",".join([("repository.getCommit(%d)" % commit.getId(db)) for commit in all_commits]) data = { "review_id": review.id, "changeset": changeset_arg, "commits": commits } for extension_id, author_id, extension_name, sha1, script, function in rows: author = dbutils.User.fromId(db, author_id) if sha1 is None: extension_path = getExtensionPath(author.name, extension_name) else: extension_path = getExtensionInstallPath(sha1) class Error(Exception): pass def print_header(): header = "%s::%s()" % (script, function) print >>output, "\n[%s] %s\n[%s] %s" % (extension_name, header, extension_name, "=" * len(header)) try: try: manifest = Manifest.load(extension_path) except ManifestError, error: raise Error("Invalid MANIFEST:\n%s" % error.message) for role in manifest.roles: if isinstance(role, ProcessCommitsRole) and role.script == script and role.function == function: break else: continue argv = """ (function () { var review = new critic.Review(%(review_id)d); var repository = review.repository; var changeset = %(changeset)s; var commitset = new critic.CommitSet(%(commits)s); return [review, changeset, commitset]; })() """ % data argv = re.sub("[ \n]+", " ", argv.strip()) try: stdout_data = executeProcess(manifest, role, extension_id, user.id, argv, configuration.extensions.SHORT_TIMEOUT) except ProcessTimeout: raise Error("Timeout after %d seconds." % configuration.extensions.SHORT_TIMEOUT) except ProcessError, error: if error.returncode < 0: if -error.returncode == signal.SIGXCPU: raise Error("CPU time limit (5 seconds) exceeded.") else: raise Error("Process terminated by signal %d." % -error.returncode) else: raise Error("Process returned %d.\n%s" % (error.returncode, error.stderr)) if stdout_data.strip(): print_header() for line in stdout_data.splitlines(): print >>output, "[%s] %s" % (extension_name, line) except Error, error: print_header() print >>output, "[%s] Extension error: %s" % (extension_name, error.message)
def execute(db, req, user): cursor = db.cursor() cursor.execute("""SELECT extensions.id, extensions.author, extensions.name, extensionversions.sha1, roles.path, roles.script, roles.function FROM extensions JOIN extensionversions ON (extensionversions.extension=extensions.id) JOIN extensionroles_page AS roles ON (roles.version=extensionversions.id) WHERE uid=%s""", (user.id,)) for extension_id, author_id, extension_name, sha1, regexp, script, function in cursor: if re.match(regexp, req.path): def param(raw): parts = raw.split("=", 1) if len(parts) == 1: return "%s: null" % jsify(decodeURIComponent(raw)) else: return "%s: %s" % (jsify(decodeURIComponent(parts[0])), jsify(decodeURIComponent(parts[1]))) if req.query: query = "Object.freeze({ raw: %s, params: Object.freeze({ %s }) })" % (jsify(req.query), ", ".join(map(param, req.query.split("&")))) else: query = "null" headers = "Object.create(null, { %s })" % ", ".join(["%s: { value: %s, enumerable: true }" % (jsify(name), jsify(value)) for name, value in req.getRequestHeaders().items()]) author = dbutils.User.fromId(db, author_id) if sha1 is None: extension_path = os.path.join(configuration.extensions.SEARCH_ROOT, author.name, "CriticExtensions", extension_name) else: extension_path = os.path.join(configuration.extensions.INSTALL_DIR, sha1) manifest = Manifest.load(extension_path) for role in manifest.roles: if isinstance(role, PageRole) and role.regexp == regexp and role.script == script and role.function == function: break else: continue argv = ("[%(method)s, %(path)s, %(query)s, %(headers)s]" % { 'method': jsify(req.method), 'path': jsify(req.path), 'query': query, 'headers': headers }) if req.method == "POST": stdin_data = req.read() else: stdin_data = None before = time.time() try: stdout_data = executeProcess(manifest, role, extension_id, user.id, argv, configuration.extensions.LONG_TIMEOUT, stdin=stdin_data, rlimit_cpu=60) except ProcessTimeout: req.setStatus(500, "Extension Timeout") return "Extension timed out!" except ProcessError, error: req.setStatus(500, "Extension Failure") if error.returncode < 0: if -error.returncode == signal.SIGXCPU: return "Extension failure: time limit (5 CPU seconds) exceeded\n" else: return "Extension failure: terminated by signal %d\n" % -error.returncode else: return "Extension failure: returned %d\n%s" % (error.returncode, error.stderr) after = time.time() status = None headers = {} if not stdout_data: return False while True: try: line, stdout_data = stdout_data.split("\n", 1) except: req.setStatus(500, "Extension Error") return "Extension error: output format error.\n%r\n" % stdout_data if status is None: try: status = int(line.strip()) except: req.setStatus(500, "Extension Error") return "Extension error: first line should contain only a numeric HTTP status code.\n%r\n" % line elif not line: break else: try: name, value = line.split(":", 1) except: req.setStatus(500, "Extension Error") return "Extension error: header line should be on 'name: value' format.\n%r\n" % line headers[name.strip()] = value.strip() if status is None: req.setStatus(500, "Extension Error") return "Extension error: first line should contain only a numeric HTTP status code.\n" content_type = "text/plain" for name, value in headers.items(): if name.lower() == "content-type": content_type = value del headers[name] else: headers[name] = value req.setStatus(status) req.setContentType(content_type) for name, value in headers.items(): req.addResponseHeader(name, value) if content_type == "text/tutorial": req.setContentType("text/html") return renderTutorial(db, user, stdout_data) if content_type.startswith("text/html"): stdout_data += "\n\n<!-- extension execution time: %.2f seconds -->\n" % (after - before) return stdout_data