def renderLocalRepository(target): page.utils.generateRepositorySelect(db, user, target) cursor.execute("""SELECT repositories.id, repositories.name, repositories.path, branches.name FROM repositories LEFT OUTER JOIN branches ON (branches.id=repositories.branch) ORDER BY id""") for repository_id, name, path, branch_name in cursor.fetchall(): local_names = ["*"] if branch_name: local_names.append(branch_name) cursor.execute("""SELECT remote FROM trackedbranches WHERE repository=%s AND local_name=ANY (%s) ORDER BY local_name LIMIT 1""", (repository_id, local_names)) row = cursor.fetchone() if row: default_remotes[name] = row[0] else: default_remotes[name] = None default_branches[name] = branch_name document.addInternalScript("var default_remotes = %s;" % json_encode(default_remotes)) document.addInternalScript("var default_branches = %s;" % json_encode(default_branches))
def renderLocalRepository(target): page.utils.generateRepositorySelect(db, user, target) cursor.execute("""SELECT repositories.id, repositories.name, repositories.path FROM repositories ORDER BY repositories.id""") for repository_id, name, path in cursor.fetchall(): def findRemote(local_name): cursor.execute("""SELECT remote FROM trackedbranches WHERE repository=%s AND local_name=%s""", (repository_id, local_name)) row = cursor.fetchone() if row: return row[0] repository = gitutils.Repository.fromId(db, repository_id) remote = branch_name = None for branch in repository.getSignificantBranches(db): remote = findRemote(branch.name) if remote: branch_name = branch.name break if not remote: remote = findRemote("*") default_remotes[name] = remote default_branches[name] = branch_name document.addInternalScript("var default_remotes = %s;" % json_encode(default_remotes)) document.addInternalScript("var default_branches = %s;" % json_encode(default_branches))
def renderLocalRepository(target): repositories = target.select("repository") cursor.execute("""SELECT repositories.id, repositories.name, repositories.path, branches.name FROM repositories LEFT OUTER JOIN branches ON (branches.id=repositories.branch) ORDER BY id""") for repository_id, name, path, branch_name in cursor.fetchall(): option = repositories.option("repository", value=name, selected="selected" if name == default_repository else None) option.text("%s [%s]" % (name, gitutils.Repository.constructURL(db, user, path))) local_names = ["*"] if branch_name: local_names.append(branch_name) cursor.execute("""SELECT remote FROM trackedbranches WHERE repository=%s AND local_name=ANY (%s) ORDER BY local_name LIMIT 1""", (repository_id, local_names)) row = cursor.fetchone() if row: default_remotes[name] = row[0] else: default_remotes[name] = None default_branches[name] = branch_name document.addInternalScript("var default_remotes = %s;" % json_encode(default_remotes)) document.addInternalScript("var default_branches = %s;" % json_encode(default_branches))
def renderLocalRepository(target): page.utils.generateRepositorySelect(db, user, target) cursor.execute( """SELECT repositories.id, repositories.name, repositories.path, branches.name FROM repositories LEFT OUTER JOIN branches ON (branches.id=repositories.branch) ORDER BY repositories.id""") for repository_id, name, path, branch_name in cursor.fetchall(): local_names = ["*"] if branch_name: local_names.append(branch_name) cursor.execute( """SELECT remote FROM trackedbranches WHERE repository=%s AND local_name=ANY (%s) ORDER BY local_name LIMIT 1""", (repository_id, local_names)) row = cursor.fetchone() if row: default_remotes[name] = row[0] else: default_remotes[name] = None default_branches[name] = branch_name document.addInternalScript("var default_remotes = %s;" % json_encode(default_remotes)) document.addInternalScript("var default_branches = %s;" % json_encode(default_branches))
def renderLocalRepository(target): repositories = target.select("repository") cursor.execute( """SELECT repositories.id, repositories.name, repositories.path, branches.name FROM repositories LEFT OUTER JOIN branches ON (branches.id=repositories.branch) ORDER BY id""" ) for repository_id, name, path, branch_name in cursor: option = repositories.option( "repository", value=name, selected="selected" if name == default_repository else None ) option.text("%s [%s:%s]" % (name, configuration.base.HOSTNAME, path)) local_names = ["*"] if branch_name: local_names.append(branch_name) cursor.execute( """SELECT remote FROM trackedbranches WHERE repository=%s AND local_name=ANY (%s) ORDER BY local_name LIMIT 1""", (repository_id, local_names), ) def splitRemote(remote): if remote.startswith("git://"): host, path = remote[6:].split("/", 1) host = "git://" + host else: host, path = remote.split(":", 1) return host, path row = cursor.fetchone() if row: default_remotes[name] = splitRemote(row[0]) else: default_remotes[name] = None default_branches[name] = branch_name document.addInternalScript("var default_remotes = %s;" % json_encode(default_remotes)) document.addInternalScript("var default_branches = %s;" % json_encode(default_branches))
def renderLocalRepository(target): repositories = target.select("repository") cursor.execute( """SELECT repositories.id, repositories.name, repositories.path, branches.name FROM repositories LEFT OUTER JOIN branches ON (branches.id=repositories.branch) ORDER BY id""") for repository_id, name, path, branch_name in cursor: option = repositories.option( "repository", value=name, selected="selected" if name == default_repository else None) option.text("%s [%s:%s]" % (name, configuration.base.HOSTNAME, path)) local_names = ["*"] if branch_name: local_names.append(branch_name) cursor.execute( """SELECT remote FROM trackedbranches WHERE repository=%s AND local_name=ANY (%s) ORDER BY local_name LIMIT 1""", (repository_id, local_names)) def splitRemote(remote): if remote.startswith("git://"): host, path = remote[6:].split("/", 1) host = "git://" + host else: host, path = remote.split(":", 1) return host, path row = cursor.fetchone() if row: default_remotes[name] = splitRemote(row[0]) else: default_remotes[name] = None default_branches[name] = branch_name document.addInternalScript("var default_remotes = %s;" % json_encode(default_remotes)) document.addInternalScript("var default_branches = %s;" % json_encode(default_branches))
def execute_command(self, client, command): client.write( json_encode({ "status": "error", "error": "command not supported" })) client.close()
def process(self, db, user, service_name): if not user.hasRole(db, "administrator"): raise OperationFailure( code="notallowed", title="Not allowed!", message="Only a system administrator can restart services.") if service_name == "wsgi": for pid in os.listdir(configuration.paths.WSGI_PIDFILE_DIR): try: os.kill(int(pid), signal.SIGINT) except: pass return OperationResult() else: connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) connection.connect( configuration.services.SERVICEMANAGER["address"]) connection.send( textutils.json_encode({ "command": "restart", "service": service_name })) connection.shutdown(socket.SHUT_WR) data = "" while True: received = connection.recv(4096) if not received: break data += received result = textutils.json_decode(data) if result["status"] == "ok": return OperationResult() else: raise OperationError, result["error"]
def __str__(self): return textutils.json_encode({ "status": "failure", "code": self.__code, "title": self.__title, "message": self.__message })
def construct_query(query): if not query: return "null" params = urlparse.parse_qs(query, keep_blank_values=True) for key in params: values = params[key] if len(values) == 1: if not values[0]: params[key] = None else: params[key] = values[0] return ("Object.freeze({ raw: %s, params: Object.freeze(%s) })" % (json_encode(query), json_encode(params)))
def handle_input(self, _file, data): lines = data.splitlines() user_name = lines[0] # The second line is the value of the REMOTE_USER environment # variable (from the environment with which the git hook ran.) # # We use it as the actual user only if the actual user was the # Critic system user, meaning the push was performed by the # branch tracker service, the web front-end (for instance via # 'git http-backend') or an extension. if user_name == configuration.base.SYSTEM_USER_NAME and lines[1]: user_name = lines[1] self.__request = { "user_name": user_name, "repository_name": lines[2], "flags": lines[3], "refs": [{ "name": name, "old_sha1": old_sha1, "new_sha1": new_sha1 } for old_sha1, new_sha1, name in map(str.split, lines[4:])] } self.server.info("session started: %s / %s" % (self.__request["user_name"], self.__request["repository_name"])) child_process = GitHookServer.ChildProcess(self.server, self) child_process.write(json_encode(self.__request)) child_process.close() self.server.add_peer(child_process)
def finished(self, process): if process.did_time_out: status = status_text = "timeout" else: status = "ok" if process.returncode == 0: status_text = "success" else: status_text = "error(%d)" % process.returncode self.server.debug("Process finished: %s [pid=%d]" % (status_text, process.pid)) if process.stdout: self.server.debug(" stdout=%r" % process.stdout) if process.stderr: self.server.debug(" stderr=%r" % process.stderr) self.write(textutils.json_encode({ "status": status, "stdout": process.stdout, "stderr": process.stderr, "returncode": process.returncode })) self.close()
def handle_input(self, data): lines = data.splitlines() self.__request = { "user_name": lines[0], "repository_name": lines[1], "refs": [{ "name": name, "old_sha1": old_sha1, "new_sha1": new_sha1 } for old_sha1, new_sha1, name in map(str.split, lines[2:])] } self.server.info("session started: %s / %s" % (self.__request["user_name"], self.__request["repository_name"])) child_process = GitHookServer.ChildProcess(self.server, self) child_process.write(json_encode(self.__request)) child_process.close() self.server.add_peer(child_process)
def handle_input(self, data): lines = data.splitlines() user_name = lines[0] # The second line is the value of the REMOTE_USER environment # variable (from the environment with which the git hook ran.) # # We use it as the actual user only if the actual user was the # Critic system user, meaning the push was performed by the # branch tracker service, the web front-end (for instance via # 'git http-backend') or an extension. if user_name == configuration.base.SYSTEM_USER_NAME and lines[1]: user_name = lines[1] self.__request = { "user_name": user_name, "repository_name": lines[2], "flags": lines[3], "refs": [{ "name": name, "old_sha1": old_sha1, "new_sha1": new_sha1 } for old_sha1, new_sha1, name in map(str.split, lines[4:])] } self.server.info("session started: %s / %s" % (self.__request["user_name"], self.__request["repository_name"])) child_process = GitHookServer.ChildProcess(self.server, self) child_process.write(json_encode(self.__request)) child_process.close() self.server.add_peer(child_process)
def process(self, db, user, service_name): if not user.hasRole(db, "administrator"): raise OperationFailure( code="notallowed", title="Not allowed!", message="Only a system administrator can restart services." ) if service_name == "wsgi": for pid in os.listdir(configuration.paths.WSGI_PIDFILE_DIR): try: os.kill(int(pid), signal.SIGINT) except: pass return OperationResult() else: connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) connection.connect(configuration.services.SERVICEMANAGER["address"]) connection.send(textutils.json_encode({"command": "restart", "service": service_name})) connection.shutdown(socket.SHUT_WR) data = "" while True: received = connection.recv(4096) if not received: break data += received result = textutils.json_decode(data) if result["status"] == "ok": return OperationResult() else: raise OperationError, result["error"]
def construct_query(query): if not query: return "null" params = urlparse.parse_qs(query, keep_blank_values=True) for key in params: values = params[key] if len(values) == 1: if not values[0]: params[key] = None else: params[key] = values[0] return ( "Object.freeze({ raw: %s, params: Object.freeze(%s) })" % (json_encode(query), json_encode(params)))
def __call__(self, req, db, user): from operation.typechecker import TypeCheckerContext if user.isAnonymous() and not self.__accept_anonymous_user: return OperationFailureMustLogin() if req.method == "POST": data = req.read() else: data = req.getParameter("data") if not data: raise OperationError("no input") try: value = json_decode(data) except ValueError as error: raise OperationError("invalid input: %s" % str(error)) try: self.__checker(value, TypeCheckerContext(req, db, user)) return self.process(db, user, **value) except OperationError as error: return error except OperationFailure as failure: return failure except dbutils.NoSuchUser as error: return OperationFailure(code="nosuchuser", title="Who is '%s'?" % error.name, message="There is no user in Critic's database named that.") except dbutils.NoSuchReview as error: return OperationFailure(code="nosuchreview", title="Invalid review ID", message="The review ID r/%d is not valid." % error.id) except dbutils.TransactionRollbackError: return OperationFailure(code="transactionrollback", title="Transaction rolled back", message="Your database transaction rolled back, probably due to a deadlock. Please try again.") except: # Decode value again since the type checkers might have modified it. value = json_decode(data) error_message = ("User: %s\nReferrer: %s\nData: %s\n\n%s" % (user.name, req.getReferrer(), json_encode(self.sanitize(value), indent=2), traceback.format_exc())) db.rollback() import mailutils import configuration if not user.hasRole(db, "developer"): mailutils.sendExceptionMessage(db, "wsgi[%s]" % req.path, error_message) if configuration.debug.IS_DEVELOPMENT or user.hasRole(db, "developer"): return OperationError(error_message) else: return OperationError("An unexpected error occurred. " + "A message has been sent to the system administrator(s) " + "with details about the problem.")
def add_result(self, result): if self.__async: # Client is already gone, so we don't really care about the # results. return self.__results.append(result) if len(self.__results) == len(self.__requests): self.write(json_encode(self.__results)) self.close()
def __init__(self, server, clients, request): super(JSONJobServer.Job, self).__init__(server, [sys.executable, sys.argv[0], "--json-job"], stderr=subprocess.STDOUT) self.clients = clients self.request = request self.write(json_encode(request)) self.close()
def perform_job(): import syntaxhighlight.generate request = json_decode(sys.stdin.read()) request["highlighted"] = syntaxhighlight.generate.generateHighlight( repository_path=request["repository_path"], sha1=request["sha1"], language=request["language"]) sys.stdout.write(json_encode(request))
def execute_command(self, client, command): if command["command"] == "purge": purged_count = self.__purge() client.write(json_encode({ "status": "ok", "purged": purged_count })) client.close() else: super(ChangesetServer, self).execute_command(client, command)
def execute_command(self, client, command): if command["command"] == "purge": purged_count = self.__purge() client.write(json_encode({ "status": "ok", "purged": purged_count })) client.close() else: super(ChangesetServer, self).execute_command(client, command)
def perform_job(): import syntaxhighlight.generate request = json_decode(sys.stdin.read()) request["highlighted"] = syntaxhighlight.generate.generateHighlight( repository_path=request["repository_path"], sha1=request["sha1"], language=request["language"]) sys.stdout.write(json_encode(request))
def execute_command(self, client, command): if command["command"] == "compact": uncompressed_count, compressed_count, purged_files_count, purged_contexts_count = self.__compact() client.write(json_encode({ "status": "ok", "uncompressed": uncompressed_count, "compressed": compressed_count, "purged_files": purged_files_count, "purged_contexts": purged_contexts_count })) client.close() else: super(HighlightServer, self).execute_command(client, command)
def handle_input(self, data): lines = data.splitlines() self.__request = { "user_name": lines[0], "repository_name": lines[1], "refs": [{ "name": name, "old_sha1": old_sha1, "new_sha1": new_sha1 } for old_sha1, new_sha1, name in map(str.split, lines[2:])] } self.server.info("session started: %s / %s" % (self.__request["user_name"], self.__request["repository_name"])) child_process = GitHookServer.ChildProcess(self.server, self) child_process.write(json_encode(self.__request)) child_process.close() self.server.add_peer(child_process)
def __init__(self, client, branch_id, timeout, log_offset): super(BranchTrackerHook.WaitForUpdate, self).__init__( client.server, [sys.executable, sys.argv[0], "--wait-for-update"], stderr=STDOUT) self.client = client self.client.write("wait\n") self.write( json_encode({ "branch_id": branch_id, "timeout": timeout, "log_offset": log_offset })) self.close()
def perform_job(): soft_limit, hard_limit = getrlimit(RLIMIT_RSS) rss_limit = configuration.services.CHANGESET["rss_limit"] if soft_limit < rss_limit: setrlimit(RLIMIT_RSS, (rss_limit, hard_limit)) from changeset.create import createChangeset request = json_decode(sys.stdin.read()) try: db = dbutils.Database() createChangeset(db, request) db.close() sys.stdout.write(json_encode(request)) except: print "Request:" print json_encode(request, indent=2) print print_exc(file=sys.stdout)
def perform_job(): soft_limit, hard_limit = getrlimit(RLIMIT_RSS) rss_limit = configuration.services.CHANGESET["rss_limit"] if soft_limit < rss_limit: setrlimit(RLIMIT_RSS, (rss_limit, hard_limit)) from changeset.create import createChangeset request = json_decode(sys.stdin.read()) try: db = dbutils.Database() createChangeset(db, request) db.close() sys.stdout.write(json_encode(request)) except: print "Request:" print json_encode(request, indent=2) print print_exc(file=sys.stdout)
def execute_command(self, client, command): if command["command"] == "compact": uncompressed_count, compressed_count, purged_files_count, purged_contexts_count = self.__compact( ) client.write( json_encode({ "status": "ok", "uncompressed": uncompressed_count, "compressed": compressed_count, "purged_files": purged_files_count, "purged_contexts": purged_contexts_count })) client.close() else: super(HighlightServer, self).execute_command(client, command)
def requestChangesets(requests): try: connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) connection.connect(configuration.services.CHANGESET["address"]) connection.send(json_encode(requests)) connection.shutdown(socket.SHUT_WR) data = "" while True: received = connection.recv(4096) if not received: break data += received connection.close() except socket.error, error: raise Exception, "Changeset background service failed: %s" % error[1]
def executeProcess(manifest, role, extension_id, user_id, argv, timeout, stdin=None, rlimit_cpu=5, rlimit_rss=256): flavor = manifest.flavor if manifest.flavor not in configuration.extensions.FLAVORS: flavor = configuration.extensions.DEFAULT_FLAVOR executable = configuration.extensions.FLAVORS[flavor]["executable"] library = configuration.extensions.FLAVORS[flavor]["library"] process_argv = [executable, "--rlimit-cpu=%ds" % rlimit_cpu, "--rlimit-rss=%dm" % rlimit_rss, os.path.join(library, "critic-launcher.js")] stdin_data = "%s\n" % json_encode({ "criticjs_path": os.path.join(library, "critic2.js"), "rlimit": { "cpu": rlimit_cpu, "rss": rlimit_rss }, "hostname": configuration.base.HOSTNAME, "dbname": configuration.database.PARAMETERS["database"], "dbuser": configuration.database.PARAMETERS["user"], "git": configuration.executables.GIT, "python": configuration.executables.PYTHON, "python_path": "%s:%s" % (configuration.paths.CONFIG_DIR, configuration.paths.INSTALL_DIR), "repository_work_copy_path": os.path.join(configuration.paths.DATA_DIR, "temporary", "EXTENSIONS"), "changeset_address": configuration.services.CHANGESET["address"], "maildelivery_pid_path": configuration.services.MAILDELIVERY["pidfile_path"], "is_development": configuration.debug.IS_DEVELOPMENT, "extension_path": manifest.path, "extension_id": extension_id, "user_id": user_id, "role": role.name(), "script_path": role.script, "fn": role.function, "argv": argv }) if stdin is not None: stdin_data += stdin process = subprocess.Popen(process_argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=manifest.path) communicate = Communicate(process) communicate.setInput(stdin_data) communicate.setTimout(timeout) return communicate.run()[0]
def requestChangesets(requests): try: connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) connection.connect(configuration.services.CHANGESET["address"]) connection.send(json_encode(requests)) connection.shutdown(socket.SHUT_WR) data = "" while True: received = connection.recv(4096) if not received: break data += received connection.close() except socket.error, error: raise Exception, "Changeset background service failed: %s" % error[1]
def suggestreview(req, db, _user): repository_id = req.getParameter("repository", filter=int) sha1 = req.getParameter("sha1") repository = gitutils.Repository.fromId(db, repository_id) commit = gitutils.Commit.fromSHA1(db, repository, sha1) cursor = db.cursor() suggestions = {} def addSuggestions(): for review_id, summary in cursor: review = dbutils.Review.fromId(db, review_id, load_commits=False) if review.state != "dropped": suggestions[str(review_id)] = "(%s) %s" % (review.getReviewState(db), summary) summary = commit.summary() while True: match = re.search("[A-Z][A-Z0-9]*-[0-9]+", summary) if match: pattern = "r/%" + match.group(0) + "%" cursor.execute( """SELECT reviews.id, reviews.summary FROM reviews JOIN branches ON (reviews.branch=branches.id) WHERE branches.name LIKE %s""", (pattern,), ) addSuggestions() summary = summary[match.end() :] else: break cursor.execute( """SELECT reviews.id, reviews.summary FROM reviews WHERE reviews.summary=%s""", (commit.summary(),), ) addSuggestions() return json_encode(suggestions)
def requestChangesets(requests): try: connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) connection.connect(configuration.services.CHANGESET["address"]) connection.send(json_encode(requests)) connection.shutdown(socket.SHUT_WR) data = "" while True: received = connection.recv(4096) if not received: break data += received connection.close() except socket.error as error: raise ChangesetBackgroundServiceError(error[1]) try: results = json_decode(data) except ValueError: raise ChangesetBackgroundServiceError( "returned an invalid response: %r" % data) if type(results) != list: # If not a list, the result is probably an error message. raise ChangesetBackgroundServiceError(str(results)) if len(results) != len(requests): raise ChangesetBackgroundServiceError("didn't process all requests") errors = [] for result in results: if "error" in result: errors.append(result["error"]) if errors: raise ChangesetBackgroundServiceError( "one or more requests failed:\n%s" % "\n".join(map(indent, errors)))
def requestChangesets(requests): try: connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) connection.connect(configuration.services.CHANGESET["address"]) connection.send(json_encode(requests)) connection.shutdown(socket.SHUT_WR) data = "" while True: received = connection.recv(4096) if not received: break data += received connection.close() except socket.error as error: raise ChangesetBackgroundServiceError(error[1]) try: results = json_decode(data) except ValueError: raise ChangesetBackgroundServiceError( "returned an invalid response: %r" % data) if type(results) != list: # If not a list, the result is probably an error message. raise ChangesetBackgroundServiceError(str(results)) if len(results) != len(requests): raise ChangesetBackgroundServiceError("didn't process all requests") errors = [] for result in results: if "error" in result: errors.append(result["error"]) if errors: raise ChangesetBackgroundServiceError( "one or more requests failed:\n%s" % "\n".join(map(indent, errors)))
def suggestreview(req, db, _user): repository_id = req.getParameter("repository", filter=int) sha1 = req.getParameter("sha1") repository = gitutils.Repository.fromId(db, repository_id) commit = gitutils.Commit.fromSHA1(db, repository, sha1) cursor = db.cursor() suggestions = {} def addSuggestions(): for review_id, summary in cursor: review = dbutils.Review.fromId(db, review_id) if review.state != 'dropped': suggestions[str(review_id)] = "(%s) %s" % ( review.getReviewState(db), summary) summary = commit.summary() while True: match = re.search("[A-Z][A-Z0-9]*-[0-9]+", summary) if match: pattern = "r/%" + match.group(0) + "%" cursor.execute( """SELECT reviews.id, reviews.summary FROM reviews JOIN branches ON (reviews.branch=branches.id) WHERE branches.name LIKE %s""", (pattern, )) addSuggestions() summary = summary[match.end():] else: break cursor.execute( """SELECT reviews.id, reviews.summary FROM reviews WHERE reviews.summary=%s""", (commit.summary(), )) addSuggestions() return json_encode(suggestions)
def requestHighlights(repository, sha1s): requests = [{ "repository_path": repository.path, "sha1": sha1, "path": path, "language": language } for sha1, (path, language) in sha1s.items() if not syntaxhighlight.isHighlighted(sha1, language)] if not requests: return try: connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) connection.connect(configuration.services.HIGHLIGHT["address"]) connection.send(json_encode(requests)) connection.shutdown(socket.SHUT_WR) data = "" while True: received = connection.recv(4096) if not received: break data += received connection.close() except socket.error, error: raise HighlightBackgroundServiceError(error[1])
def requestHighlights(repository, sha1s): requests = [{ "repository_path": repository.path, "sha1": sha1, "path": path, "language": language } for sha1, (path, language) in sha1s.items() if not syntaxhighlight.isHighlighted(sha1, language)] if not requests: return try: connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) connection.connect(configuration.services.HIGHLIGHT["address"]) connection.send(json_encode(requests)) connection.shutdown(socket.SHUT_WR) data = "" while True: received = connection.recv(4096) if not received: break data += received connection.close() except socket.error as error: raise HighlightBackgroundServiceError(error[1]) try: results = json_decode(data) except ValueError: raise HighlightBackgroundServiceError( "returned an invalid response (%r)" % data) if type(results) != list: # If not a list, the result is probably an error message. raise HighlightBackgroundServiceError(str(results)) if len(results) != len(requests): raise HighlightBackgroundServiceError("didn't process all requests")
modules.add(moduleFromFile(file_id)) teams_per_modules.setdefault(frozenset(modules), set()).update(team) for modules, team in teams_per_modules.items(): row = shared.tr("reviewers") cell = row.td("reviewers") members = sorted([dbutils.User.fromId(db, user_id).fullname for user_id in team]) for member in members: cell.text(member).br() row.td("willreview").innerHTML("<span class='also'>also</span> review changes in") cell = row.td("files") for path in diff.File.eliminateCommonPrefixes(sorted(modules)): cell.span("file").innerHTML(path).br() paths = json_encode(list(modules)) user_ids = json_encode(list(team)) cell = row.td("buttons") cell.button("accept", critic_paths=paths, critic_user_ids=user_ids).text("I will review this!") cell.button("deny", critic_paths=paths, critic_user_ids=user_ids).text("They will review this!") yield flush(target) profiler.check("shared assignments") cursor.execute("SELECT batches.id, users.fullname, batches.comment, batches.time FROM batches JOIN users ON (users.id=batches.uid) WHERE batches.review=%s ORDER BY batches.id DESC", [review.id]) rows = cursor.fetchall() if rows: notes = dict([(chain.id, chain) for chain in open_notes])
def wrap(raw_source, mode): if mode == "json": return "\n".join( textutils.json_encode([[None, line]]) for line in diff.parse.splitlines(raw_source)) return htmlutils.htmlify(raw_source)
def process_request(environ, start_response): request_start = time.time() critic = api.critic.startSession(for_user=True) db = critic.database user = None try: try: req = request.Request(db, environ, start_response) # Handle static resources very early. We don't bother with checking # for an authenticated user; static resources aren't sensitive, and # are referenced from special-case resources like the login page and # error messages that, that we want to display even if something # went wrong with the authentication. if req.path.startswith("static-resource/"): return handleStaticResource(req) if req.path.startswith("externalauth/"): provider_name = req.path[len("externalauth/"):] if provider_name in auth.PROVIDERS: provider = auth.PROVIDERS[provider_name] authorize_url = provider.start(db, req) if authorize_url: raise request.Found(authorize_url) if req.path.startswith("oauth/"): provider_name = req.path[len("oauth/"):] if provider_name in auth.PROVIDERS: provider = auth.PROVIDERS[provider_name] if isinstance(provider, auth.OAuthProvider): finishOAuth(db, req, provider) auth.checkSession(db, req) auth.AccessControl.accessHTTP(db, req) user = req.user user.loadPreferences(db) if user.status == 'retired': # If a retired user accesses the system, change the status back # to 'current' again. with db.updating_cursor("users") as cursor: cursor.execute("""UPDATE users SET status='current' WHERE id=%s""", (user.id,)) user.status = 'current' if not user.getPreference(db, "debug.profiling.databaseQueries"): db.disableProfiling() original_path = req.path if not req.path: if user.isAnonymous(): location = "tutorial" else: location = user.getPreference(db, "defaultPage") if req.query: location += "?" + req.query raise request.MovedTemporarily(location) if req.path == "redirect": target = req.getParameter("target", "/") raise request.SeeOther(target) if req.path == "findreview": # This raises either DisplayMessage or MovedTemporarily. findreview(req, db) # Require a .git suffix on HTTP(S) repository URLs unless the user- # agent starts with "git/" (as Git's normally does.) # # Our objectives are: # # 1) Not to require Git's user-agent to be its default value, since # the user might have to override it to get through firewalls. # 2) Never to send regular user requests to 'git http-backend' by # mistake. # # This is a compromise. if req.getRequestHeader("User-Agent", "").startswith("git/"): suffix = None else: suffix = ".git" if handleRepositoryPath(db, req, user, suffix): db = None return [] # Extension "page" roles. Prefixing a path with "!/" bypasses all # extensions. # # Also bypass extensions if the user is anonymous unless general # anonymous access is allowed. If it's not and the user is still # anonymous, access was allowed because of a path-based exception, # which was not intended to allow access to an extension. if req.path.startswith("!/"): req.path = req.path[2:] elif configuration.extensions.ENABLED: handled = extensions.role.page.execute(db, req, user) if isinstance(handled, basestring): req.start() return [handled] if req.path.startswith("r/"): req.updateQuery({ "id": [req.path[2:]] }) req.path = "showreview" if configuration.extensions.ENABLED: match = RE_EXTENSION_RESOURCE.match(req.path) if match: content_type, resource = extensions.resource.get(req, db, user, match.group(1)) if resource: req.setContentType(content_type) if content_type.startswith("image/"): req.addResponseHeader("Cache-Control", "max-age=3600") req.start() return [resource] else: req.setStatus(404) req.start() return [] if req.path.startswith("download/"): return handleDownload(db, req, user) if req.path == "api" or req.path.startswith("api/"): try: result = jsonapi.handleRequest(critic, req) except jsonapi.Error as error: req.setStatus(error.http_status) result = { "error": { "title": error.title, "message": error.message }} else: req.setStatus(200) accept_header = req.getRequestHeader("Accept") if accept_header == "application/vnd.api+json": default_indent = None else: default_indent = 2 indent = req.getParameter("indent", default_indent, filter=int) if indent == 0: # json.encode(..., indent=0) still gives line-breaks, just # no indentation. This is not so useful, so set indent to # None instead, which disables formatting entirely. indent = None req.setContentType("application/vnd.api+json") req.start() return [json_encode(result, indent=indent)] operationfn = OPERATIONS.get(req.path) if operationfn: result = operationfn(req, db, user) if isinstance(result, (OperationResult, OperationError)): req.setContentType("text/json") if isinstance(result, OperationResult): if db.profiling: result.set("__profiling__", formatDBProfiling(db)) result.set("__time__", time.time() - request_start) elif not req.hasContentType(): req.setContentType("text/plain") req.start() if isinstance(result, unicode): return [result.encode("utf8")] else: return [str(result)] impersonate_user = user if not user.isAnonymous(): user_parameter = req.getParameter("user", None) if user_parameter: impersonate_user = dbutils.User.fromName(db, user_parameter) while True: pagefn = PAGES.get(req.path) if pagefn: try: result = pagefn(req, db, impersonate_user) if db.profiling and not (isinstance(result, str) or isinstance(result, Document)): source = "" for fragment in result: source += fragment result = source if isinstance(result, page.utils.ResponseBody): req.setContentType(result.content_type) req.start() return [result.data] if isinstance(result, str) or isinstance(result, Document): req.setContentType("text/html") req.start() result = str(result) result += ("<!-- total request time: %.2f ms -->" % ((time.time() - request_start) * 1000)) if db.profiling: result += ("<!--\n\n%s\n\n -->" % formatDBProfiling(db)) return [result] result = WrappedResult(db, req, user, result) req.setContentType("text/html") req.start() # Prevent the finally clause below from closing the # connection. WrappedResult does it instead. db = None return result except gitutils.NoSuchRepository as error: raise page.utils.DisplayMessage( title="Invalid URI Parameter!", body=error.message) except gitutils.GitReferenceError as error: if error.ref: raise page.utils.DisplayMessage( title="Specified ref not found", body=("There is no ref named \"%s\" in %s." % (error.ref, error.repository))) elif error.sha1: raise page.utils.DisplayMessage( title="SHA-1 not found", body=error.message) else: raise except dbutils.NoSuchUser as error: raise page.utils.DisplayMessage( title="Invalid URI Parameter!", body=error.message) except dbutils.NoSuchReview as error: raise page.utils.DisplayMessage( title="Invalid URI Parameter!", body=error.message) if "/" in req.path: repository_name, _, rest = req.path.partition("/") repository = gitutils.Repository.fromName(db, repository_name) if repository: req.path = rest else: repository = None def revparsePlain(item): try: return gitutils.getTaggedCommit(repository, repository.revparse(item)) except: raise revparse = revparsePlain if repository is None: review_id = req.getParameter("review", None, filter=int) if review_id: cursor = db.cursor() cursor.execute("""SELECT repository FROM branches JOIN reviews ON (reviews.branch=branches.id) WHERE reviews.id=%s""", (review_id,)) row = cursor.fetchone() if row: repository = gitutils.Repository.fromId(db, row[0]) def revparseWithReview(item): if re.match("^[0-9a-f]+$", item): cursor.execute("""SELECT sha1 FROM commits JOIN changesets ON (changesets.child=commits.id) JOIN reviewchangesets ON (reviewchangesets.changeset=changesets.id) WHERE reviewchangesets.review=%s AND commits.sha1 LIKE %s""", (review_id, item + "%")) row = cursor.fetchone() if row: return row[0] else: return revparsePlain(item) revparse = revparseWithReview if repository is None: repository = user.getDefaultRepository(db) if gitutils.re_sha1.match(req.path): if repository and not repository.iscommit(req.path): repository = None if not repository: try: repository = gitutils.Repository.fromSHA1(db, req.path) except gitutils.GitReferenceError: repository = None if repository: try: items = filter(None, map(revparse, req.path.split(".."))) updated_query = {} if len(items) == 1: updated_query["repository"] = [repository.name] updated_query["sha1"] = [items[0]] elif len(items) == 2: updated_query["repository"] = [repository.name] updated_query["from"] = [items[0]] updated_query["to"] = [items[1]] if updated_query: req.updateQuery(updated_query) req.path = "showcommit" continue except gitutils.GitReferenceError: pass break raise page.utils.DisplayMessage( title="Not found!", body="Page not handled: /%s" % original_path, status=404) except GeneratorExit: raise except auth.AccessDenied as error: return handleDisplayMessage( db, req, request.DisplayMessage( title="Access denied", body=error.message, status=403)) except request.HTTPResponse as response: return response.execute(db, req) except request.MissingWSGIRemoteUser as error: return handleMissingWSGIRemoteUser(db, req) except page.utils.DisplayMessage as message: return handleDisplayMessage(db, req, message) except page.utils.DisplayFormattedText as formatted_text: return handleDisplayFormattedText(db, req, formatted_text) except Exception: # crash might be psycopg2.ProgrammingError so rollback to avoid # "InternalError: current transaction is aborted" inside handleException() if db and db.closed(): db = None elif db: db.rollback() error_title, error_body = handleException(db, req, user) error_body = reflow("\n\n".join(error_body)) error_message = "\n".join([error_title, "=" * len(error_title), "", error_body]) assert not req.isStarted() req.setStatus(500) req.setContentType("text/plain") req.start() return [error_message] finally: if db: db.close()
def main(): parser = argparse.ArgumentParser() parser.add_argument("-u", dest="user_id", type=int) parser.add_argument("-l", dest="auth_labels", action="append", default=[]) parser.add_argument("command", nargs="*") arguments = parser.parse_args() try: init(arguments.user_id, arguments.auth_labels) for command in arguments.command: pending_mails = None if command == "generate-mails-for-batch": data = json_decode(sys.stdin.readline()) batch_id = data["batch_id"] was_accepted = data["was_accepted"] is_accepted = data["is_accepted"] pending_mails = reviewing.utils.generateMailsForBatch(db, batch_id, was_accepted, is_accepted) elif command == "generate-mails-for-assignments-transaction": data = json_decode(sys.stdin.readline()) transaction_id = data["transaction_id"] pending_mails = reviewing.utils.generateMailsForAssignmentsTransaction(db, transaction_id) elif command == "apply-filters": data = json_decode(sys.stdin.readline()) filters = reviewing.filters.Filters() user = dbutils.User.fromId(db, data["user_id"]) if "user_id" in data else None if "review_id" in data: review = dbutils.Review.fromId(db, data["review_id"]) filters.setFiles(db, review=review) filters.load(db, review=review, user=user, added_review_filters=data.get("added_review_filters", []), removed_review_filters=data.get("removed_review_filters", [])) else: repository = gitutils.Repository.fromId(db, data["repository_id"]) filters.setFiles(db, file_ids=data["file_ids"]) filters.load(db, repository=repository, recursive=data.get("recursive", False), user=user) sys.stdout.write(json_encode(filters.data) + "\n") elif command == "generate-custom-mails": pending_mails = [] for data in json_decode(sys.stdin.readline()): from_user = dbutils.User.fromId(db, data["sender"]) if data.get("recipients"): recipients = [dbutils.User.fromId(db, user_id) for user_id in data["recipients"]] else: recipients = None subject = data["subject"] headers = data.get("headers") body = data["body"] if "review_id" in data: review = dbutils.Review.fromId(db, data["review_id"]) else: review = None pending_mails.extend(sendCustomMail( from_user, recipients, subject, headers, body, review)) elif command == "set-review-state": data = json_decode(sys.stdin.readline()) error = "" try: user = dbutils.User.fromId(db, data["user_id"]) review = dbutils.Review.fromId(db, data["review_id"]) if review.state != data["old_state"]: error = "invalid old state" elif data["new_state"] == "open": review.reopen(db, user) elif data["new_state"] == "closed": review.close(db, user) elif data["new_state"] == "dropped": review.drop(db, user) else: error = "invalid new state" except dbutils.NoSuchUser: error = "invalid user id" except dbutils.NoSuchReview: error = "invalid review id" except Exception as error: error = str(error) sys.stdout.write(error + "\n") elif command in HANDLERS: data_in = json_decode(sys.stdin.readline()) data_out = HANDLERS[command](data_in) sys.stdout.write(json_encode(data_out) + "\n") else: sys.stdout.write(json_encode("unknown command: %s" % command) + "\n") sys.exit(0) if pending_mails is not None: sys.stdout.write(json_encode(pending_mails) + "\n") finish() except Exception: sys.stdout.write(json_encode(traceback.format_exc()) + "\n") finally: abort()
def __init__(self, client, branch_id, timeout, log_offset): super(BranchTrackerHook.WaitForUpdate, self).__init__(client.server, [sys.executable, sys.argv[0], "--wait-for-update"], stderr=STDOUT) self.client = client self.client.write("wait\n") self.write(json_encode({ "branch_id": branch_id, "timeout": timeout, "log_offset": log_offset })) self.close()
if rows: for (hook_output,) in rows: output += hook_output status = "output" break cursor.execute("SELECT updating FROM trackedbranches WHERE id=%s", (branch_id,)) if not cursor.fetchone()[0]: # Update performed, but no log entries added. status = "no-output" break else: status = "timeout" sys.stdout.write(json_encode({ "status": status, "output": output or None })) sys.stdout.flush() db.close() else: import configuration from background.utils import PeerServer from textutils import json_decode from subprocess import STDOUT class BranchTrackerHook(PeerServer): class WaitForUpdate(PeerServer.ChildProcess): def __init__(self, client, branch_id, timeout, log_offset): super(BranchTrackerHook.WaitForUpdate, self).__init__(client.server, [sys.executable, sys.argv[0], "--wait-for-update"], stderr=STDOUT) self.client = client
def __str__(self): return json_encode({ "status": "failure", "code": self.__code, "title": self.__title, "message": self.__message })
for (hook_output, ) in rows: output += hook_output status = "output" break cursor.execute("SELECT updating FROM trackedbranches WHERE id=%s", (branch_id, )) if not cursor.fetchone()[0]: # Update performed, but no log entries added. status = "no-output" break else: status = "timeout" sys.stdout.write(json_encode({"status": status, "output": output or None})) sys.stdout.flush() db.close() else: import configuration from background.utils import PeerServer from textutils import json_decode from subprocess import STDOUT class BranchTrackerHook(PeerServer): class WaitForUpdate(PeerServer.ChildProcess): def __init__(self, client, branch_id, timeout, log_offset): super(BranchTrackerHook.WaitForUpdate, self).__init__( client.server,
def process_request(environ, start_response): request_start = time.time() critic = api.critic.startSession() db = critic.database user = None try: try: req = request.Request(db, environ, start_response) req.setUser(db) if req.user is None: if configuration.base.AUTHENTICATION_MODE == "host": user = dbutils.User.makeAnonymous() elif configuration.base.SESSION_TYPE == "httpauth": req.requestHTTPAuthentication() return [] elif req.path.startswith("externalauth/"): provider_name = req.path[len("externalauth/"):] raise request.DoExternalAuthentication(provider_name) elif req.path.startswith("oauth/"): provider_name = req.path[len("oauth/"):] if provider_name in auth.PROVIDERS: provider = auth.PROVIDERS[provider_name] if isinstance(provider, auth.OAuthProvider): if finishOAuth(db, req, provider): return [] elif configuration.base.SESSION_TYPE == "cookie": if req.cookies.get("has_sid") == "1": req.ensureSecure() if configuration.base.ALLOW_ANONYMOUS_USER \ or req.path in request.INSECURE_PATHS \ or req.path.startswith("static-resource/"): user = dbutils.User.makeAnonymous() # Don't try to redirect POST requests to the login page. elif req.method == "GET": if configuration.base.AUTHENTICATION_MODE == "critic": raise request.NeedLogin(req) else: raise request.DoExternalAuthentication( configuration.base.AUTHENTICATION_MODE, req.getTargetURL()) if not user: req.setStatus(403) req.start() return [] else: try: user = dbutils.User.fromName(db, req.user) except dbutils.NoSuchUser: if configuration.base.AUTHENTICATION_MODE == "host": email = getUserEmailAddress(req.user) user = dbutils.User.create(db, req.user, req.user, email, email_verified=None) db.commit() else: # This can't really happen. raise if not user.isAnonymous(): critic.setActualUser(api.user.fetch(critic, user_id=user.id)) user.loadPreferences(db) if user.status == 'retired': cursor = db.cursor() cursor.execute("UPDATE users SET status='current' WHERE id=%s", (user.id, )) user = dbutils.User.fromId(db, user.id) db.commit() if not user.getPreference(db, "debug.profiling.databaseQueries"): db.disableProfiling() if not req.path: if user.isAnonymous(): location = "tutorial" else: location = user.getPreference(db, "defaultPage") if req.query: location += "?" + req.query req.setStatus(307) req.addResponseHeader("Location", location) req.start() return [] if req.path == "redirect": target = req.getParameter("target", "/") if req.method == "POST": # Don't use HTTP redirect for POST requests. req.setContentType("text/html") req.start() return [ "<meta http-equiv='refresh' content='0; %s'>" % htmlify(target) ] else: raise request.MovedTemporarily(target) # Require a .git suffix on HTTP(S) repository URLs unless the user- # agent starts with "git/" (as Git's normally does.) # # Our objectives are: # # 1) Not to require Git's user-agent to be its default value, since # the user might have to override it to get through firewalls. # 2) Never to send regular user requests to 'git http-backend' by # mistake. # # This is a compromise. if req.getRequestHeader("User-Agent", "").startswith("git/"): suffix = None else: suffix = ".git" if handleRepositoryPath(db, req, user, suffix): db = None return [] if req.path.startswith("!/"): req.path = req.path[2:] elif configuration.extensions.ENABLED: handled = extensions.role.page.execute(db, req, user) if isinstance(handled, basestring): req.start() return [handled] if req.path.startswith("static-resource/"): return handleStaticResource(req) if req.path.startswith("r/"): req.updateQuery({"id": [req.path[2:]]}) req.path = "showreview" if configuration.extensions.ENABLED: match = RE_EXTENSION_RESOURCE.match(req.path) if match: content_type, resource = extensions.resource.get( req, db, user, match.group(1)) if resource: req.setContentType(content_type) if content_type.startswith("image/"): req.addResponseHeader("Cache-Control", "max-age=3600") req.start() return [resource] else: req.setStatus(404) req.start() return [] if req.path.startswith("download/"): operationfn = download elif req.path == "api" or req.path.startswith("api/"): try: result = jsonapi.handle(critic, req) except jsonapi.Error as error: req.setStatus(error.http_status) result = { "error": { "title": error.title, "message": error.message } } else: req.setStatus(200) accept_header = req.getRequestHeader("Accept") if accept_header == "application/vnd.api+json": default_indent = None else: default_indent = 2 indent = req.getParameter("indent", default_indent, filter=int) if indent == 0: # json.encode(..., indent=0) still gives line-breaks, just # no indentation. This is not so useful, so set indent to # None instead, which disables formatting entirely. indent = None req.setContentType("application/vnd.api+json") req.start() return [json_encode(result, indent=indent)] else: operationfn = OPERATIONS.get(req.path) if operationfn: result = operationfn(req, db, user) if isinstance(result, (OperationResult, OperationError)): req.setContentType("text/json") if isinstance(result, OperationResult): if db.profiling: result.set("__profiling__", formatDBProfiling(db)) result.set("__time__", time.time() - request_start) elif not req.hasContentType(): req.setContentType("text/plain") req.start() if isinstance(result, unicode): return [result.encode("utf8")] else: return [str(result)] override_user = req.getParameter("user", None) while True: pagefn = PAGES.get(req.path) if pagefn: try: if not user.isAnonymous() and override_user: user = dbutils.User.fromName(db, override_user) req.setContentType("text/html") result = pagefn(req, db, user) if db.profiling and not (isinstance(result, str) or isinstance(result, Document)): source = "" for fragment in result: source += fragment result = source if isinstance(result, str) or isinstance( result, Document): req.start() result = str(result) result += ("<!-- total request time: %.2f ms -->" % ((time.time() - request_start) * 1000)) if db.profiling: result += ("<!--\n\n%s\n\n -->" % formatDBProfiling(db)) return [result] else: result = WrappedResult(db, req, user, result) req.start() # Prevent the finally clause below from closing the # connection. WrappedResult does it instead. db = None return result except gitutils.NoSuchRepository as error: raise page.utils.DisplayMessage( title="Invalid URI Parameter!", body=error.message) except gitutils.GitReferenceError as error: if error.ref: raise page.utils.DisplayMessage( title="Specified ref not found", body=("There is no ref named \"%s\" in %s." % (error.ref, error.repository))) elif error.sha1: raise page.utils.DisplayMessage( title="SHA-1 not found", body=error.message) else: raise except dbutils.NoSuchUser as error: raise page.utils.DisplayMessage( title="Invalid URI Parameter!", body=error.message) except dbutils.NoSuchReview as error: raise page.utils.DisplayMessage( title="Invalid URI Parameter!", body=error.message) path = req.path if "/" in path: repository = gitutils.Repository.fromName( db, path.split("/", 1)[0]) if repository: path = path.split("/", 1)[1] else: repository = None def revparsePlain(item): try: return gitutils.getTaggedCommit( repository, repository.revparse(item)) except: raise revparse = revparsePlain if repository is None: review_id = req.getParameter("review", None, filter=int) if review_id: cursor = db.cursor() cursor.execute( """SELECT repository FROM branches JOIN reviews ON (reviews.branch=branches.id) WHERE reviews.id=%s""", (review_id, )) row = cursor.fetchone() if row: repository = gitutils.Repository.fromId(db, row[0]) def revparseWithReview(item): if re.match("^[0-9a-f]+$", item): cursor.execute( """SELECT sha1 FROM commits JOIN changesets ON (changesets.child=commits.id) JOIN reviewchangesets ON (reviewchangesets.changeset=changesets.id) WHERE reviewchangesets.review=%s AND commits.sha1 LIKE %s""", (review_id, item + "%")) row = cursor.fetchone() if row: return row[0] else: return revparsePlain(item) revparse = revparseWithReview if repository is None: repository = gitutils.Repository.fromName( db, user.getPreference(db, "defaultRepository")) if gitutils.re_sha1.match(path): if repository and not repository.iscommit(path): repository = None if not repository: try: repository = gitutils.Repository.fromSHA1( db, path) except gitutils.GitReferenceError: repository = None if repository: try: items = filter(None, map(revparse, path.split(".."))) updated_query = {} if len(items) == 1: updated_query["repository"] = [repository.name] updated_query["sha1"] = [items[0]] elif len(items) == 2: updated_query["repository"] = [repository.name] updated_query["from"] = [items[0]] updated_query["to"] = [items[1]] if updated_query: req.updateQuery(updated_query) req.path = "showcommit" continue except gitutils.GitReferenceError: pass break raise page.utils.DisplayMessage(title="Not found!", body="Page not handled: /%s" % path, status=404) except GeneratorExit: raise except page.utils.NotModified: req.setStatus(304) req.start() return [] except request.MovedTemporarily as redirect: req.setStatus(307) req.addResponseHeader("Location", redirect.location) if redirect.no_cache: req.addResponseHeader("Cache-Control", "no-cache") req.start() return [] except request.DoExternalAuthentication as command: command.execute(db, req) return [] except request.MissingWSGIRemoteUser as error: # req object is not initialized yet. start_response("200 OK", [("Content-Type", "text/html")]) return [ """\ <pre>error: Critic was configured with '--auth-mode host' but there was no REMOTE_USER variable in the WSGI environ dict provided by the web server. To fix this you can either reinstall Critic using '--auth-mode critic' (to let Critic handle user authentication automatically), or you can configure user authentication properly in the web server. For apache2, the latter can be done by adding the something like the following to the apache site configuration for Critic: <Location /> AuthType Basic AuthName "Authentication Required" AuthUserFile "/path/to/critic-main.htpasswd.users" Require valid-user </Location> If you need more dynamic http authentication you can instead setup mod_wsgi with a custom WSGIAuthUserScript directive. This will cause the provided credentials to be passed to a Python function called check_password() that you can implement yourself. This way you can validate the user/pass via any existing database or for example an LDAP server. For more information on setting up such authentication in apache2, see: <a href="%(url)s">%(url)s</a></pre>""" % { "url": "http://code.google.com/p/modwsgi/wiki/AccessControlMechanisms#Apache_Authentication_Provider" } ] except page.utils.DisplayMessage as message: if user is None: user = dbutils.User.makeAnonymous() document = page.utils.displayMessage(db, req, user, title=message.title, message=message.body, review=message.review, is_html=message.html) req.setContentType("text/html") req.setStatus(message.status) req.start() return [str(document)] except page.utils.DisplayFormattedText as formatted_text: if user is None: user = dbutils.User.makeAnonymous() document = page.utils.displayFormattedText(db, req, user, formatted_text.source) req.setContentType("text/html") req.start() return [str(document)] except Exception: # crash might be psycopg2.ProgrammingError so rollback to avoid # "InternalError: current transaction is aborted" inside handleException() if db and db.closed(): db = None elif db: db.rollback() error_title, error_body = handleException(db, req, user) error_body = reflow("\n\n".join(error_body)) error_message = "\n".join( [error_title, "=" * len(error_title), "", error_body]) assert not req.isStarted() req.setStatus(500) req.setContentType("text/plain") req.start() return [error_message] finally: if db: db.close()
pending_mails = reviewing.utils.generateMailsForAssignmentsTransaction(db, transaction_id) elif command == "apply-filters": data = json_decode(sys.stdin.readline()) filters = reviewing.filters.Filters() user = dbutils.User.fromId(db, data["user_id"]) if "user_id" in data else None if "review_id" in data: review = dbutils.Review.fromId(db, data["review_id"], load_commits=False) filters.setFiles(db, review=review) filters.load(db, review=review, user=user, added_review_filters=data.get("added_review_filters", []), removed_review_filters=data.get("removed_review_filters", [])) else: repository = gitutils.Repository.fromId(db, data["repository_id"]) filters.setFiles(db, file_ids=data["file_ids"]) filters.load(db, repository=repository, recursive=data.get("recursive", False), user=user) sys.stdout.write(json_encode(filters.data) + "\n") elif command == "generate-custom-mails": pending_mails = [] for data in json_decode(sys.stdin.readline()): from_user = dbutils.User.fromId(db, data["sender"]) if data.get("recipients"): recipients = [dbutils.User.fromId(db, user_id) for user_id in data["recipients"]] else: recipients = None subject = data["subject"] headers = data.get("headers") body = data["body"] if "review_id" in data: review = dbutils.Review.fromId(db, data["review_id"]) else:
def execute_command(self, client, command): client.write(json_encode({ "status": "error", "error": "command not supported" })) client.close()
def add_result(self, result): self.__results.append(result) if len(self.__results) == len(self.__requests): self.write(json_encode(self.__results)) self.close()
def __init__(self, server, client, request): super(JSONJobServer.Job, self).__init__(server, [sys.executable, sys.argv[0], "--json-job"], stderr=subprocess.STDOUT) self.clients = [client] self.request = request self.write(json_encode(request)) self.close()
def executeProcess(db, manifest, role_name, script, function, extension_id, user_id, argv, timeout, stdin=None, rlimit_rss=256): # If |user_id| is not the same as |db.user|, then one user's access of the # system is triggering an extension on behalf of another user. This will # for instance happen when one user is adding changes to a review, # triggering an extension filter hook set up by another user. # # In this case, we need to check that the other user can access the # extension. # # If |user_id| is the same as |db.user|, we need to use |db.profiles|, which # may contain a profile associated with an access token that was used to # authenticate the user. if user_id != db.user.id: user = dbutils.User.fromId(db, user_id) authentication_labels = auth.DATABASE.getAuthenticationLabels(user) profiles = [ auth.AccessControlProfile.forUser(db, user, authentication_labels) ] else: authentication_labels = db.authentication_labels profiles = db.profiles extension = Extension.fromId(db, extension_id) if not auth.AccessControlProfile.isAllowedExtension( profiles, "execute", extension): raise auth.AccessDenied("Access denied to extension: execute %s" % extension.getKey()) flavor = manifest.flavor if manifest.flavor not in configuration.extensions.FLAVORS: flavor = configuration.extensions.DEFAULT_FLAVOR stdin_data = "%s\n" % json_encode({ "library_path": configuration.extensions.FLAVORS[flavor]["library"], "rlimit": { "rss": rlimit_rss }, "hostname": configuration.base.HOSTNAME, "dbname": configuration.database.PARAMETERS["database"], "dbuser": configuration.database.PARAMETERS["user"], "git": configuration.executables.GIT, "python": configuration.executables.PYTHON, "python_path": "%s:%s" % (configuration.paths.CONFIG_DIR, configuration.paths.INSTALL_DIR), "repository_work_copy_path": configuration.extensions.WORKCOPY_DIR, "changeset_address": configuration.services.CHANGESET["address"], "branchtracker_pid_path": configuration.services.BRANCHTRACKER["pidfile_path"], "maildelivery_pid_path": configuration.services.MAILDELIVERY["pidfile_path"], "is_development": configuration.debug.IS_DEVELOPMENT, "extension_path": manifest.path, "extension_id": extension_id, "user_id": user_id, "authentication_labels": list(authentication_labels), "role": role_name, "script_path": script, "fn": function, "argv": argv }) if stdin is not None: stdin_data += stdin # Double the timeout. Timeouts are primarily handled by the extension runner # service, which returns an error response on timeout. This deadline here is # thus mostly to catch the extension runner service itself timing out. deadline = time.time() + timeout * 2 try: connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) connection.settimeout(max(0, deadline - time.time())) connection.connect(configuration.services.EXTENSIONRUNNER["address"]) connection.sendall( json_encode({ "stdin": stdin_data, "flavor": flavor, "timeout": timeout })) connection.shutdown(socket.SHUT_WR) data = "" while True: connection.settimeout(max(0, deadline - time.time())) try: received = connection.recv(4096) except socket.error as error: if error.errno == errno.EINTR: continue raise if not received: break data += received connection.close() except socket.timeout as error: raise ProcessTimeout(timeout) except socket.error as error: raise ProcessError("failed to read response: %s" % error) try: data = json_decode(data) except ValueError as error: raise ProcessError("failed to decode response: %s" % error) if data["status"] == "timeout": raise ProcessTimeout(timeout) if data["status"] == "error": raise ProcessError(data["error"]) if data["returncode"] != 0: raise ProcessFailure(data["returncode"], data["stderr"]) return data["stdout"]
def renderHome(req, db, user): if user.isAnonymous(): raise page.utils.NeedLogin(req) profiler = profiling.Profiler() cursor = db.cursor() readonly = req.getParameter( "readonly", "yes" if user.name != req.user else "no") == "yes" repository = req.getParameter("repository", None, gitutils.Repository.FromParameter(db)) verified_email_id = req.getParameter("email_verified", None, int) if not repository: repository = user.getDefaultRepository(db) title_fullname = user.fullname if title_fullname[-1] == 's': title_fullname += "'" else: title_fullname += "'s" cursor.execute( "SELECT email FROM usergitemails WHERE uid=%s ORDER BY email ASC", (user.id, )) gitemails = ", ".join([email for (email, ) in cursor]) document = htmlutils.Document(req) html = document.html() head = html.head() body = html.body() if user.name == req.user: actual_user = None else: actual_user = req.getUser(db) def renderHeaderItems(target): if readonly and actual_user and actual_user.hasRole( db, "administrator"): target.a("button", href="/home?user=%s&readonly=no" % user.name).text("Edit") page.utils.generateHeader(body, db, user, generate_right=renderHeaderItems, current_page="home") document.addExternalStylesheet("resource/home.css") document.addExternalScript("resource/home.js") document.addExternalScript("resource/autocomplete.js") if repository: document.addInternalScript(repository.getJS()) else: document.addInternalScript("var repository = null;") if actual_user and actual_user.hasRole(db, "administrator"): document.addInternalScript("var administrator = true;") else: document.addInternalScript("var administrator = false;") document.addInternalScript(user.getJS()) document.addInternalScript("user.gitEmails = %s;" % jsify(gitemails)) document.addInternalScript( "var verifyEmailAddresses = %s;" % jsify(configuration.base.VERIFY_EMAIL_ADDRESSES)) document.setTitle("%s Home" % title_fullname) target = body.div("main") basic = target.table('paleyellow basic', align='center') basic.tr().td('h1', colspan=3).h1().text("%s Home" % title_fullname) def row(heading, value, help=None, extra_class=None): if extra_class: row_class = "line " + extra_class else: row_class = "line" main_row = basic.tr(row_class) main_row.td('heading').text("%s:" % heading) value_cell = main_row.td('value', colspan=2) if callable(value): value(value_cell) else: value_cell.text(value) basic.tr('help').td('help', colspan=3).text(help) def renderFullname(target): if readonly: target.text(user.fullname) else: target.input("value", id="user_fullname", value=user.fullname) target.span("status", id="status_fullname") buttons = target.span("buttons") buttons.button(onclick="saveFullname();").text("Save") buttons.button(onclick="resetFullname();").text("Reset") def renderEmail(target): if not actual_user or actual_user.hasRole(db, "administrator"): cursor.execute( """SELECT id, email, verified FROM useremails WHERE uid=%s ORDER BY id ASC""", (user.id, )) rows = cursor.fetchall() if rows: if len(rows) > 1: target.addClass("multiple") addresses = target.div("addresses") for email_id, email, verified in rows: checked = "checked" if email == user.email else None selected = " selected" if email == user.email else "" label = addresses.label("address inset flex" + selected, data_email_id=email_id) if len(rows) > 1: label.input(name="email", type="radio", value=email, checked=checked) label.span("value").text(email) actions = label.span("actions") if verified is False: actions.a("action unverified", href="#").text("[unverified]") elif verified is True: now = " now" if email_id == verified_email_id else "" actions.span("action verified" + now).text("[verified]") actions.a("action delete", href="#").text("[delete]") else: target.i().text("No email address") target.span("buttons").button("addemail").text("Add email address") elif user.email is None: target.i().text("No email address") elif user.email_verified is False: # Pending verification: don't show to other users. target.i().text("Email address not verified") else: target.span("inset").text(user.email) def renderGitEmails(target): if readonly: target.text(gitemails) else: target.input("value", id="user_gitemails", value=gitemails) target.span("status", id="status_gitemails") buttons = target.span("buttons") buttons.button(onclick="saveGitEmails();").text("Save") buttons.button(onclick="resetGitEmails();").text("Reset") def renderPassword(target): cursor.execute("SELECT password IS NOT NULL FROM users WHERE id=%s", (user.id, )) has_password = cursor.fetchone()[0] if not has_password: target.text("not set") else: target.text("****") if not readonly: if not has_password or (actual_user and actual_user.hasRole( db, "administrator")): target.span("buttons").button( onclick="setPassword();").text("Set password") else: target.span("buttons").button( onclick="changePassword();").text("Change password") row("User ID", str(user.id)) row("User Name", user.name) row("Display Name", renderFullname, "This is the name used when displaying commits or comments.") row("Primary Email", renderEmail, "This is the primary email address, to which emails are sent.", extra_class="email") row("Git Emails", renderGitEmails, "These email addresses are used to map Git commits to the user.") if configuration.base.AUTHENTICATION_MODE == "critic": row("Password", renderPassword, extra_class="password") cursor.execute( """SELECT provider, account FROM externalusers WHERE uid=%s""", (user.id, )) external_accounts = [(auth.PROVIDERS[provider_name], account) for provider_name, account in cursor if provider_name in auth.PROVIDERS] if external_accounts: basic.tr().td('h2', colspan=3).h2().text("External Accounts") for provider, account in external_accounts: def renderExternalAccount(target): url = provider.getAccountURL(account) target.a("external", href=url).text(account) row(provider.getTitle(), renderExternalAccount) profiler.check("user information") filters = page.utils.PaleYellowTable(body, "Filters") filters.titleRight.a("button", href="/tutorial?item=filters").text("Tutorial") cursor.execute( """SELECT repositories.id, repositories.name, repositories.path, filters.id, filters.type, filters.path, NULL, filters.delegate FROM repositories JOIN filters ON (filters.repository=repositories.id) WHERE filters.uid=%s""", (user.id, )) rows = cursor.fetchall() if configuration.extensions.ENABLED: cursor.execute( """SELECT repositories.id, repositories.name, repositories.path, filters.id, 'extensionhook', filters.path, filters.name, filters.data FROM repositories JOIN extensionhookfilters AS filters ON (filters.repository=repositories.id) WHERE filters.uid=%s""", (user.id, )) rows.extend(cursor.fetchall()) FILTER_TYPES = ["reviewer", "watcher", "ignored", "extensionhook"] def rowSortKey(row): (repository_id, repository_name, repository_path, filter_id, filter_type, filter_path, filter_name, filter_data) = row # Rows are grouped by repository first and type second, so sort by # repository name and filter type primarily. # # Secondarily sort by filter name (only for extension hook filters; is # None for regular filters) and filter path. This sorting is mostly to # achieve a stable order; it has no greater meaning. return (repository_name, FILTER_TYPES.index(filter_type), filter_name, filter_path) rows.sort(key=rowSortKey) if rows: repository = None repository_filters = None tbody_reviewer = None tbody_watcher = None tbody_ignored = None tbody_extensionhook = None count_matched_files = {} for (repository_id, repository_name, repository_path, filter_id, filter_type, filter_path, filter_name, filter_data) in rows: if not repository or repository.id != repository_id: repository = gitutils.Repository.fromId(db, repository_id) repository_url = repository.getURL(db, user) filters.addSection(repository_name, repository_url) repository_filters = filters.addCentered().table( "filters callout") tbody_reviewer = tbody_watcher = tbody_ignored = tbody_extensionhook = None if filter_type == "reviewer": if not tbody_reviewer: tbody_reviewer = repository_filters.tbody() tbody_reviewer.tr().th(colspan=5).text("Reviewer") tbody = tbody_reviewer elif filter_type == "watcher": if not tbody_watcher: tbody_watcher = repository_filters.tbody() tbody_watcher.tr().th(colspan=5).text("Watcher") tbody = tbody_watcher elif filter_type == "ignored": if not tbody_ignored: tbody_ignored = repository_filters.tbody() tbody_ignored.tr().th(colspan=5).text("Ignored") tbody = tbody_ignored else: if not tbody_extensionhook: tbody_extensionhook = repository_filters.tbody() tbody_extensionhook.tr().th( colspan=5).text("Extension hooks") tbody = tbody_extensionhook row = tbody.tr() row.td("path").text(filter_path) if filter_type != "extensionhook": delegates = row.td("delegates", colspan=2) if filter_data: delegates.i().text("Delegates: ") delegates.span("names").text(", ".join( filter_data.split(","))) else: role = extensions.role.filterhook.getFilterHookRole( db, filter_id) if role: title = row.td("title") title.text(role.title) data = row.td("data") data.text(filter_data) else: row.td(colspan=2).i().text("Invalid filter") if filter_path == "/": row.td("files").text("all files") else: href = "javascript:void(showMatchedFiles(%s, %s));" % (jsify( repository.name), jsify(filter_path)) row.td("files").a(href=href, id=("f%d" % filter_id)).text("? files") count_matched_files.setdefault(repository_id, []).append(filter_id) links = row.td("links") arguments = (jsify(repository.name), filter_id, jsify(filter_type), jsify(filter_path), jsify(filter_data)) links.a(href="javascript:void(editFilter(%s, %d, %s, %s, %s));" % arguments).text("[edit]") if filter_type != "extensionhook": links.a( href= "javascript:if (deleteFilterById(%d)) location.reload(); void(0);" % filter_id).text("[delete]") links.a(href="javascript:location.href='/config?filter=%d';" % filter_id).text("[preferences]") else: links.a( href= "javascript:if (deleteExtensionHookFilterById(%d)) location.reload(); void(0);" % filter_id).text("[delete]") document.addInternalScript("var count_matched_files = %s;" % json_encode(count_matched_files.values())) else: filters.addCentered().p().b().text("No filters") # Additionally check if there are in fact no repositories. cursor.execute("SELECT 1 FROM repositories") if not cursor.fetchone(): document.addInternalScript("var no_repositories = true;") if not readonly: filters.addSeparator() filters.addCentered().button( onclick="editFilter();").text("Add filter") profiler.check("filters") hidden = body.div("hidden", style="display: none") if configuration.extensions.ENABLED: filterhooks = extensions.role.filterhook.listFilterHooks(db, user) else: filterhooks = [] with hidden.div("filterdialog") as dialog: paragraph = dialog.p() paragraph.b().text("Repository:") paragraph.br() page.utils.generateRepositorySelect(db, user, paragraph, name="repository") paragraph = dialog.p() paragraph.b().text("Filter type:") paragraph.br() filter_type = paragraph.select(name="type") filter_type.option(value="reviewer").text("Reviewer") filter_type.option(value="watcher").text("Watcher") filter_type.option(value="ignored").text("Ignored") for extension, manifest, roles in filterhooks: optgroup = filter_type.optgroup(label=extension.getTitle(db)) for role in roles: option = optgroup.option( value="extensionhook", data_extension_id=extension.getExtensionID(db), data_filterhook_name=role.name) option.text(role.title) paragraph = dialog.p() paragraph.b().text("Path:") paragraph.br() paragraph.input(name="path", type="text") paragraph.span("matchedfiles") regular_div = dialog.div("regular") paragraph = regular_div.p() paragraph.b().text("Delegates:") paragraph.br() paragraph.input(name="delegates", type="text") paragraph = regular_div.p() label = paragraph.label() label.input(name="apply", type="checkbox", checked="checked") label.b().text("Apply to existing reviews") for extension, manifest, roles in filterhooks: for role in roles: if not role.data_description: continue filterhook_id = "%d_%s" % (extension.getExtensionID(db), role.name) extensionhook_div = dialog.div("extensionhook " + filterhook_id, style="display: none") extensionhook_div.innerHTML(role.data_description) paragraph = extensionhook_div.p() paragraph.b().text("Data:") paragraph.br() paragraph.input(type="text") profiler.output(db, user, document) return document
pending_mails = reviewing.utils.generateMailsForBatch(db, batch_id, was_accepted, is_accepted) elif command == "generate-mails-for-assignments-transaction": data = json_decode(sys.stdin.readline()) transaction_id = data["transaction_id"] pending_mails = reviewing.utils.generateMailsForAssignmentsTransaction(db, transaction_id) elif command == "apply-filters": data = json_decode(sys.stdin.readline()) filters = reviewing.filters.Filters() user = dbutils.User.fromId(db, data["user_id"]) if "user_id" in data else None if "review_id" in data: review = dbutils.Review.fromId(db, data["review_id"], load_commits=False) filters.setFiles(db, review=review) filters.load(db, review=review, user=user, added_review_filters=data.get("added_review_filters", []), removed_review_filters=data.get("removed_review_filters", [])) else: repository = gitutils.Repository.fromId(db, data["repository_id"]) filters.setFiles(db, file_ids=data["file_ids"]) filters.load(db, repository=repository, recursive=data.get("recursive", False), user=user) sys.stdout.write(json_encode(filters.data) + "\n") else: print "unknown command: %s" % command sys.exit(1) if pending_mails is not None: sys.stdout.write(json_encode(pending_mails) + "\n") finish() finally: abort()
def renderSearch(req, db, user): document = htmlutils.Document(req) document.setTitle("Review Search") html = document.html() head = html.head() body = html.body() page.utils.generateHeader(body, db, user, current_page="search") document.addExternalStylesheet("resource/search.css") document.addExternalScript("resource/search.js") document.addExternalScript("resource/autocomplete.js") document.addInternalScript(user.getJS()) cursor = db.cursor() cursor.execute("SELECT name, fullname FROM users") users = dict(cursor) document.addInternalScript("var users = %s;" % textutils.json_encode(users)) def renderQuickSearch(target): wrap = target.div("quicksearch callout") wrap.p().text("""A Quick Search dialog can be opened from any page using the "F" keyboard shortcut.""") wrap.p().a(href="/tutorial?item=search").text("More information") def renderInput(target, label, name, placeholder=""): fieldset = target.fieldset("search-" + name) fieldset.label("input-label").text(label) fieldset.input(type="text", name=name, placeholder=placeholder) def renderInputWithOptions(target, label, name, options, placeholder=""): fieldset = target.fieldset("search-" + name) fieldset.label("input-label").text(label) checkGroup = fieldset.div("input-options checkbox-group") for option in options: opt_label = checkGroup.label() opt_label.input(type="checkbox", name=option["name"], checked="checked" if "checked" in option else None) opt_label.text(option["label"]) fieldset.input(type="text", name=name, placeholder=placeholder) def renderFreetext(target): options = [{ "name": "freetextSummary", "label": "Summary", "checked": True }, { "name": "freetextDescription", "label": "Description", "checked": True }] renderInputWithOptions(target, label="Search term", name="freetext", placeholder="free text search", options=options) def renderState(target): state = target.fieldset("search-state") state.label("input-label").text("State") select = state.select(name="state") select.option(value="", selected="selected").text("Any state") select.option(value="open").text("Open") select.option(value="pending").text("Pending") select.option(value="accepted").text("Accepted") select.option(value="closed").text("Finished") select.option(value="dropped").text("Dropped") def renderUser(target): options = [{ "name": "userOwner", "label": "Owner", "checked": True }, { "name": "userReviewer", "label": "Reviewer" }] renderInputWithOptions(target, label="User", name="user", placeholder="user name(s)", options=options) def renderRepository(target): fieldset = target.fieldset("search-repository") fieldset.label("input-label").text("Repository") page.utils.generateRepositorySelect(db, user, fieldset, name="repository", selected=False, placeholder_text="Any repository", allow_selecting_none=True) section = body.section("paleyellow section") section.h1("section-heading").text("Review Search") url_terms = [] for name, value in urlparse.parse_qsl(req.query): if name == "q": url_terms.append(value) elif name.startswith("q"): url_terms.append("%s:%s" % (name[1:], value)) wrap = section.div("flex") search = wrap.form("search", name="search") if url_terms: row = search.div("flex") query = row.fieldset("search-query") query.label("input-label").text("Search query") query.input(type="text", name="query", value=" ".join(url_terms)) result = section.div("search-result", style="display: none") result.h2().text("Search result") result.div("callout") else: row = search.div("flex") renderFreetext(row) renderState(row) renderUser(search) row = search.div("flex") renderRepository(row) renderInput(row, "Branch", "branch") renderInput(search, "Path", "path") buttons = search.div("search-buttons") if url_terms: buttons.button(type="submit").text("Search again") buttons.a("button", href="/search").text("Show full search form") else: buttons.button(type="submit").text("Search") renderQuickSearch(wrap) return document