def run(self): with dbutils.Database() as db: # Do an initial load/update of timezones. # # The 'timezones' table initially (post-installation) only contains # the Universal/UTC timezone; this call adds all the others that the # PostgreSQL database server knows about. dbutils.loadTimezones(db) super(Maintenance, self).run()
def __compact(self): import syntaxhighlight now = time.time() max_age_uncompressed = 7 * 24 * 60 * 60 max_age_compressed = 90 * 24 * 60 * 60 uncompressed_count = 0 compressed_count = 0 purged_paths = [] db = dbutils.Database() cursor = db.cursor() cursor.execute("CREATE TEMPORARY TABLE purged (sha1 CHAR(40) PRIMARY KEY)") cache_path = configuration.services.HIGHLIGHT["cache_dir"] for section in sorted(os.listdir(cache_path)): if len(section) == 2: for filename in os.listdir("%s/%s" % (cache_path, section)): fullname = "%s/%s/%s" % (cache_path, section, filename) age = now - os.stat(fullname).st_mtime if len(filename) > 38 and filename[38] == "." and filename[39:] in syntaxhighlight.LANGUAGES: if age > max_age_uncompressed: self.debug("compressing: %s/%s" % (section, filename)) worker = process(["/bin/bzip2", fullname]) worker.wait() compressed_count += 1 else: uncompressed_count += 1 elif len(filename) > 42 and filename[38] == "." and filename[-4] == "." and filename[39:-4] in syntaxhighlight.LANGUAGES: if filename.endswith(".bz2"): if age > max_age_compressed: self.debug("purging: %s/%s" % (section, filename)) cursor.execute("INSERT INTO purged (sha1) VALUES (%s)", (section + filename[:-4],)) purged_paths.append(fullname) else: compressed_count += 1 elif filename.endswith(".ctx"): self.debug("deleting context file: %s/%s" % (section, filename)) os.unlink(fullname) self.debug("uncompressed=%d / compressed=%d / purged=%d" % (uncompressed_count, compressed_count, len(purged_paths))) if purged_paths: for path in purged_paths: os.unlink(path) cursor.execute("DELETE FROM codecontexts USING purged WHERE codecontexts.sha1=purged.sha1") db.commit() db.close()
def __init__(self): service = configuration.services.HIGHLIGHT super(HighlightServer, self).__init__(service) self.db = dbutils.Database() if "compact_at" in service: hour, minute = service["compact_at"] self.register_maintenance(hour=hour, minute=minute, callback=self.__compact)
def emit(self, record): import mailutils try: import dbutils db = dbutils.Database() except: db = None mailutils.sendAdministratorErrorReport(db, self.__logfile_name, record.message.splitlines()[0], self.formatter.format(record)) if db: db.close()
def keepalives(): # Run Repository.packKeepaliveRefs() and make sure it seems to do its job # correctly. Since it's run as a nightly maintenance task, it would # otherwise not be exercised by testing. import gitutils import dbutils db = dbutils.Database() cursor = db.cursor() cursor.execute("SELECT id FROM repositories") for (repository_id, ) in cursor: repository = gitutils.Repository.fromId(db, repository_id) # Make sure there's at least one loose keepalive ref. repository.keepalive(repository.revparse("HEAD")) loose_keepalive_refs_before = repository.run( "for-each-ref", "--format=%(objectname)", gitutils.KEEPALIVE_REF_PREFIX).splitlines() assert len(loose_keepalive_refs_before) > 0 repository.packKeepaliveRefs() loose_keepalive_refs_after = repository.run( "for-each-ref", "--format=%(objectname)", gitutils.KEEPALIVE_REF_PREFIX).splitlines() assert len(loose_keepalive_refs_after) == 0 chain_before = repository.revparse(gitutils.KEEPALIVE_REF_CHAIN) # Check that all previous loose keepalive refs are now ancestors of the # keepalive chain ref (IOW, are being kept alive by it.) for sha1 in loose_keepalive_refs_before: mergebase = repository.mergebase([sha1, chain_before]) assert mergebase == sha1 # Make sure there's a loose keepalive ref again. repository.keepalive(repository.revparse("HEAD")) repository.packKeepaliveRefs() chain_after = repository.revparse(gitutils.KEEPALIVE_REF_CHAIN) # Make sure the chain didn't change. assert chain_before == chain_after, ("%s != %s" % (chain_before, chain_after)) print "keepalives: ok"
def run(self): if not configuration.extensions.ENABLED: self.info("service stopping: extension support not enabled") return failed_events = set() while not self.terminated: self.interrupted = False with dbutils.Database() as db: cursor = db.cursor() cursor.execute("""SELECT id FROM extensionfilterhookevents ORDER BY id ASC""") finished_events = [] for (event_id, ) in cursor: if event_id not in failed_events: try: extensions.role.filterhook.processFilterHookEvent( db, event_id, self.debug) except Exception: self.exception() failed_events.add(event_id) else: finished_events.append(event_id) cursor.execute( """DELETE FROM extensionfilterhookevents WHERE id=ANY (%s)""", (finished_events, )) db.commit() timeout = self.run_maintenance() if timeout is None: timeout = 86400 self.debug("sleeping %d seconds" % timeout) self.signal_idle_state() before = time.time() time.sleep(timeout) if self.interrupted: self.debug("sleep interrupted after %.2f seconds" % (time.time() - before))
def install(data): path = os.path.join(installation.root_dir, "src", "data", "preferences.json") with open(path) as preferences_file: preferences = json.load(preferences_file) import dbutils with installation.utils.as_critic_system_user(): with dbutils.Database() as db: for item in sorted(preferences.keys()): add_preference(db, item, preferences[item], silent=True) db.commit() if not installation.quiet: print "Added %d preferences." % len(preferences) return True
def __purge(self): db = dbutils.Database() cursor = db.cursor() cursor.execute("""SELECT COUNT(*) FROM changesets JOIN customchangesets ON (customchangesets.changeset=changesets.id) WHERE time < NOW() - INTERVAL '3 months'""") npurged = cursor.fetchone()[0] if npurged: self.info("purging %d old custom changesets" % npurged) cursor.execute( "DELETE FROM changesets USING customchangesets WHERE id=changeset AND time < NOW() - INTERVAL '3 months'" ) db.commit() db.close() return npurged
def perform_job(): soft_limit, hard_limit = getrlimit(RLIMIT_RSS) rss_limit = configuration.services.CHANGESET["rss_limit"] if soft_limit < rss_limit: setrlimit(RLIMIT_RSS, (rss_limit, hard_limit)) from changeset.create import createChangeset request = json_decode(sys.stdin.read()) try: db = dbutils.Database() createChangeset(db, request) db.close() sys.stdout.write(json_encode(request)) except: print "Request:" print json_encode(request, indent=2) print print_exc(file=sys.stdout)
def handle_input(self, data): try: data = json_decode(data) except ValueError: return message = "connection from %s:%d:" % self.__peeraddress message += "\n repository: %s" % data["repository"] if data.has_key("timeout"): message += "\n timeout: %d" % data["timeout"] if data["branches"]: message += "\n branches: %s" % ", ".join( data["branches"]) if data["tags"]: message += "\n tags: %s" % ", ".join(data["tags"]) self.server.info(message) db = dbutils.Database() try: cursor = db.cursor() notify_tracker = False wait_for_reply = False for branch in data["branches"]: cursor.execute( """SELECT id, local_name FROM trackedbranches WHERE remote=%s AND remote_name=%s AND NOT disabled AND next IS NOT NULL""", (data["repository"], branch)) row = cursor.fetchone() if row: branch_id, local_name = row cursor.execute( """UPDATE trackedbranches SET next=NULL WHERE id=%s""", (branch_id, )) notify_tracker = True self.server.debug("tracked branch: %s" % local_name) if len(data["branches"] ) == 1 and local_name.startswith("r/"): wait_for_reply = (True, branch_id) self.server.debug(" will wait for reply") if data["tags"]: cursor.execute( """SELECT id FROM trackedbranches WHERE remote=%s AND remote_name=%s AND NOT disabled AND next IS NOT NULL""", (data["repository"], "*")) row = cursor.fetchone() if row: branch_id = row[0] cursor.execute( """UPDATE trackedbranches SET next=NULL WHERE id=%s""", (branch_id, )) notify_tracker = True db.commit() if notify_tracker: if wait_for_reply: branch_id = wait_for_reply[1] cursor.execute( "SELECT COUNT(*) FROM trackedbranchlog WHERE branch=%s", (branch_id, )) log_offset = cursor.fetchone()[0] self.server.add_peer( BranchTrackerHook.WaitForUpdate( self, branch_id, data.get("timeout", 30), log_offset)) try: branchtracker_pid = int( open(configuration.services.BRANCHTRACKER[ "pidfile_path"]).read().strip()) os.kill(branchtracker_pid, signal.SIGHUP) except: self.server.exception() return if wait_for_reply: return self.close() finally: try: db.close() except: pass
def main(environ, start_response): request_start = time.time() db = dbutils.Database() user = None try: try: req = request.Request(db, environ, start_response) if req.user is None: if configuration.base.AUTHENTICATION_MODE == "critic": if configuration.base.SESSION_TYPE == "httpauth": req.setStatus(401) req.addResponseHeader("WWW-Authenticate", "Basic realm=\"Critic\"") req.start() return elif configuration.base.ALLOW_ANONYMOUS_USER or req.path in ( "login", "validatelogin"): user = dbutils.User.makeAnonymous() elif req.method == "GET": raise page.utils.NeedLogin, req else: # Don't try to redirect POST requests to the login page. req.setStatus(403) req.start() return else: try: user = dbutils.User.fromName(db, req.user) except dbutils.NoSuchUser: cursor = db.cursor() cursor.execute( """INSERT INTO users (name, email, fullname) VALUES (%s, %s, %s) RETURNING id""", (req.user, getUserEmailAddress(req.user), req.user)) user = dbutils.User.fromId(db, cursor.fetchone()[0]) db.commit() user.loadPreferences(db) if user.status == 'retired': cursor = db.cursor() cursor.execute("UPDATE users SET status='current' WHERE id=%s", (user.id, )) user = dbutils.User.fromId(db, user.id) db.commit() if not user.getPreference(db, "debug.profiling.databaseQueries"): db.disableProfiling() if not req.path: if user.isAnonymous(): location = "tutorial" else: location = user.getPreference(db, "defaultPage") if req.query: location += "?" + req.query req.setStatus(307) req.addResponseHeader("Location", location) req.start() return if req.path == "redirect": target = req.getParameter("target", "/") if req.method == "POST": # Don't use HTTP redirect for POST requests. req.setContentType("text/html") req.start() yield "<meta http-equiv='refresh' content='0; %s'>" % htmlify( target) return else: raise page.utils.MovedTemporarily, target if req.path.startswith("!/"): req.path = req.path[2:] elif configuration.extensions.ENABLED: handled = extensions.executePage(db, req, user) if handled: req.start() yield handled return if req.path.startswith("r/"): req.query = "id=" + req.path[2:] + ("&" + req.query if req.query else "") req.path = "showreview" if configuration.extensions.ENABLED: match = RE_EXTENSION_RESOURCE.match(req.path) if match: content_type, resource = extensions.getExtensionResource( req, db, user, match.group(1)) if resource: req.setContentType(content_type) req.start() yield resource return else: req.setStatus(404) req.start() return if req.path.startswith("download/"): operation = download else: operation = operations.get(req.path) if operation: req.setContentType("text/plain") try: result = operation(req, db, user) except OperationError, error: result = error except page.utils.DisplayMessage, message: result = "error:" + message.title if message.body: result += " " + message.body except Exception, exception: result = "error:\n" + "".join( traceback.format_exception(*sys.exc_info()))
def __maintenance(self): with dbutils.Database() as db: cursor = db.cursor() # Update the UTC offsets of all timezones. # # The PostgreSQL database server has accurate (DST-adjusted) values, # but is very slow to query, so we cache the UTC offsets in our # 'timezones' table. This call updates that cache every night. # (This is obviously a no-op most nights, but we don't want to have # to care about which nights it isn't.) self.debug("updating timezones") dbutils.updateTimezones(db) if self.terminated: return # Execute scheduled review branch archivals. if configuration.base.ARCHIVE_REVIEW_BRANCHES: repository = None cursor.execute( """SELECT branches.repository, branches.id, branches.name FROM scheduledreviewbrancharchivals JOIN reviews ON (reviews.id=scheduledreviewbrancharchivals.review) JOIN branches ON (branches.id=reviews.branch) WHERE scheduledreviewbrancharchivals.deadline <= NOW() AND reviews.state IN ('closed', 'dropped') AND NOT branches.archived ORDER BY branches.repository""", for_update=True) for repository_id, branch_id, branch_name in cursor: if not repository or repository.id != repository_id: if repository: repository.stopBatch() repository = gitutils.Repository.fromId( db, repository_id) self.info("archiving branches in: " + repository.name) self.info(" " + branch_name) branch = dbutils.Branch.fromId(db, branch_id, repository=repository) try: branch.archive(db) except Exception: self.exception(as_warning=True) # Since NOW() returns the same value each time within a single # transaction, this is guaranteed to delete only the set of # archivals we selected above. cursor.execute("""DELETE FROM scheduledreviewbrancharchivals WHERE deadline <= NOW()""") db.commit() # Run a garbage collect in all Git repositories, to keep them neat # and tidy. Also pack keepalive refs. cursor.execute("SELECT name FROM repositories") for (repository_name, ) in cursor: self.debug("repository GC: %s" % repository_name) try: repository = gitutils.Repository.fromName( db, repository_name) repository.packKeepaliveRefs() repository.run("gc", "--prune=1 day", "--quiet") repository.stopBatch() except Exception: self.exception("repository GC failed: %s" % repository_name) if self.terminated: return if configuration.extensions.ENABLED: now = time.time() max_age = 7 * 24 * 60 * 60 base_path = os.path.join(configuration.paths.DATA_DIR, "temporary", "EXTENSIONS") for user_name in os.listdir(base_path): user_dir = os.path.join(base_path, user_name) for extension_id in os.listdir(user_dir): extension_dir = os.path.join(user_dir, extension_id) for repository_name in os.listdir(extension_dir): repository_dir = os.path.join( extension_dir, repository_name) age = now - os.stat(repository_dir).st_mtime if age > max_age: self.info("Removing repository work copy: %s" % repository_dir) shutil.rmtree(repository_dir)
def upgrade(arguments, data): git = data["installation.prereqs.git"] path = "src/data/preferences.json" old_sha1 = data["sha1"] old_file_sha1 = installation.utils.get_file_sha1(git, old_sha1, path) new_sha1 = installation.utils.run_git([git, "rev-parse", "HEAD"], cwd=installation.root_dir).strip() new_file_sha1 = installation.utils.get_file_sha1(git, new_sha1, path) if old_file_sha1: old_source = installation.utils.run_git( [git, "cat-file", "blob", old_file_sha1], cwd=installation.root_dir) old_preferences = json.loads(old_source) else: old_preferences = {} preferences_path = os.path.join(installation.root_dir, path) with open(preferences_path) as preferences_file: new_preferences = json.load(preferences_file) def update_preferences(old_preferences, new_preferences, db_preferences): for item in new_preferences.keys(): if item not in db_preferences: add_preference(db, item, new_preferences[item]) elif db_preferences[item] != new_preferences[item]: type_changed = False if db_preferences[item]["type"] != new_preferences[item][ "type"]: # If the type has changed, we really have to update it; code # will depend on it having the right type. update = True type_changed = True elif item in old_preferences \ and db_preferences[item] == old_preferences[item]: # The preference in the database is identical to what we # originally installed; there should be no harm in updating # it. update = True elif db_preferences[item]["default"] == new_preferences[item][ "default"]: # The default value is the same => only description or flags # has changed. Probably safe to silently update. update = True else: if item in old_preferences \ and db_preferences[item]["default"] != old_preferences[item]["default"]: # The default value appears to have been changed in the # database. Ask the user before overwriting it with an # updated default value. print print textwrap.fill( "The default value for the preference '%s' has been " "changed in this version of Critic, but it appears to " "also have been modified in the database." % item) default = False else: # The default value has changed, and we don't know if # the value is what was originally installed, because we # don't know what was originally installed. Ask the # user before overwriting the current value. print print textwrap.fill( "The default value for the preference '%s' has been " "changed in this version of Critic." % item) default = True print print " Value in database: %r" % db_preferences[item][ "default"] print " New/updated value: %r" % new_preferences[item][ "default"] print update = installation.input.yes_or_no( "Would you like to update the database with the new value?", default=default) if update: update_preference(db, item, new_preferences[item], type_changed) # Only check for preferences to remove if the preference data has # changed. Otherwise, every upgrade would ask to remove any extra # preferences in the database. if old_file_sha1 != new_file_sha1: for item in db_preferences.keys(): if item not in new_preferences: if item in old_preferences \ and db_preferences[item] == old_preferences[item]: # The preference in the database is identical to what we # originally installed; there should be no harm in # updating it. remove = True else: print print textwrap.fill( "The preference '%s' exists in the database but " "not in the installation data, meaning it would " "not have been added to the database if this " "version of Critic was installed from scratch." % item) print remove = installation.input.yes_or_no( "Would you like to remove it from the database?", default=True) if remove: remove_preference(db, item) db.commit() import dbutils with installation.utils.as_critic_system_user(): with dbutils.Database() as db: update_preferences(old_preferences, new_preferences, load_preferences(db)) return True
def __compact(self): import syntaxhighlight cache_dir = configuration.services.HIGHLIGHT["cache_dir"] if not os.path.isdir(cache_dir): # Newly installed system that hasn't highlighted anything. return 0, 0, 0, 0 self.info("cache compacting started") now = time.time() max_age_uncompressed = 7 * 24 * 60 * 60 max_age_compressed = 90 * 24 * 60 * 60 uncompressed_count = 0 compressed_count = 0 purged_paths = [] db = dbutils.Database() cursor = db.cursor() cursor.execute( "CREATE TEMPORARY TABLE purged (sha1 CHAR(40) PRIMARY KEY)") cursor.execute( "INSERT INTO purged (sha1) SELECT DISTINCT sha1 FROM codecontexts" ) for section in sorted(os.listdir(cache_dir)): if len(section) == 2: for filename in os.listdir("%s/%s" % (cache_dir, section)): fullname = "%s/%s/%s" % (cache_dir, section, filename) age = now - os.stat(fullname).st_mtime if len(filename ) > 38 and filename[38] == "." and filename[ 39:] in syntaxhighlight.LANGUAGES: cursor.execute("DELETE FROM purged WHERE sha1=%s", (section + filename[:38], )) if age > max_age_uncompressed: self.debug("compressing: %s/%s" % (section, filename)) worker = process([find_bzip2(), fullname]) worker.wait() compressed_count += 1 else: uncompressed_count += 1 elif len(filename ) > 41 and filename[38] == "." and filename[ -4] == "." and filename[ 39:-4] in syntaxhighlight.LANGUAGES: if filename.endswith(".bz2"): if age > max_age_compressed: self.debug("purging: %s/%s" % (section, filename)) purged_paths.append(fullname) else: cursor.execute( "DELETE FROM purged WHERE sha1=%s", (section + filename[:38], )) compressed_count += 1 elif filename.endswith(".ctx"): self.debug("deleting context file: %s/%s" % (section, filename)) os.unlink(fullname) else: os.unlink(fullname) self.info( "cache compacting finished: uncompressed=%d / compressed=%d / purged=%d" % (uncompressed_count, compressed_count, len(purged_paths))) if purged_paths: for path in purged_paths: os.unlink(path) cursor.execute("SELECT COUNT(*) FROM purged") purged_contexts = cursor.fetchone()[0] cursor.execute( "DELETE FROM codecontexts USING purged WHERE codecontexts.sha1=purged.sha1" ) db.commit() db.close() return uncompressed_count, compressed_count, len( purged_paths), purged_contexts
import time sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), ".."))) import dbutils from textutils import json_encode, json_decode if "--wait-for-update" in sys.argv: data = json_decode(sys.stdin.read()) branch_id = data["branch_id"] timeout = data["timeout"] log_offset = data["log_offset"] db = dbutils.Database() cursor = db.cursor() cursor.execute("SELECT MAX(time) FROM trackedbranchlog WHERE branch=%s", (branch_id, )) last_log_entry = cursor.fetchone()[0] start = time.time() status = None output = "" while time.time() - start < timeout: time.sleep(0.5) db.commit()
def __init__(self): super(HighlightServer, self).__init__(service=configuration.services.HIGHLIGHT) self.db = dbutils.Database() self.register_maintenance(hour=3, minute=15, callback=self.__compact)
def startSession(): return api.critic.Critic(Critic(dbutils.Database()))
def startSession(allow_unsafe_cursors=True): return api.critic.Critic(Critic(dbutils.Database(allow_unsafe_cursors)))
def init(): global db db = dbutils.Database()
def cursors(): import dbutils class TestException(Exception): pass # Create some playground tables. We'll drop them later if all goes well, # but it doesn't really matter if we don't. db = dbutils.Database() db.cursor().execute( "CREATE TABLE playground1 ( x INTEGER PRIMARY KEY, y INTEGER )") db.cursor().execute( "CREATE TABLE playground2 ( x INTEGER PRIMARY KEY, y INTEGER )") db.commit() db.close() # Basic testing of read-only / updating cursors. with dbutils.Database() as db: ro_cursor = db.readonly_cursor() with db.updating_cursor("playground1") as cursor: cursor.executemany( "INSERT INTO playground1 (x, y) VALUES (%s, %s)", [(1, 1), (2, 2), (3, 3)]) db.rollback() ro_cursor.execute("SELECT x, y FROM playground1") assert len(list(ro_cursor)) == 3 try: with db.updating_cursor("playground1") as cursor: cursor.execute( "INSERT INTO playground1 (x, y) VALUES (%s, %s)", (4, 4)) raise TestException except TestException: pass db.commit() ro_cursor.execute("SELECT x, y FROM playground1") assert len(list(ro_cursor)) == 3 try: with db.updating_cursor("playground1") as cursor: cursor.execute( "INSERT INTO playground2 (x, y) VALUES (%s, %s)", (1, 1)) except dbutils.InvalidCursorError as error: assert error.message == "invalid table for updating cursor: playground2" db.commit() ro_cursor.execute("SELECT x, y FROM playground2") assert len(list(ro_cursor)) == 0 try: with db.updating_cursor("playground2") as cursor: cursor.execute("DELETE FROM playground1 WHERE x=1") except dbutils.InvalidCursorError as error: assert error.message == "invalid table for updating cursor: playground1" db.commit() ro_cursor.execute("SELECT x, y FROM playground1") assert len(list(ro_cursor)) == 3 with db.updating_cursor("playground1") as cursor: cursor.execute("DELETE FROM playground1 WHERE x=1") db.rollback() ro_cursor.execute("SELECT x, y FROM playground1") assert len(list(ro_cursor)) == 2 with db.updating_cursor("playground1") as cursor: cursor.execute("UPDATE playground1 SET y=-2 WHERE x=2") db.rollback() ro_cursor.execute("SELECT y FROM playground1 WHERE x=2") assert ro_cursor.fetchone()[0] == -2 with db.updating_cursor("playground1") as cursor: try: with db.updating_cursor("playground2"): assert False except dbutils.InvalidCursorError as error: assert error.message == "concurrent updating cursor requested" stored_cursor = None with db.updating_cursor("playground1") as cursor: stored_cursor = cursor try: stored_cursor.execute("UPDATE playground1 SET y=-3 WHERE x=3") except dbutils.InvalidCursorError as error: assert error.message == "disabled updating cursor used" db.commit() ro_cursor.execute("SELECT y FROM playground1 WHERE x=3") assert ro_cursor.fetchone()[0] == 3 try: with db.updating_cursor("playground1") as cursor: cursor.execute("DROP TABLE playground1") except dbutils.InvalidCursorError as error: assert error.message == "unrecognized query: DROP", error.message try: with db.updating_cursor("playground1") as cursor: cursor.execute("DELETE FROM playground1") db.commit() except dbutils.InvalidCursorError as error: assert error.message == "manual commit when using updating cursor", error.message db.commit() ro_cursor.execute("SELECT x, y FROM playground1") assert len(list(ro_cursor)) == 2 # Test mixing of unsafe cursor and updating cursor. with dbutils.Database() as db: ro_cursor = db.readonly_cursor() unsafe_cursor = db.cursor() with db.updating_cursor("playground1") as cursor: cursor.execute("DELETE FROM playground1") cursor.executemany( "INSERT INTO playground1 (x, y) VALUES (%s, %s)", [(1, 1), (2, 2), (3, 3)]) db.rollback() ro_cursor.execute("SELECT x, y FROM playground1") assert len(list(ro_cursor)) == 3 # Can't create an updating cursor after executing an updating query # using an unsafe cursor. try: unsafe_cursor.execute("DELETE FROM playground1") with db.updating_cursor("playground1") as cursor: assert False except dbutils.InvalidCursorError as error: assert error.message == "mixed unsafe and updating cursors" db.rollback() # Can't commit an updating cursor after executing an updating query # using an unsafe cursor. try: with db.updating_cursor("playground1") as cursor: cursor.execute( "INSERT INTO playground1 (x, y) VALUES (%s, %s)", (4, 4)) unsafe_cursor.execute("DELETE FROM playground1") except dbutils.InvalidCursorError as error: assert error.message == "mixed unsafe and updating cursors" db.commit() ro_cursor.execute("SELECT x, y FROM playground1") assert len(list(ro_cursor)) == 3 # If the transaction is committed or rolled back after execution of # updating query using unsafe cursor, then use of updating cursor is # fine. unsafe_cursor.execute("DELETE FROM playground1") db.rollback() with db.updating_cursor("playground1") as cursor: cursor.execute("UPDATE playground1 SET y=-2 WHERE x=2") db.rollback() ro_cursor.execute("SELECT y FROM playground1") assert set(y for (y, ) in ro_cursor) == set([1, -2, 3]) # If the transaction is committed or rolled back after execution of # updating query using unsafe cursor, then use of updating cursor is # fine. unsafe_cursor.execute("UPDATE playground1 SET y=-1 WHERE x=1") db.commit() with db.updating_cursor("playground1") as cursor: cursor.execute("UPDATE playground1 SET y=-3 WHERE x=3") db.rollback() ro_cursor.execute("SELECT y FROM playground1") assert set(y for (y, ) in ro_cursor) == set([-1, -2, -3]) # Drop the playground table. db = dbutils.Database() db.cursor().execute("DROP TABLE playground1") db.cursor().execute("DROP TABLE playground2") db.commit() db.close() print "cursors: ok"
def run(self): self.db = dbutils.Database() while not self.terminated: self.interrupted = False cursor = self.db.cursor() cursor.execute("""SELECT id, repository, local_name, remote, remote_name, forced FROM trackedbranches WHERE NOT disabled AND (next IS NULL OR next < NOW()) ORDER BY next ASC NULLS FIRST""") rows = cursor.fetchall() for trackedbranch_id, repository_id, local_name, remote, remote_name, forced in rows: if local_name == "*": self.info("checking tags in %s" % remote) else: self.info("checking %s in %s" % (remote_name, remote)) cursor.execute("""UPDATE trackedbranches SET previous=NOW(), next=NOW() + delay, updating=TRUE WHERE id=%s""", (trackedbranch_id,)) self.db.commit() if self.update(trackedbranch_id, repository_id, local_name, remote, remote_name, forced): cursor.execute("""UPDATE trackedbranches SET updating=FALSE WHERE id=%s""", (trackedbranch_id,)) cursor.execute("""SELECT next::text FROM trackedbranches WHERE id=%s""", (trackedbranch_id,)) self.info(" next scheduled update at %s" % cursor.fetchone()) else: cursor.execute("""UPDATE trackedbranches SET updating=FALSE, disabled=TRUE WHERE id=%s""", (trackedbranch_id,)) self.info(" tracking disabled") self.db.commit() if self.terminated: break cursor.execute("""SELECT 1 FROM trackedbranches WHERE NOT disabled AND next IS NULL""") if not cursor.fetchone(): maintenance_delay = self.run_maintenance() if maintenance_delay is None: maintenance_delay = 3600 cursor.execute("""SELECT COUNT(*), EXTRACT('epoch' FROM (MIN(next) - NOW())) FROM trackedbranches WHERE NOT disabled""") enabled_branches, update_delay = cursor.fetchone() if not enabled_branches: self.info("nothing to do") update_delay = 3600 else: update_delay = max(0, int(update_delay)) delay = min(maintenance_delay, update_delay) if delay: self.signal_idle_state() self.debug("sleeping %d seconds" % delay) gitutils.Repository.forEach(self.db, lambda db, repository: repository.stopBatch()) self.db.commit() before = time.time() time.sleep(delay) if self.interrupted: self.debug("sleep interrupted after %.2f seconds" % (time.time() - before)) self.db.commit()
def run(self): self.db = dbutils.Database() while not self.terminated: self.interrupted = False cursor = self.db.cursor() cursor.execute( """SELECT id, repository, local_name, remote, remote_name, forced FROM trackedbranches WHERE NOT disabled AND (next IS NULL OR next < NOW()) ORDER BY next ASC NULLS FIRST""") rows = cursor.fetchall() for trackedbranch_id, repository_id, local_name, remote, remote_name, forced in rows: if local_name == "*": self.info("checking tags in %s" % remote) else: self.info("checking %s in %s" % (remote_name, remote)) cursor.execute( """UPDATE trackedbranches SET previous=NOW(), next=NOW() + delay, updating=TRUE WHERE repository=%s AND local_name=%s RETURNING next::text""", (repository_id, local_name)) next_at = cursor.fetchone()[0] self.db.commit() if self.update(trackedbranch_id, repository_id, local_name, remote, remote_name, forced): cursor.execute( """UPDATE trackedbranches SET updating=FALSE WHERE repository=%s AND local_name=%s""", (repository_id, local_name)) self.info(" next scheduled update at %s" % next_at) else: cursor.execute( """UPDATE trackedbranches SET updating=FALSE, disabled=TRUE WHERE repository=%s AND local_name=%s""", (repository_id, local_name)) self.info(" tracking disabled") self.db.commit() if self.terminated: break cursor.execute("""SELECT 1 FROM trackedbranches WHERE NOT disabled AND next IS NULL""") if not cursor.fetchone(): cursor.execute("""SELECT 1 FROM trackedbranches WHERE NOT disabled""") if not cursor.fetchone(): self.info("nothing to do; sleeping one hour") delay = 3600 else: cursor.execute( """SELECT EXTRACT('epoch' FROM (MIN(next) - NOW())) FROM trackedbranches WHERE NOT disabled""") delay = max(0, int(cursor.fetchone()[0] or 0)) if delay: self.debug("sleeping %d seconds" % delay) if delay: before = time.time() time.sleep(delay) if self.interrupted: self.debug("sleep interrupted after %.2f seconds" % (time.time() - before)) self.db.commit()
def slave(): import StringIO import traceback import dbutils import gitutils import index def reject(message): sys_stdout.write(json_encode({"status": "reject", "message": message})) sys.exit(0) def error(message): sys_stdout.write(json_encode({"status": "error", "error": message})) sys.exit(0) db = dbutils.Database() try: data = sys.stdin.read() request = json_decode(data) create_branches = [] delete_branches = [] update_branches = [] create_tags = [] delete_tags = [] update_tags = [] repository = gitutils.Repository.fromName(db, request["repository_name"]) user = index.getUser(db, request["user_name"]) if request["flags"] and user.isSystem(): flags = dict( flag.split("=", 1) for flag in request["flags"].split(",")) else: flags = {} sys.stdout = StringIO.StringIO() commits_to_process = set() for ref in request["refs"]: name = ref["name"] old_sha1 = ref["old_sha1"] new_sha1 = ref["new_sha1"] if "//" in name: reject("invalid ref name: '%s'" % name) if not name.startswith("refs/"): reject("unexpected ref name: '%s'" % name) if new_sha1 != '0000000000000000000000000000000000000000': commits_to_process.add(new_sha1) name = name[len("refs/"):] if name.startswith("heads/"): name = name[len("heads/"):] if new_sha1 == '0000000000000000000000000000000000000000': delete_branches.append((name, old_sha1)) elif old_sha1 == '0000000000000000000000000000000000000000': create_branches.append((name, new_sha1)) else: update_branches.append((name, old_sha1, new_sha1)) elif name.startswith("tags/"): name = name[len("tags/"):] if old_sha1 == '0000000000000000000000000000000000000000': create_tags.append((name, new_sha1)) elif new_sha1 == '0000000000000000000000000000000000000000': delete_tags.append(name) else: update_tags.append((name, old_sha1, new_sha1)) elif name.startswith("temporary/") or name.startswith( "keepalive/"): # len("temporary/") == len("keepalive/") name = name[len("temporary/"):] if name != new_sha1: reject("invalid update of '%s'; value is not %s" % (ref["name"], name)) else: reject("unexpected ref name: '%s'" % ref["name"]) multiple = (len(delete_branches) + len(update_branches) + len(create_branches) + len(delete_tags) + len(update_tags) + len(create_tags)) > 1 info = [] for sha1 in commits_to_process: index.processCommits(db, repository, sha1) for name, old in delete_branches: index.deleteBranch(db, user, repository, name, old) info.append("branch deleted: %s" % name) for name, old, new in update_branches: index.updateBranch(db, user, repository, name, old, new, multiple, flags) info.append("branch updated: %s (%s..%s)" % (name, old[:8], new[:8])) index.createBranches(db, user, repository, create_branches, flags) for name, new in create_branches: info.append("branch created: %s (%s)" % (name, new[:8])) for name in delete_tags: index.deleteTag(db, user, repository, name) info.append("tag deleted: %s" % name) for name, old, new in update_tags: index.updateTag(db, user, repository, name, old, new) info.append("tag updated: %s (%s..%s)" % (name, old[:8], new[:8])) for name, new in create_tags: index.createTag(db, user, repository, name, new) info.append("tag created: %s (%s)" % (name, new[:8])) sys_stdout.write( json_encode({ "status": "ok", "accept": True, "output": sys.stdout.getvalue(), "info": info })) db.commit() except index.IndexException as exception: sys_stdout.write( json_encode({ "status": "ok", "accept": False, "output": exception.message, "info": info })) except SystemExit: raise except: exception = traceback.format_exc() message = """\ %s Request: %s %s""" % (exception.splitlines()[-1], json_encode( request, indent=2), traceback.format_exc()) sys_stdout.write(json_encode({"status": "error", "error": message})) finally: db.close()
def handle_input(self, data): try: data = json_decode(data) except ValueError: return message = "connection from %s:%d:" % self.__peeraddress message += "\n repository: %s" % data["repository"] if data.has_key("timeout"): message += "\n timeout: %d" % data["timeout"] if data["branches"]: message += "\n branches: %s" % ", ".join( data["branches"]) if data["tags"]: message += "\n tags: %s" % ", ".join(data["tags"]) self.server.info(message) db = dbutils.Database() try: cursor = db.cursor() notify_tracker = False wait_for_reply = False # Make sure the 'knownremotes' table has this remote listed # as "pushing" since it obviously is. cursor.execute( """SELECT pushing FROM knownremotes WHERE url=%s""", (data["repository"], )) row = cursor.fetchone() if not row: cursor.execute( """INSERT INTO knownremotes (url, pushing) VALUES (%s, TRUE)""", (data["repository"], )) elif not row[0]: cursor.execute( """UPDATE knownremotes SET pushing=TRUE WHERE url=%s""", (data["repository"], )) # If we just recorded this remote as "pushing," adjust the # configured updating frequency of any existing tracked # branches from it. if not row or not row[0]: cursor.execute( """UPDATE trackedbranches SET delay='1 week' WHERE remote=%s""", (data["repository"], )) for branch in data["branches"]: cursor.execute( """SELECT id, local_name FROM trackedbranches WHERE remote=%s AND remote_name=%s AND NOT disabled AND next IS NOT NULL""", (data["repository"], branch)) row = cursor.fetchone() if row: branch_id, local_name = row cursor.execute( """UPDATE trackedbranches SET next=NULL WHERE id=%s""", (branch_id, )) notify_tracker = True self.server.debug("tracked branch: %s" % local_name) if len(data["branches"] ) == 1 and local_name.startswith("r/"): wait_for_reply = (True, branch_id) self.server.debug(" will wait for reply") if data["tags"]: cursor.execute( """SELECT id FROM trackedbranches WHERE remote=%s AND remote_name=%s AND NOT disabled AND next IS NOT NULL""", (data["repository"], "*")) row = cursor.fetchone() if row: branch_id = row[0] cursor.execute( """UPDATE trackedbranches SET next=NULL WHERE id=%s""", (branch_id, )) notify_tracker = True db.commit() if notify_tracker: if wait_for_reply: branch_id = wait_for_reply[1] cursor.execute( "SELECT COUNT(*) FROM trackedbranchlog WHERE branch=%s", (branch_id, )) log_offset = cursor.fetchone()[0] self.server.add_peer( BranchTrackerHook.WaitForUpdate( self, branch_id, data.get("timeout", 30), log_offset)) try: branchtracker_pid = int( open(configuration.services.BRANCHTRACKER[ "pidfile_path"]).read().strip()) os.kill(branchtracker_pid, signal.SIGHUP) except: self.server.exception() return if wait_for_reply: return self.close() finally: try: db.close() except: pass
def process_request(environ, start_response): request_start = time.time() db = dbutils.Database() user = None try: try: req = request.Request(db, environ, start_response) req.setUser(db) if req.user is None: if configuration.base.AUTHENTICATION_MODE == "host": user = dbutils.User.makeAnonymous() elif configuration.base.SESSION_TYPE == "httpauth": req.requestHTTPAuthentication() return [] elif req.path.startswith("externalauth/"): provider_name = req.path[len("externalauth/"):] raise request.DoExternalAuthentication(provider_name) elif req.path.startswith("oauth/"): provider_name = req.path[len("oauth/"):] if provider_name in auth.PROVIDERS: provider = auth.PROVIDERS[provider_name] if isinstance(provider, auth.OAuthProvider): if finishOAuth(db, req, provider): return [] elif configuration.base.SESSION_TYPE == "cookie": if req.cookies.get("has_sid") == "1": req.ensureSecure() if configuration.base.ALLOW_ANONYMOUS_USER \ or req.path in request.INSECURE_PATHS \ or req.path.startswith("static-resource/"): user = dbutils.User.makeAnonymous() # Don't try to redirect POST requests to the login page. elif req.method == "GET": if configuration.base.AUTHENTICATION_MODE == "critic": raise request.NeedLogin(req) else: raise request.DoExternalAuthentication( configuration.base.AUTHENTICATION_MODE, req.getTargetURL()) if not user: req.setStatus(403) req.start() return [] else: try: user = dbutils.User.fromName(db, req.user) except dbutils.NoSuchUser: if configuration.base.AUTHENTICATION_MODE == "host": email = getUserEmailAddress(req.user) user = dbutils.User.create(db, req.user, req.user, email, email_verified=None) db.commit() else: # This can't really happen. raise user.loadPreferences(db) if user.status == 'retired': cursor = db.cursor() cursor.execute("UPDATE users SET status='current' WHERE id=%s", (user.id, )) user = dbutils.User.fromId(db, user.id) db.commit() if not user.getPreference(db, "debug.profiling.databaseQueries"): db.disableProfiling() if not req.path: if user.isAnonymous(): location = "tutorial" else: location = user.getPreference(db, "defaultPage") if req.query: location += "?" + req.query req.setStatus(307) req.addResponseHeader("Location", location) req.start() return [] if req.path == "redirect": target = req.getParameter("target", "/") if req.method == "POST": # Don't use HTTP redirect for POST requests. req.setContentType("text/html") req.start() return [ "<meta http-equiv='refresh' content='0; %s'>" % htmlify(target) ] else: raise request.MovedTemporarily(target) # Require a .git suffix on HTTP(S) repository URLs unless the user- # agent starts with "git/" (as Git's normally does.) # # Our objectives are: # # 1) Not to require Git's user-agent to be its default value, since # the user might have to override it to get through firewalls. # 2) Never to send regular user requests to 'git http-backend' by # mistake. # # This is a compromise. if req.getRequestHeader("User-Agent", "").startswith("git/"): suffix = None else: suffix = ".git" if handleRepositoryPath(db, req, user, suffix): db = None return [] if req.path.startswith("!/"): req.path = req.path[2:] elif configuration.extensions.ENABLED: handled = extensions.role.page.execute(db, req, user) if isinstance(handled, basestring): req.start() return [handled] if req.path.startswith("static-resource/"): return handleStaticResource(req) if req.path.startswith("r/"): req.query = "id=" + req.path[2:] + ("&" + req.query if req.query else "") req.path = "showreview" if configuration.extensions.ENABLED: match = RE_EXTENSION_RESOURCE.match(req.path) if match: content_type, resource = extensions.resource.get( req, db, user, match.group(1)) if resource: req.setContentType(content_type) if content_type.startswith("image/"): req.addResponseHeader("Cache-Control", "max-age=3600") req.start() return [resource] else: req.setStatus(404) req.start() return [] if req.path.startswith("download/"): operationfn = download else: operationfn = OPERATIONS.get(req.path) if operationfn: result = operationfn(req, db, user) if isinstance(result, (OperationResult, OperationError)): req.setContentType("text/json") if isinstance(result, OperationResult): if db.profiling: result.set("__profiling__", formatDBProfiling(db)) result.set("__time__", time.time() - request_start) elif not req.hasContentType(): req.setContentType("text/plain") req.start() if isinstance(result, unicode): return [result.encode("utf8")] else: return [str(result)] override_user = req.getParameter("user", None) while True: pagefn = PAGES.get(req.path) if pagefn: try: if not user.isAnonymous() and override_user: user = dbutils.User.fromName(db, override_user) req.setContentType("text/html") result = pagefn(req, db, user) if db.profiling and not (isinstance(result, str) or isinstance(result, Document)): source = "" for fragment in result: source += fragment result = source if isinstance(result, str) or isinstance( result, Document): req.start() result = str(result) result += ("<!-- total request time: %.2f ms -->" % ((time.time() - request_start) * 1000)) if db.profiling: result += ("<!--\n\n%s\n\n -->" % formatDBProfiling(db)) return [result] else: result = WrappedResult(db, req, user, result) req.start() # Prevent the finally clause below from closing the # connection. WrappedResult does it instead. db = None return result except gitutils.NoSuchRepository as error: raise page.utils.DisplayMessage( title="Invalid URI Parameter!", body=error.message) except gitutils.GitReferenceError as error: if error.ref: raise page.utils.DisplayMessage( title="Specified ref not found", body=("There is no ref named \"%s\" in %s." % (error.ref, error.repository))) elif error.sha1: raise page.utils.DisplayMessage( title="SHA-1 not found", body=error.message) else: raise except dbutils.NoSuchUser as error: raise page.utils.DisplayMessage( title="Invalid URI Parameter!", body=error.message) except dbutils.NoSuchReview as error: raise page.utils.DisplayMessage( title="Invalid URI Parameter!", body=error.message) path = req.path if "/" in path: repository = gitutils.Repository.fromName( db, path.split("/", 1)[0]) if repository: path = path.split("/", 1)[1] else: repository = None def revparsePlain(item): try: return gitutils.getTaggedCommit( repository, repository.revparse(item)) except: raise revparse = revparsePlain if repository is None: review_id = req.getParameter("review", None, filter=int) if review_id: cursor = db.cursor() cursor.execute( """SELECT repository FROM branches JOIN reviews ON (reviews.branch=branches.id) WHERE reviews.id=%s""", (review_id, )) row = cursor.fetchone() if row: repository = gitutils.Repository.fromId(db, row[0]) def revparseWithReview(item): if re.match("^[0-9a-f]+$", item): cursor.execute( """SELECT sha1 FROM commits JOIN changesets ON (changesets.child=commits.id) JOIN reviewchangesets ON (reviewchangesets.changeset=changesets.id) WHERE reviewchangesets.review=%s AND commits.sha1 LIKE %s""", (review_id, item + "%")) row = cursor.fetchone() if row: return row[0] else: return revparsePlain(item) revparse = revparseWithReview if repository is None: repository = gitutils.Repository.fromName( db, user.getPreference(db, "defaultRepository")) if gitutils.re_sha1.match(path): if repository and not repository.iscommit(path): repository = None if not repository: try: repository = gitutils.Repository.fromSHA1( db, path) except gitutils.GitReferenceError: repository = None if repository: try: items = filter(None, map(revparse, path.split(".."))) query = None if len(items) == 1: query = ("repository=%d&sha1=%s" % (repository.id, items[0])) elif len(items) == 2: query = ("repository=%d&from=%s&to=%s" % (repository.id, items[0], items[1])) if query: if req.query: query += "&" + req.query req.query = query req.path = "showcommit" continue except gitutils.GitReferenceError: pass break req.setStatus(404) raise page.utils.DisplayMessage(title="Not found!", body="Page not handled: /%s" % path) except GeneratorExit: raise except page.utils.NotModified: req.setStatus(304) req.start() return [] except request.MovedTemporarily as redirect: req.setStatus(307) req.addResponseHeader("Location", redirect.location) if redirect.no_cache: req.addResponseHeader("Cache-Control", "no-cache") req.start() return [] except request.DoExternalAuthentication as command: command.execute(db, req) return [] except request.MissingWSGIRemoteUser as error: # req object is not initialized yet. start_response("200 OK", [("Content-Type", "text/html")]) return [ """\ <pre>error: Critic was configured with '--auth-mode host' but there was no REMOTE_USER variable in the WSGI environ dict provided by the web server. To fix this you can either reinstall Critic using '--auth-mode critic' (to let Critic handle user authentication automatically), or you can configure user authentication properly in the web server. For apache2, the latter can be done by adding the something like the following to the apache site configuration for Critic: <Location /> AuthType Basic AuthName "Authentication Required" AuthUserFile "/path/to/critic-main.htpasswd.users" Require valid-user </Location> If you need more dynamic http authentication you can instead setup mod_wsgi with a custom WSGIAuthUserScript directive. This will cause the provided credentials to be passed to a Python function called check_password() that you can implement yourself. This way you can validate the user/pass via any existing database or for example an LDAP server. For more information on setting up such authentication in apache2, see: <a href="%(url)s">%(url)s</a></pre>""" % { "url": "http://code.google.com/p/modwsgi/wiki/AccessControlMechanisms#Apache_Authentication_Provider" } ] except page.utils.DisplayMessage as message: if user is None: user = dbutils.User.makeAnonymous() document = page.utils.displayMessage(db, req, user, title=message.title, message=message.body, review=message.review, is_html=message.html) req.setContentType("text/html") req.start() return [str(document)] except Exception: # crash might be psycopg2.ProgrammingError so rollback to avoid # "InternalError: current transaction is aborted" inside handleException() if db and db.closed(): db = None elif db: db.rollback() error_title, error_body = handleException(db, req, user) error_body = reflow("\n\n".join(error_body)) error_message = "\n".join( [error_title, "=" * len(error_title), "", error_body]) assert not req.isStarted() req.setStatus(500) req.setContentType("text/plain") req.start() return [error_message] finally: if db: db.close()