def _create(self, cursor):
"""Internal function to create a new stats target, meant to be run from
inside a database interaction. This is actually a recursive operation
that tries to create parent stats targets if necessary.
NOTE: this -must- ignore duplicate keys to avoid a race condition in which
one thread, in _autoCreateTargetFor, decides to create a new target
but before that target is fully created another thread also decides
it needs a new target.
"""
parent = self.parent()
if parent:
# If we have a parent, we have to worry about creating it
# if it doesn't exist and generating the proper parent path.
parent._autoCreateTargetFor(
cursor, cursor.execute,
"INSERT IGNORE INTO stats_catalog (parent_path, target_path) VALUES(%s, %s)"
% (Database.quote(parent.path, 'varchar'),
Database.quote(self.path, 'varchar')))
else:
# This is the root node. We still need to insert a parent to keep the
# table consistent, but our parent in this case is NULL.
cursor.execute(
"INSERT IGNORE INTO stats_catalog (target_path) VALUES(%s)" %
Database.quote(self.path, 'varchar'))
def _reinforce(self, cursor):
"""Database interaction implementing reinforce()"""
# First touch the edge to make sure it exists. We have to do this
# inside two autoCreateTargetFor levels, to create both stats targets
# if they don't yet exist.
self.a._autoCreateTargetFor(
cursor,
self.b._autoCreateTargetFor,
cursor,
cursor.execute,
"INSERT IGNORE INTO stats_relations "
"(target_a_path, target_b_path) VALUES(%s, %s)"
% (Database.quote(self.a.path, "varchar"), Database.quote(self.b.path, "varchar")),
)
cursor.execute(
"UPDATE stats_relations "
"SET strength = strength + 1, freshness = %s "
"WHERE target_a_path = %s AND target_b_path = %s"
% (
Database.quote(int(time.time()), "bigint"),
Database.quote(self.a.path, "varchar"),
Database.quote(self.b.path, "varchar"),
)
)
def initQuery(self):
self.query = self.query % dict(
path=Database.quote(self.targetPath, 'varchar'),
limit=self.numItems,
counter=Database.quote(self.counter, 'varchar'),
counter_attrib=self.counterAttrib,
sort=self.sort,
)
def notify(self, scope):
"""Notify all subscribers to this stats target of a change in 'scope'"""
# Get a list of applicable triggers from the database
Database.pool.runQuery("SELECT id, `trigger` FROM stats_subscriptions "
"WHERE target_path = %s "
"AND (scope is NULL or scope = %s)" %
(Database.quote(self.target.path, 'varchar'),
Database.quote(scope, 'varchar'))).addCallback(self.runTriggers)
def notify(self, scope):
"""Notify all subscribers to this stats target of a change in 'scope'"""
# Get a list of applicable triggers from the database
Database.pool.runQuery("SELECT id, `trigger` FROM stats_subscriptions "
"WHERE target_path = %s "
"AND (scope is NULL or scope = %s)" %
(Database.quote(self.target.path, 'varchar'),
Database.quote(scope, 'varchar'))).addCallback(self.runTriggers)
def initQuery(self):
self.query = self.query % dict(
path = Database.quote(self.targetPath, 'varchar'),
limit = self.numItems,
counter = Database.quote(self.counter, 'varchar'),
counter_attrib = self.counterAttrib,
sort = self.sort,
hint = self.hint,
)
def _createCounter(self, cursor, name):
"""Internal function to create one blank counter if it doesn't exist."""
try:
cursor.execute("INSERT INTO stats_counters (target_path, name) VALUES(%s, %s)" %
(Database.quote(self.target.path, 'varchar'),
Database.quote(name, 'varchar')))
except:
# Ignore duplicate key errors
if str(sys.exc_info()[1]).find("duplicate key") < 0:
raise
def _createCounter(self, cursor, name):
"""Internal function to create one blank counter if it doesn't exist."""
try:
cursor.execute("INSERT INTO stats_counters (target_path, name) VALUES(%s, %s)" %
(Database.quote(self.target.path, 'varchar'),
Database.quote(name, 'varchar')))
except:
# Ignore duplicate key errors
if str(sys.exc_info()[1]).find("duplicate key") < 0:
raise
def _triggerFailure(self, cursor, failure, id, maxFailures=3):
# Increment the consecutive failure count
log.msg("Failed to notify subscriber %d for %r: %r" % (id, self.target, failure))
cursor.execute("UPDATE stats_subscriptions SET failures = failures + 1 WHERE id = %s" %
Database.quote(id, 'bigint'))
# Cancel the subscription if we've had too many failures
cursor.execute("DELETE FROM stats_subscriptions WHERE id = %s AND failures > %s" %
(Database.quote(id, 'bigint'),
Database.quote(maxFailures, 'int')))
if cursor.rowcount:
log.msg("Unsubscribing subscriber %d for %r, more than %d consecutive failures" %
(id, self.target, maxFailures))
def _triggerFailure(self, cursor, failure, id, maxFailures=3):
# Increment the consecutive failure count
log.msg("Failed to notify subscriber %d for %r: %r" % (id, self.target, failure))
cursor.execute("UPDATE stats_subscriptions SET failures = failures + 1 WHERE id = %s" %
Database.quote(id, 'bigint'))
# Cancel the subscription if we've had too many failures
cursor.execute("DELETE FROM stats_subscriptions WHERE id = %s AND failures > %s" %
(Database.quote(id, 'bigint'),
Database.quote(maxFailures, 'int')))
if cursor.rowcount:
log.msg("Unsubscribing subscriber %d for %r, more than %d consecutive failures" %
(id, self.target, maxFailures))
def _runQuery(self, cursor, filter):
# Set up and run two SQL queries, one for each side of the graph link that this
# target may be on. We can't do this in one step and still have the server use
# its indexes effectively.
filterSql = RelatedFilter(filter).sql
sections = {}
for thisSide, otherSide in (('a', 'b'), ('b', 'a')):
cursor.execute(self.query % dict(
path=Database.quote(self.target.path, 'varchar'),
filter=filterSql,
thisSide=thisSide,
otherSide=otherSide,
))
while 1:
row = cursor.fetchone()
if not row:
break
# Drop rows into sections according to their parent path
try:
sections[row[0]].append(row[1:])
except KeyError:
sections[row[0]] = [row[1:]]
# Sort sections descending by freshness
for items in sections.itervalues():
items.sort()
items.reverse()
return sections
def _subscribe(self, cursor, target, client, trigger, scope=None, ttl=25*60*60):
"""A database interaction for adding subscriptions.
'target' must be the StatsTarget object this subscription is for.
'client' is the IP address of the client requesting this subscription.
'trigger' is a trigger pickle, as returned by makeTrigger
'scope' refers to a part of the stats target this refers to. By default, all of it.
'ttl' is the time to live for this subscription, 25 hours by default.
"""
cursor.execute("INSERT INTO stats_subscriptions "
"(target_path, expiration, scope, client, `trigger`) "
"VALUES (%s, %s, %s, %s, '%s')" %
(Database.quote(target.path, 'varchar'),
Database.quote(int(time.time() + ttl), 'bigint'),
Database.quote(scope, 'varchar'),
Database.quote(client, 'varchar'),
Database.quoteBlob(trigger)))
def _runQuery(self, cursor, filter):
# Set up and run two SQL queries, one for each side of the graph link that this
# target may be on. We can't do this in one step and still have the server use
# its indexes effectively.
filterSql = RelatedFilter(filter).sql
sections = {}
for thisSide, otherSide in ( ('a','b'), ('b', 'a') ):
cursor.execute(self.query % dict(
path = Database.quote(self.target.path, 'varchar'),
filter = filterSql,
thisSide = thisSide,
otherSide = otherSide,
))
while 1:
row = cursor.fetchone()
if not row:
break
# Drop rows into sections according to their parent path
try:
sections[row[0]].append(row[1:])
except KeyError:
sections[row[0]] = [row[1:]]
# Sort sections descending by freshness
for items in sections.itervalues():
items.sort()
items.reverse()
return sections
def clear(self):
"""Delete everything associated with this stats target. Returns a Deferred
indicating the completion of this operation.
"""
# Delete the item in stats_target- the other tables will be
# deleted due to cascading foreign keys
return Database.pool.runOperation("DELETE FROM stats_catalog WHERE target_path = %s" %
Database.quote(self.path, 'varchar'))
def clear(self):
"""Delete everything associated with this stats target. Returns a Deferred
indicating the completion of this operation.
"""
# Delete the item in stats_target- the other tables will be
# deleted due to cascading foreign keys
return Database.pool.runOperation("DELETE FROM stats_catalog WHERE target_path = %s" %
Database.quote(self.path, 'varchar'))
def _reinforce(self, cursor):
"""Database interaction implementing reinforce()"""
# First touch the edge to make sure it exists. We have to do this
# inside two autoCreateTargetFor levels, to create both stats targets
# if they don't yet exist.
self.a._autoCreateTargetFor(
cursor, self.b._autoCreateTargetFor, cursor, cursor.execute,
"INSERT IGNORE INTO stats_relations "
"(target_a_path, target_b_path) VALUES(%s, %s)" %
(Database.quote(self.a.path, 'varchar'),
Database.quote(self.b.path, 'varchar')))
cursor.execute("UPDATE stats_relations "
"SET strength = strength + 1, freshness = %s "
"WHERE target_a_path = %s AND target_b_path = %s" %
(Database.quote(int(time.time()), 'bigint'),
Database.quote(self.a.path, 'varchar'),
Database.quote(self.b.path, 'varchar')))
def render_rows(self, context):
photo_query = """
SELECT IM.path, IM.width, IM.height
FROM stats_statstarget ST
LEFT OUTER JOIN images_imageinstance IM
ON (IM.source_id = ST.photo_id AND IM.thumbnail_size = 256)
WHERE ST.path = %s
""" % Database.quote(
self.target.path, "varchar"
)
# XXX: This is hacky. Search for exclusive owners of this target.
owner_query = """
SELECT UA.id, UA.access, UA.user_id
FROM stats_statstarget ST
LEFT OUTER JOIN accounts_project PROJ ON (PROJ.target_id = ST.id)
LEFT OUTER JOIN accounts_author AUTH ON (AUTH.target_id = ST.id)
LEFT OUTER JOIN django_content_type CT_AUTH
ON (CT_AUTH.app_label = 'accounts' AND CT_AUTH.model = 'author')
LEFT OUTER JOIN django_content_type CT_PROJ
ON (CT_PROJ.app_label = 'accounts' AND CT_PROJ.model = 'project')
LEFT OUTER JOIN accounts_userasset UA
ON ( (UA.content_type_id = CT_AUTH.id AND UA.object_id = AUTH.id)
OR (UA.content_type_id = CT_PROJ.id AND UA.object_id = PROJ.id))
WHERE ST.path = %s AND UA.access > 1
""" % Database.quote(
self.target.path, "varchar"
)
# Grab the metadata keys we'll need and wait for them to become available
result = defer.Deferred()
defer.gatherResults(
[
self.metadata.getValue("url"),
self.metadata.getValue("description"),
Database.pool.runQuery(photo_query),
Database.pool.runQuery(owner_query),
]
).addCallback(self._render_rows, context, result).addErrback(result.errback)
return result
def _catalog(self, cursor):
"""Database interaction representing the internals of catalog()"""
cursor.execute("SELECT target_path FROM stats_catalog WHERE parent_path = %s" %
Database.quote(self.path, 'varchar'))
results = []
while True:
row = cursor.fetchone()
if row is None:
break
results.append(StatsTarget(row[0]))
return results
def _catalog(self, cursor):
"""Database interaction representing the internals of catalog()"""
cursor.execute("SELECT target_path FROM stats_catalog WHERE parent_path = %s" %
Database.quote(self.path, 'varchar'))
results = []
while True:
row = cursor.fetchone()
if row is None:
break
results.append(StatsTarget(row[0]))
return results
def render_rows(self, context):
# First we run a big SQL query to gather all the data for this catalog.
# Control is passed to _render_rows once we have the query results.
result = defer.Deferred()
Database.pool.runQuery(self.query % {
'path': Database.quote(self.target.path, 'varchar'),
'limit': self.limit,
}).addCallback(
self._render_rows, context, result
).addErrback(result.errback)
return result
def render_rows(self, context):
# First we run a big SQL query to gather all the data for this catalog.
# Control is passed to _render_rows once we have the query results.
result = defer.Deferred()
Database.pool.runQuery(
self.query % {
'path': Database.quote(self.target.path, 'varchar'),
'limit': self.limit,
}).addCallback(self._render_rows, context,
result).addErrback(result.errback)
return result
def checkOneRollover(self, cursor, previous, current):
"""Check for rollovers in one pair of consecutive time intervals,
like yesterday/today or lastMonth/thisMonth. This is meant to
be run inside a database interaction.
"""
# Delete counters that are too old to bother keeping at all
cursor.execute(
"DELETE FROM stats_counters "
"WHERE (name = %s OR name = %s) "
"AND first_time < %s" %
(Database.quote(previous,
'varchar'), Database.quote(current, 'varchar'),
Database.quote(
long(TimeUtil.Interval(previous).getFirstTimestamp()),
'bigint')))
# Roll over remaining counters that are too old for current
# but still within the range of previous. Note that there is a
# race condition in which this update will fail because a timer
# has been incremented between it and the above DELETE. It could
# be prevented by locking the table, but it's probably not worth
# it. If the rollover fails this time, it will get another chance.
cursor.execute(
"UPDATE stats_counters SET name = %s "
"WHERE name = %s "
"AND first_time < %s" %
(Database.quote(previous,
'varchar'), Database.quote(current, 'varchar'),
Database.quote(
long(TimeUtil.Interval(current).getFirstTimestamp()),
'bigint')))
def checkOneRollover(self, cursor, previous, current):
"""Check for rollovers in one pair of consecutive time intervals,
like yesterday/today or lastMonth/thisMonth. This is meant to
be run inside a database interaction.
"""
# Delete counters that are too old to bother keeping at all
cursor.execute("DELETE FROM stats_counters "
"WHERE (name = %s OR name = %s) "
"AND first_time < %s" %
(Database.quote(previous, 'varchar'),
Database.quote(current, 'varchar'),
Database.quote(long(TimeUtil.Interval(previous).getFirstTimestamp()), 'bigint')))
# Roll over remaining counters that are too old for current
# but still within the range of previous. Note that there is a
# race condition in which this update will fail because a timer
# has been incremented between it and the above DELETE. It could
# be prevented by locking the table, but it's probably not worth
# it. If the rollover fails this time, it will get another chance.
cursor.execute("UPDATE stats_counters SET name = %s "
"WHERE name = %s "
"AND first_time < %s" %
(Database.quote(previous, 'varchar'),
Database.quote(current, 'varchar'),
Database.quote(long(TimeUtil.Interval(current).getFirstTimestamp()), 'bigint')))
def render_photo(self, context):
# First figure out if we have a photo. Actually render it in the Deferred if we do.
photo_query = """
SELECT IM.path
FROM stats_statstarget ST
LEFT OUTER JOIN images_imageinstance IM
ON (IM.source_id = ST.photo_id AND IM.thumbnail_size = 128)
WHERE ST.path = %s
""" % Database.quote(self.target.path, 'varchar')
result = defer.Deferred()
Database.pool.runQuery(photo_query).addCallback(
self._render_photo, context, result).addErrback(result.errback)
return result
def _subscribe(self,
cursor,
target,
client,
trigger,
scope=None,
ttl=25 * 60 * 60):
"""A database interaction for adding subscriptions.
'target' must be the StatsTarget object this subscription is for.
'client' is the IP address of the client requesting this subscription.
'trigger' is a trigger pickle, as returned by makeTrigger
'scope' refers to a part of the stats target this refers to. By default, all of it.
'ttl' is the time to live for this subscription, 25 hours by default.
"""
cursor.execute(
"INSERT INTO stats_subscriptions "
"(target_path, expiration, scope, client, `trigger`) "
"VALUES (%s, %s, %s, %s, '%s')" %
(Database.quote(target.path, 'varchar'),
Database.quote(int(time.time() + ttl), 'bigint'),
Database.quote(scope, 'varchar'), Database.quote(
client, 'varchar'), Database.quoteBlob(trigger)))
def render_rows(self, context):
photo_query = """
SELECT IM.path, IM.width, IM.height
FROM stats_statstarget ST
LEFT OUTER JOIN images_imageinstance IM
ON (IM.source_id = ST.photo_id AND IM.thumbnail_size = 256)
WHERE ST.path = %s
""" % Database.quote(self.target.path, 'varchar')
# XXX: This is hacky. Search for exclusive owners of this target.
owner_query = """
SELECT UA.id, UA.access, UA.user_id
FROM stats_statstarget ST
LEFT OUTER JOIN accounts_project PROJ ON (PROJ.target_id = ST.id)
LEFT OUTER JOIN accounts_author AUTH ON (AUTH.target_id = ST.id)
LEFT OUTER JOIN django_content_type CT_AUTH
ON (CT_AUTH.app_label = 'accounts' AND CT_AUTH.model = 'author')
LEFT OUTER JOIN django_content_type CT_PROJ
ON (CT_PROJ.app_label = 'accounts' AND CT_PROJ.model = 'project')
LEFT OUTER JOIN accounts_userasset UA
ON ( (UA.content_type_id = CT_AUTH.id AND UA.object_id = AUTH.id)
OR (UA.content_type_id = CT_PROJ.id AND UA.object_id = PROJ.id))
WHERE ST.path = %s AND UA.access > 1
""" % Database.quote(self.target.path, 'varchar')
# Grab the metadata keys we'll need and wait for them to become available
result = defer.Deferred()
defer.gatherResults([
self.metadata.getValue('url'),
self.metadata.getValue('description'),
Database.pool.runQuery(photo_query),
Database.pool.runQuery(owner_query),
]).addCallback(self._render_rows, context, result).addErrback(result.errback)
return result
def _create(self, cursor):
"""Internal function to create a new stats target, meant to be run from
inside a database interaction. This is actually a recursive operation
that tries to create parent stats targets if necessary.
NOTE: this -must- ignore duplicate keys to avoid a race condition in which
one thread, in _autoCreateTargetFor, decides to create a new target
but before that target is fully created another thread also decides
it needs a new target.
"""
parent = self.parent()
if parent:
# If we have a parent, we have to worry about creating it
# if it doesn't exist and generating the proper parent path.
parent._autoCreateTargetFor(cursor, cursor.execute,
"INSERT IGNORE INTO stats_catalog (parent_path, target_path) VALUES(%s, %s)" %
(Database.quote(parent.path, 'varchar'),
Database.quote(self.path, 'varchar')))
else:
# This is the root node. We still need to insert a parent to keep the
# table consistent, but our parent in this case is NULL.
cursor.execute("INSERT IGNORE INTO stats_catalog (target_path) VALUES(%s)" %
Database.quote(self.path, 'varchar'))
def _updateCache(self, cursor):
"""Database interaction to update our counter cache"""
cursor.execute("SELECT name, first_time, last_time, event_count FROM stats_counters WHERE"
" target_path = %s" %
Database.quote(self.target.path, 'varchar'))
results = {}
while True:
row = cursor.fetchone()
if row is None:
break
results[row[0]] = {
'firstEventTime': row[1],
'lastEventTime': row[2],
'eventCount': row[3],
}
self.cache = results
def _updateCache(self, cursor):
"""Database interaction to update our counter cache"""
cursor.execute(
"SELECT name, first_time, last_time, event_count FROM stats_counters WHERE"
" target_path = %s" % Database.quote(self.target.path, 'varchar'))
results = {}
while True:
row = cursor.fetchone()
if row is None:
break
results[row[0]] = {
'firstEventTime': row[1],
'lastEventTime': row[2],
'eventCount': row[3],
}
self.cache = results
def _update_cache(self, cursor):
"""Retrieve all metadata keys for this target, populating self._cache."""
# XXX: This only works for text keys. Images can't be fully represented
# in the old system, so we're ignoring them until the transition is complete.
keys = ('title', 'subtitle', 'url', 'description', 'links-filter', 'related-filter')
# Our new column names use underscores instead of dashes
columns = [key.replace('-', '_') for key in keys]
cursor.execute("SELECT %s FROM stats_statstarget WHERE path = %s" % (
', '.join(columns),
Database.quote(self.target.path, 'varchar')))
row = cursor.fetchone()
self._cache = {}
if row:
for i, key in enumerate(keys):
self._cache[key] = row[i]
def _update_cache(self, cursor):
"""Retrieve all metadata keys for this target, populating self._cache."""
# XXX: This only works for text keys. Images can't be fully represented
# in the old system, so we're ignoring them until the transition is complete.
keys = ('title', 'subtitle', 'url', 'description', 'links-filter',
'related-filter')
# Our new column names use underscores instead of dashes
columns = [key.replace('-', '_') for key in keys]
cursor.execute(
"SELECT %s FROM stats_statstarget WHERE path = %s" %
(', '.join(columns), Database.quote(self.target.path, 'varchar')))
row = cursor.fetchone()
self._cache = {}
if row:
for i, key in enumerate(keys):
self._cache[key] = row[i]
def _incrementCounter(self, cursor, name):
"""Increment one counter, creating it if necessary"""
now = int(time.time())
# Invalidate the cache for this counter if we have one
self.cache = None
# Insert a default value, which will be ignored if the counter already exists
cursor.execute("INSERT IGNORE INTO stats_counters (target_path, name, first_time) VALUES(%s, %s, %s)" %
(Database.quote(self.target.path, 'varchar'),
Database.quote(name, 'varchar'),
Database.quote(now, 'bigint')))
# Increment the counter and update its timestamp
cursor.execute("UPDATE stats_counters SET "
"event_count = event_count + 1,"
"last_time = %s "
"WHERE target_path = %s AND name = %s" %
(Database.quote(now, 'bigint'),
Database.quote(self.target.path, 'varchar'),
Database.quote(name, 'varchar')))
def _incrementCounter(self, cursor, name):
"""Increment one counter, creating it if necessary"""
now = int(time.time())
# Invalidate the cache for this counter if we have one
self.cache = None
# Insert a default value, which will be ignored if the counter already exists
cursor.execute(
"INSERT IGNORE INTO stats_counters (target_path, name, first_time) VALUES(%s, %s, %s)"
% (Database.quote(self.target.path, 'varchar'),
Database.quote(name, 'varchar'), Database.quote(now, 'bigint')))
# Increment the counter and update its timestamp
cursor.execute("UPDATE stats_counters SET "
"event_count = event_count + 1,"
"last_time = %s "
"WHERE target_path = %s AND name = %s" %
(Database.quote(now, 'bigint'),
Database.quote(self.target.path, 'varchar'),
Database.quote(name, 'varchar')))
def pruneSubscriptions(self, cursor, maxFailures=3):
"""Delete subscriptions that have expired"""
cursor.execute("DELETE FROM stats_subscriptions WHERE expiration < %s" %
Database.quote(int(time.time()), 'bigint'))
# sized without stats_messages.
#
# If the conversion takes a while and you want to pick
# up the commits you missed while the conversion was taking
# place, note the ID of the last converted message and add
# a "WHERE id > foo" clause to the stats_messages query.
#
# -- Micah Dowty
#
import sys, os
sys.path[0] = os.path.join(sys.path[0], "..")
from LibCIA import Database, XML
from LibCIA.Stats.Target import StatsTarget
Database.init(serverCursor=True)
cursor = Database.pool.connect().cursor()
# Make sure we're starting with version 3
cursor.execute("SELECT value FROM meta WHERE name = 'version'")
if cursor.fetchall()[0][0] != "6":
raise Exception("This script must only be run on version 6 databases")
# To avoid spending all our time reading buffer headers, cache frequently used targets
targetCache = {}
targetHits = {}
targetCacheMax = 128
rowsProcessed = 0
prevId = 0
DO NOT use this on a server with important data, as
the database is reinitialized before each trial! Don't
use it on a server running twistd for other purposes,
or an important server of any kind. This program
may eat your pets and cause your toaster to impersonate
former US senators.
usage: regression.py first-rev last-rev output.csv
"""
import sys, os, time, re, xmlrpclib, shutil
import RandomMessage
sys.path[0] = os.path.join(sys.path[0], "..")
from LibCIA import Database
Database.init({'db': None})
dbcursor = Database.pool.connect().cursor()
def readStatements(filename):
"""Return a sequence of SQL statements from the given file"""
lines = []
for line in open(filename).xreadlines():
line = line.strip()
if not line.startswith("--"):
lines.append(line)
fulltext = " ".join(lines)
for statement in fulltext.split(";"):
statement = statement.strip()
if statement:
yield statement
DO NOT use this on a server with important data, as
the database is reinitialized before each trial! Don't
use it on a server running twistd for other purposes,
or an important server of any kind. This program
may eat your pets and cause your toaster to impersonate
former US senators.
usage: regression.py first-rev last-rev output.csv
"""
import sys, os, time, re, xmlrpclib, shutil
import RandomMessage
sys.path[0] = os.path.join(sys.path[0], "..")
from LibCIA import Database
Database.init({'db': None})
dbcursor = Database.pool.connect().cursor()
def readStatements(filename):
"""Return a sequence of SQL statements from the given file"""
lines = []
for line in open(filename).xreadlines():
line = line.strip()
if not line.startswith("--"):
lines.append(line)
fulltext = " ".join(lines)
for statement in fulltext.split(";"):
statement = statement.strip()
if statement:
yield statement
def triggerSuccess(self, result, id):
"""Record a successful trigger run for the given subscription id"""
# Zero the consecutive failure count
Database.pool.runOperation(
"UPDATE stats_subscriptions SET failures = 0 WHERE id = %s" %
Database.quote(id, 'bigint'))
def triggerSuccess(self, result, id):
"""Record a successful trigger run for the given subscription id"""
# Zero the consecutive failure count
Database.pool.runOperation("UPDATE stats_subscriptions SET failures = 0 WHERE id = %s" %
Database.quote(id, 'bigint'))
#
# If the conversion takes a while and you want to pick
# up the commits you missed while the conversion was taking
# place, note the ID of the last converted message and add
# a "WHERE id > foo" clause to the stats_messages query.
#
# -- Micah Dowty
#
import sys, os
sys.path[0] = os.path.join(sys.path[0], "..")
from LibCIA import Database, XML
from LibCIA.Stats.Target import StatsTarget
Database.init(serverCursor=True)
cursor = Database.pool.connect().cursor()
# Make sure we're starting with version 3
cursor.execute("SELECT value FROM meta WHERE name = 'version'")
if cursor.fetchall()[0][0] != "6":
raise Exception("This script must only be run on version 6 databases")
# To avoid spending all our time reading buffer headers, cache frequently used targets
targetCache = {}
targetHits = {}
targetCacheMax = 128
rowsProcessed = 0
prevId = 0
#
# Upgrades a CIA database from version 3 to version 4.
#
# This script must migrate security data to the new format.
# Back up your database before running this, as failure to
# finish could cause all security data to be lost!
#
# -- Micah Dowty
#
import sys, os, time, md5
sys.path[0] = os.path.join(sys.path[0], "..")
from LibCIA import Database
Database.init()
cursor = Database.pool.connect().cursor()
# Make sure we're starting with version 3
cursor.execute("SELECT value FROM meta WHERE name = 'version'")
if cursor.fetchone()[0] != "3":
raise Exception("This script must only be run on version 3 databases")
# Read in all security data from the old capabilities table...
# Each key gets a list of capabilities, but only one owner.
cursor.execute("SELECT * FROM capabilities")
key_owner = {}
key_ownerMail = {}
key_capabilities = {}
while True:
row = cursor.fetchone()
if row:
def pruneSubscriptions(self, cursor, maxFailures=3):
"""Delete subscriptions that have expired"""
cursor.execute(
"DELETE FROM stats_subscriptions WHERE expiration < %s" %
Database.quote(int(time.time()), 'bigint'))
def _unsubscribe(self, cursor, client):
"""Remove all subscriptions for the given client.
This is intended to be run inside a database interaction.
"""
cursor.execute("DELETE FROM stats_subscriptions WHERE client = %s" %
Database.quote(client, 'varchar'))
def clear(self):
"""Delete all counters for this target. Returns a Deferred"""
self.cache = {}
return Database.pool.runOperation(
"DELETE FROM stats_counters WHERE target_path = %s" %
Database.quote(self.target.path, 'varchar'))
def _unsubscribe(self, cursor, client):
"""Remove all subscriptions for the given client.
This is intended to be run inside a database interaction.
"""
cursor.execute("DELETE FROM stats_subscriptions WHERE client = %s" %
Database.quote(client, 'varchar'))
def clear(self):
"""Delete all counters for this target. Returns a Deferred"""
self.cache = {}
return Database.pool.runOperation("DELETE FROM stats_counters WHERE target_path = %s" %
Database.quote(self.target.path, 'varchar'))
