def main():
logger = logging.getLogger("topcrashes_summary")
logger.setLevel(logging.WARNING)
stderrLog = logging.StreamHandler()
stderrLog.setLevel(logging.WARNING)
stderrLogFormatter = logging.Formatter('%(asctime)s %(levelname)s - %(message)s')
stderrLog.setFormatter(stderrLogFormatter)
logger.addHandler(stderrLog)
kwargs = {}
for i in sys.argv[1:]:
if i.startswith('-h') or i.startswith('--he'):
help()
sys.exit(0)
j = i
if i.startswith('-'):
j = i.lstrip('-')
if '=' in j:
name,value = (s.strip() for s in j.split('='))
kwargs[name] = value
else:
print >> sys.stderr,"Ignoring unkown argument '%s'"%(i)
sys.argv = sys.argv[:1]
config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Create Database')
config.update(kwargs)
testDB = TestDB()
testDB.removeDB(config,logger)
testDB.createDB(config,logger)
def setUp(self):
global me
if not me:
createMe()
self.testDB = TestDB()
self.testDB.removeDB(me.config, me.logger)
self.testDB.createDB(me.config, me.logger)
#self.connection = psycopg2.connect(me.dsn)
self.connection = me.database.connection()
def createData(config,logger):
# Now do the work in several steps
connection = sdatabase.Database(config).connection()
#connection = psycopg2.connect("host=%(databaseHost)s dbname=%(databaseName)s user=%(databaseUserName)s password=%(databasePassword)s"%config)
cursor = connection.cursor()
testDB = TestDB()
try:
testDB.removeDB(config,logger)
if config.get('drop-all'):
print "Dropped the database tables ..."
return
print "Creating the database tables..."
testDB.createDB(config,logger)
print "populating the dimensions tables..."
processingDays,ignore = dbtestutil.fillMtbfTables(cursor,limit=int(config.get('product-os-count')))
startDays = [x[0] for x in processingDays]
multiplier = int(config.get('repeat-count'))
print "populating the reports table (takes about %d seconds)..."%(int(1.7+1.2*multiplier))
dbtestutil.fillReportsTable(cursor,createUrls=True,doFillMtbfTables=False, numUrls=100, multiplier=multiplier,signatureCount=int(config.get('signature-count')))
connection.commit()
extras = []
print "All done populating the 'raw' data"
if config.get('mtbf-fill'):
blurb = ""
cost = 0.20 + multiplier*0.15
if cost > 1.0:
blurb = ("(takes about %2.1f seconds)"%cost)
print "Filling the time_before_failure table %s..."%blurb # R=1: .35 seconds; 2:0.49s; 3:.064s; 4:.9 ## = .20 + R*.15
starter = None
ender = None
mtbfInstance = mtbf.Mtbf(config,logger)
for startDay in startDays:
if not starter: starter = startDay
ender = startDay
mtbfInstance.processOneMtbfWindow(processingDay=startDay)
extras.append(" - Time before fail: for days in %s through %s"%(starter,ender))
if config.get('sig-fill'):
print "Filling the top_crashes_by_signature table (takes about %s seconds)..."%(20+11*multiplier) # R=1: 27.3 secs; 2:38.5s; 3=48.3 ## = 20 +R*11
tc = topcrasher.TopCrashesBySignature(config)
tc.processDateInterval(startDate=startDays[0],endDate = startDays[-1])
extras.append(" - Top crash by sig: for days in %s through %s"%(startDays[0],startDays[-1]))
if config.get('url-fill'):
print "Filling the top_crashes_by_url table (takes about %s seconds)..."%(4+multiplier*2) # R=1: 4 secs; 2: 5s, 3: 7.6 ## = 4+R*2
logger.info("Filling the top_crashes_by_url table (takes about %s seconds)..."%(4+multiplier*2))
tu = topcrashbyurl.TopCrashesByUrl(config)
tu.processDateInterval(startDate=startDays[0],endDate = startDays[-1])
extras.append(" - Top crash by url: for days in %s through %s"%(startDays[0],startDays[-1]))
print "DONE populating the database tables"
if extras:
print "\n".join(extras)
finally:
logger.info("All done. Closing connection")
connection.close()
def setup_module():
global me
if me:
return
me = Me()
me.testDB = TestDB()
me.config = configurationManager.newConfiguration(
configurationModule=testConfig,
applicationName='TestingCachedIdAccess')
myDir = os.path.split(__file__)[0]
if not myDir: myDir = '.'
replDict = {'testDir': '%s' % myDir}
for i in me.config:
try:
me.config[i] = me.config.get(i) % (replDict)
except:
pass
cia.logger.setLevel(logging.DEBUG)
me.logFilePathname = me.config.logFilePathname
logfileDir = os.path.split(me.config.logFilePathname)[0]
try:
os.makedirs(logfileDir)
except OSError, x:
if errno.EEXIST != x.errno: raise
f = open(me.config.logFilePathname, 'w')
f.close()
def setUp(self): global me self.logger = me.logger #self.connection = psycopg2.connect(me.dsn) self.connection = me.database.connection() self.testDB = TestDB() self.testDB.removeDB(me.config,self.logger) self.testDB.createDB(me.config,self.logger)
def setUp(self):
global me
if not me:
createMe()
self.connection = me.database.connection()
#self.connection = psycopg2.connect(me.dsn)
self.testDB = TestDB()
self.testDB.removeDB(me.config, me.logger)
self.testDB.createDB(me.config, me.logger)
dbtestutil.fillDimsTables(self.connection.cursor())
self.connection.commit()
def main():
logger = logging.getLogger("topcrashes_summary")
logger.setLevel(logging.WARNING)
stderrLog = logging.StreamHandler()
stderrLog.setLevel(logging.WARNING)
stderrLogFormatter = logging.Formatter(
'%(asctime)s %(levelname)s - %(message)s')
stderrLog.setFormatter(stderrLogFormatter)
logger.addHandler(stderrLog)
kwargs = {}
for i in sys.argv[1:]:
if i.startswith('-h') or i.startswith('--he'):
help()
sys.exit(0)
j = i
if i.startswith('-'):
j = i.lstrip('-')
if '=' in j:
name, value = (s.strip() for s in j.split('='))
kwargs[name] = value
else:
print >> sys.stderr, "Ignoring unkown argument '%s'" % (i)
sys.argv = sys.argv[:1]
config = configurationManager.newConfiguration(
configurationModule=testConfig, applicationName='Create Database')
config.update(kwargs)
testDB = TestDB()
testDB.removeDB(config, logger)
testDB.createDB(config, logger)
def setUp(self):
global me
# config gets messed up by some tests. Use this one by preference
self.config = configurationManager.newConfiguration(
configurationModule=testConfig,
applicationName='Testing Postgresql Utils')
for i in self.config:
try:
self.config[i] = self.config.get(i) % (replDict)
except:
pass
self.connection = psycopg2.connect(me.dsn)
self.testDB = TestDB()
def setUp(self):
global me
self.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing MTBF')
myDir = os.path.split(__file__)[0]
if not myDir: myDir = '.'
replDict = {'testDir':'%s'%myDir}
for i in self.config:
try:
self.config[i] = self.config.get(i)%(replDict)
except:
pass
self.logger = TestingLogger(me.fileLogger)
self.connection = psycopg2.connect(me.dsn)
cursor = self.connection.cursor()
self.testDB = TestDB()
self.testDB.removeDB(self.config,self.logger)
self.testDB.createDB(self.config,self.logger)
self.prods = ['zorro','vogel','lizz',]
self.oss = ['OSX','LOX','WOX',]
self.productDimData = [] # filled in by fillMtbfTables
class TestTopCrashesBySignature(unittest.TestCase):
def setUp(self):
global me
if not me:
createMe()
self.connection = me.database.connection()
#self.connection = psycopg2.connect(me.dsn)
self.testDB = TestDB()
self.testDB.removeDB(me.config, me.logger)
self.testDB.createDB(me.config, me.logger)
dbtestutil.fillDimsTables(self.connection.cursor())
self.connection.commit()
def tearDown(self):
global me
self.testDB.removeDB(me.config,me.logger)
cia.clearCache()
self.connection.close()
def prepareConfigForPeriod(self,idkeys,startDate,endDate):
"""enter a row for each idkey, start and end date. Do it repeatedly if you need different ones"""
cfgKeys = set([x[0] for x in idkeys])
cfgdata = [[x,startDate,endDate] for x in cfgKeys]
self.connection.cursor().execute("delete from product_visibility")
sql = "INSERT INTO product_visibility (productdims_id,start_date,end_date) VALUES (%s,%s,%s)"
self.connection.cursor().executemany(sql,cfgdata)
self.connection.commit()
def prepareExtractDataForPeriod(self, columnName, sizePerDay=2, numDays=5):
"""Put some crash data into reports table, return their extrema for the columnName specified"""
global me
cursor = self.connection.cursor()
data = makeReportData(sizePerDay,numDays)
self.connection.commit()
# find the actual min and max dates
minStamp = dt.datetime(2100,11,11)
maxStamp = dt.datetime(1900,11,11)
for d in data:
ds = d[columnName]
if ds < minStamp: minStamp = ds
if ds > maxStamp: maxStamp = ds
addReportData(cursor,data)
self.connection.commit()
return minStamp,maxStamp,data
def testConstructor(self):
"""
TestTopCrashesBySignature.testConstructor
"""
global me
for reqd in ["databaseHost","databaseName","databaseUserName","databasePassword",]:
cc = copy.copy(me.config)
del(cc[reqd])
assert_raises(SystemExit,topcrasher.TopCrashesBySignature,cc)
bogusMap = {'databaseHost':me.config.databaseHost,'databaseName':me.config.databaseName,'databaseUserName':'******','databasePassword':me.config.databasePassword}
assert_raises(SystemExit,topcrasher.TopCrashesBySignature,bogusMap)
#check without specific config items
now = dt.datetime.now()
tc = topcrasher.TopCrashesBySignature(me.config)
expectedStart = now.replace(hour=0,minute=0,second=0,microsecond=0) - tc.configContext.get('initialDeltaDate',dt.timedelta(days=4))
expectedEnd = now.replace(hour=0,minute=0,second=0,microsecond=0)
assert expectedStart == tc.startDate
assert tc.endDate <= now, 'But NOT %s <= %s'%(tc.endDate,now)
assert dt.timedelta(days=4) <= tc.deltaDate, 'But got %s'%(tc.deltaDate)
# Check for defaults. If the defaults are hard-code changed, this block needs equivalent changes
assert 'date_processed' == tc.dateColumnName
# end Check for defaults
# check with solo startDate, endDate or deltaDate
config = copy.copy(me.config)
config['startDate'] = dt.datetime(2009,01,01,0,0,1)
assert_raises(SystemExit,topcrasher.TopCrashesBySignature,config)
config = copy.copy(me.config)
config['endDate'] = dt.datetime(2009,01,01,0,0,1)
assert_raises(SystemExit,topcrasher.TopCrashesBySignature,config)
config = copy.copy(me.config)
config['deltaDate'] = dt.timedelta(days=3)
assert_raises(SystemExit,topcrasher.TopCrashesBySignature,config)
def testExtractDataForPeriod_ByDateProcessed(self):
"""
TestTopCrashesBySignature.testExtractDataForPeriod_ByDateProcessed(self):
- Check that we get nothing for a period outside our data
- Check that we get expected count for period inside our data
- Check that we get half-open interval (don't get data exactly at last moment)
- Check that we get all for period that surrounds our data
- Check that we get nothing for start == end
- Check that we get one moment for end = start+microsecond
- Check that we raise ValueError for period with dates reversed
"""
global me
minStamp,maxStamp,data = self.prepareExtractDataForPeriod('date_processed')
cursor = self.connection.cursor()
idCache = cia.IdCache(cursor)
keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data])
mminStamp = minStamp - dt.timedelta(days=1)
mmaxStamp = maxStamp + dt.timedelta(days=1)
self.prepareConfigForPeriod(keySet,mminStamp,mmaxStamp)
tc = topcrasher.TopCrashesBySignature(me.config)
tc.dateColumnName = 'date_processed'
# we should get nothing if we are outside our data
start = minStamp - dt.timedelta(days=10)
end = start + dt.timedelta(days=1)
crashData = {}
tc.extractDataForPeriod(start,end,crashData)
assert {} == crashData,'Expected {}, got %s'%crashData
# We should get exactly one day's worth
start = minStamp
end = start + dt.timedelta(days=1)
tc.extractDataForPeriod(start,end,crashData)
crashDataCount = 0 # all three known
for k,d in crashData.items():
crashDataCount += d['count']
assert 2 == crashDataCount, 'Expected 2, got %d'%crashDataCount
# we should get all but the last 2 (or one) item: Half-open interval
start = minStamp
end = maxStamp
crashData = {}
tc.extractDataForPeriod(start,end,crashData)
crashDataCount = 0 # all three known
for k,d in crashData.items():
crashDataCount += d['count']
assert crashDataCount == 9, 'Expected 9. Got %s'%crashDataCount
# we should get all the data
start = minStamp
end = maxStamp+dt.timedelta(milliseconds=1)
crashData = {}
tc.extractDataForPeriod(start,end,crashData)
crashDataCount = 0 # all three known
for k,d in crashData.items():
crashDataCount += d['count']
assert crashDataCount == 10, 'Expected 10. Got %s'%crashDataCount
# we should get nothing if start == end (half open interval that stops juuuust before the start)
start = minStamp
end = minStamp
crashData = {}
tc.extractDataForPeriod(start,end,crashData)
assert crashData == {}, 'But got %s'%crashData
# we should get precise stamp's data if end = start+teeny-tiny-epsilon
start = minStamp
end = start+dt.timedelta(microseconds=1)
crashData = {}
tc.extractDataForPeriod(start,end,crashData)
crashDataCount = 0 # all three known
for k,d in crashData.items():
crashDataCount += d['count']
assert crashDataCount == 1, 'Expected 1. Got %s'%crashDataCount
# We should throw if start > end
assert_raises(ValueError,tc.extractDataForPeriod,maxStamp,minStamp,crashData)
def testExtractDataForPeriod_ByClientCrashDate(self):
"""
TestTopCrashesBySignature.testExtractDataForPeriod_ByClientCrashDate(self):
- Check that we get nothing for a period outside our data
- Check that we get expected count for period inside our data
- Check that we get half-open interval (don't get data exactly at last moment)
- Check that we get all for period that surrounds our data
- Check that we raise ValueError for period with dates reversed
"""
global me
minStamp,maxStamp,data = self.prepareExtractDataForPeriod('client_crash_date')
cursor = self.connection.cursor()
idCache = cia.IdCache(cursor)
keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data])
mminStamp = minStamp - dt.timedelta(days=1)
mmaxStamp = maxStamp + dt.timedelta(days=1)
self.prepareConfigForPeriod(keySet,mminStamp,mmaxStamp)
tc = topcrasher.TopCrashesBySignature(me.config)
tc.dateColumnName = 'client_crash_date'
# we should get nothing if we are outside our data
start = minStamp - dt.timedelta(days=10)
end = start + dt.timedelta(days=1)
crashData = {}
tc.extractDataForPeriod(start,end,crashData)
assert {} == crashData,'Expected {}, got %s'%crashData
# We should get exactly one day's worth
start = minStamp
end = start + dt.timedelta(days=1)
tc.extractDataForPeriod(start,end,crashData)
crashDataCount0 = 0 # all three known
for k,d in crashData.items():
crashDataCount0 += d['count']
assert 2 == crashDataCount0, 'Expected 2, got %d'%crashDataCount0
# we should get all but the last 2 (or one) item: Half-open interval
start = minStamp
end = maxStamp
crashData = {}
tc.extractDataForPeriod(start,end,crashData)
crashDataCount0 = 0 # all three known
for k,d in crashData.items():
crashDataCount0 += d['count']
assert crashDataCount0 == 9, 'Expected 9. Got %s'%crashDataCount0
# we should get all the data
start = minStamp
end = maxStamp+dt.timedelta(milliseconds=1)
crashData = {}
tc.extractDataForPeriod(start,end,crashData)
crashDataCount0 = 0 # all three known
for k,d in crashData.items():
crashDataCount0 += d['count']
assert crashDataCount0 == 10, 'Expected 10. Got %s'%crashDataCount0
# we should get nothing if start == end (half open interval that stops juuuust before the start)
start = minStamp
end = minStamp
crashData = {}
tc.extractDataForPeriod(start,end,crashData)
assert crashData == {}, 'But got %s'%crashData
# we should get precise stamp's data if end = start+teeny-tiny-epsilon
start = minStamp
end = start+dt.timedelta(microseconds=1)
crashData = {}
tc.extractDataForPeriod(start,end,crashData)
crashDataCount0 = 0 # all three known
for k,d in crashData.items():
crashDataCount0 += d['count']
assert crashDataCount0 == 1, 'Expected 1. Got %s'%crashDataCount0
# We should throw if start > end
assert_raises(ValueError,tc.extractDataForPeriod,maxStamp,minStamp,crashData)
def testFixupCrashData(self):
"""
TestTopCrashesBySignature.testFixupCrashData(self):(slow=1)
- create a bunch of data, count it in the test, put it into the reports table, then
let TopCrashesBySignature get it back out, and assert that fixup gets the same count
"""
global me
cursor = self.connection.cursor()
idCache = cia.IdCache(cursor)
data = makeReportData(211,5) # 211 is prime, which means we should get a nice spread of data
keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data])
#keySet = set([(d['productdims_id'],d['osdims_id']) for d in data])
expect = {}
newCount = 0
oldCount = 0
for d in data:
# Keys are always signature,product,os
key = (d['signature'],idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version']))
if d['version'][-3:] == 'pre': continue # Lars buildid
hang_count = 0
plugin_count = 0
if d['hangid'] != '':
hang_count = 1
if d['process_type'] == 'plugin':
plugin_count = 1;
try:
expect[key]['count'] += 1
expect[key]['uptime'] += d['uptime']
expect[key]['hang_count'] += hang_count
expect[key]['plugin_count'] += plugin_count
oldCount += 1
except:
expect[key] = {}
expect[key]['count'] = 1
expect[key]['uptime'] = d['uptime']
expect[key]['hang_count'] = hang_count
expect[key]['plugin_count'] = plugin_count
newCount += 1
addReportData(cursor,data)
self.connection.commit()
tc = topcrasher.TopCrashesBySignature(me.config)
tc.dateColumnName = 'date_processed'
baseDate = testBaseDate
start = baseDate - dt.timedelta(days=1)
end = baseDate + dt.timedelta(days=6)
self.prepareConfigForPeriod(keySet,start,end)
summaryCrashes = {}
tc.extractDataForPeriod(start,end,summaryCrashes)
# assert expect == summaryCrashes, 'Oops. You will need to debug on this. Sorry. Try commenting this line and un-commenting the next bunch'
# the next few lines amount to "assert expect == summaryCrashes" but with more useful error detail
# check that we have a chance of equality
assert len(expect) == len(summaryCrashes), 'Expected equal lengths but expected:%s versus got:%s'%(len(expect),len(summaryCrashes))
getter = itemgetter('count') # ,'uptime') #,'hang_count','plugin_count')
# check that everything in what we got is expected
for k,v in summaryCrashes.items():
assert getter(expect[k]) == getter(v), 'for key %s: Expected %s but got %s'%(k,getter(expect[k]),getter(v))
# check that everything we expect is in what we got
for k,v in expect.items():
assert getter(v) == getter(summaryCrashes[k]), 'for key %s: Expected %s but got %s'%(k,getter(v), getter(summaryCrashes[k]))
result = tc.fixupCrashData(summaryCrashes,baseDate,dt.timedelta(minutes=19))
result.sort(key=itemgetter('count'),reverse=True)
resultx = tc.fixupCrashData(expect,baseDate,dt.timedelta(minutes=19))
resultx.sort(key=itemgetter('count'),reverse=True)
assert result == resultx
## The next few lines show more detail if there's a problem. Comment above, uncomment below
# misscount = fullcount = 0
# if result != resultx:
# assert len(result) == len(resultx), 'expected len: %s, got %s'%(len(result), len(resultx))
# for i in range(len(result)):
# r = result[i]
# rx = resultx[i]
# print "%s" % (r)
# print "%s" % (rx)
# assert len(r) == len(rx), 'at loop %s: expected len: %s got %s'(i,len(r),len(rx))
# for j in range(len(r)):
# fullcount += 1
# if r[j] != rx[j]:
# print "unequal at loop %s, item %s:\n %s\n %s"%(i,j,r[j],rx[j])
# misscount += 1
# print "Unequal for %s of %s (%3.1f%%) rows"%(misscount,fullcount,100.0*(1.0*misscount/fullcount))
# assert 0 == misscount, "They weren't equal. Look nearby for details"
expectedAll = set(['xpcom_core.dll@0x31b7a', 'js_Interpret', '_PR_MD_SEND', 'nsFormFillController::SetPopupOpen', 'nsAutoCompleteController::ClosePopup','morkRowObject::CloseRowObject(morkEnv*)', 'nsContainerFrame::ReflowChild(nsIFrame*, nsPresContext*, nsHTMLReflowMetrics&, nsHTMLReflowState const&, int, int, unsigned int, unsigned int&, nsOverflowContinuationTracker*)'])
gotAll = set([ x['signature'] for x in result])
assert expectedAll == gotAll , 'ex-got:%s\ngot-ex:%s'%(expectedAll-gotAll,gotAll-expectedAll)
def testExtractDataForPeriod_ConfigLimitedDates(self):
"""
TestTopCrashesBySignature.testExtractDataForPeriod_ConfigLimitedDates(self)
"""
global me
minStamp,maxStamp,data = self.prepareExtractDataForPeriod('date_processed',23,5)
cursor = self.connection.cursor()
idCache = cia.IdCache(cursor)
keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data])
keySSet = set([x for x in keySet if 5 != x[0]])
mminStamp = minStamp - dt.timedelta(days=1)
mmaxStamp = maxStamp + dt.timedelta(days=1)
self.prepareConfigForPeriod(keySet,mminStamp,mmaxStamp)
tc = topcrasher.TopCrashesBySignature(me.config)
tc.dateColumnName = 'date_processed'
summaryData = {}
tc.extractDataForPeriod(minStamp,maxStamp,summaryData)
#assert 110 == len(summaryData), 'Regression test only, value not calculated. Expect 110, got %s'%(len(summaryData)) #Lars buildid
assert 99 == len(summaryData), 'Regression test only, value not calculated. Expect 99, got %s'%(len(summaryData))
cursor.execute("DELETE from product_visibility")
self.connection.commit()
mminStamp = minStamp + dt.timedelta(days=1)
mmaxStamp = maxStamp + dt.timedelta(days=1)
self.prepareConfigForPeriod(keySet,mminStamp,mmaxStamp)
summaryData = {}
tc.extractDataForPeriod(minStamp,maxStamp,summaryData)
#assert 87 == len(summaryData), 'Regression test only, value not calculated. Expect 87, got %s'%(len(summaryData)) #Lars buildid
assert 78 == len(summaryData), 'Regression test only, value not calculated. Expect 78, got %s'%(len(summaryData))
def testExtractDataForPeriod_ConfigLimitedIds(self):
"""
TestTopCrashesBySignature.testExtractDataForPeriod_ConfigLimitedIds
"""
global me
cursor = self.connection.cursor()
idCache = cia.IdCache(cursor)
minStamp,maxStamp,data = self.prepareExtractDataForPeriod('date_processed',23,5)
keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data])
keySSet = set([x for x in keySet if 5 != x[0]])
mminStamp = minStamp - dt.timedelta(days=1)
mmaxStamp = maxStamp + dt.timedelta(days=1)
self.prepareConfigForPeriod(keySet,mminStamp,mmaxStamp)
tc = topcrasher.TopCrashesBySignature(me.config)
tc.dateColumnName = 'date_processed'
summaryData = {}
tc.extractDataForPeriod(minStamp,maxStamp,summaryData)
#assert 110 == len(summaryData), 'Regression test only, value not calculated. Expect 110, got %s'%(len(summaryData)) #Lars buildid
assert 99 == len(summaryData), 'Regression test only, value not calculated. Expect 99, got %s'%(len(summaryData))
cursor.execute("DELETE from product_visibility")
self.connection.commit()
self.prepareConfigForPeriod(keySSet,mminStamp,mmaxStamp)
summaryData = {}
tc.extractDataForPeriod(minStamp,maxStamp,summaryData)
#assert 96 == len(summaryData), 'Regression test only, value not calculated. Expect 96, got %s'%(len(summaryData)) #Lars buildid
assert 85 == len(summaryData), 'Regression test only, value not calculated. Expect 85, got %s'%(len(summaryData))
def testStoreFacts(self):
"""
TestTopCrashesBySignature.testStoreFacts
"""
global me
cursor = self.connection.cursor()
idCache = cia.IdCache(cursor)
minStamp, maxStamp, data = self.prepareExtractDataForPeriod('date_processed',31,5) # full set of keys
keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data])
tc = topcrasher.TopCrashesBySignature(me.config)
tc.dateColumnName = 'date_processed'
beforeDate = minStamp-dt.timedelta(minutes=30)
afterDate = maxStamp+dt.timedelta(minutes=60)
self.prepareConfigForPeriod(keySet,beforeDate,afterDate)
summaryCrashes = {}
summaryCrashes = tc.extractDataForPeriod(beforeDate,afterDate,summaryCrashes)
countSql = "SELECT count(*) from top_crashes_by_signature"
cursor.execute(countSql)
self.connection.commit()
gotCount = cursor.fetchone()[0]
assert 0 == gotCount, 'but got a count of %s'%gotCount
#Try with none
tc.storeFacts([],"the test interval")
cursor.execute(countSql)
self.connection.commit()
gotCount = cursor.fetchone()[0]
assert 0 == gotCount, 'but got %s'%gotCount
#Try with three
smKeys = summaryCrashes.keys()[:3]
sum = dict((k,summaryCrashes[k]) for k in smKeys)
stime = dt.datetime(2009,9,8,7,30)
fifteen = dt.timedelta(minutes=15)
cd = tc.fixupCrashData(sum,stime,fifteen)
tc.storeFacts(cd,"interval three")
cursor.execute(countSql)
self.connection.commit()
gotCount = cursor.fetchone()[0]
assert 3 == gotCount, 'but got %s'%gotCount
cursor.execute("SELECT window_end, window_size from top_crashes_by_signature")
got = cursor.fetchall()
self.connection.commit()
expect = [(stime,fifteen),] * gotCount
assert expect == got, 'but got %s'%(got)
cursor.execute("DELETE FROM top_crashes_by_signature")
self.connection.commit()
#Try with about half
smKeys = summaryCrashes.keys()[-130:]
sum = dict((k,summaryCrashes[k]) for k in smKeys)
cd = tc.fixupCrashData(sum,dt.datetime(2009,9,8,7,30),fifteen)
tc.storeFacts(cd,"interval about half")
cursor.execute(countSql)
self.connection.commit()
gotCount = cursor.fetchone()[0]
#assert 130 == gotCount, 'but got %s'%gotCount #Lars buildid
assert 129 == gotCount, 'but got %s'%gotCount
cursor.execute("SELECT window_end,window_size from top_crashes_by_signature")
got = cursor.fetchall()
self.connection.commit()
expect = [(stime,fifteen),] * gotCount
assert expect == got, 'but got %s'%(got)
cursor.execute("DELETE FROM top_crashes_by_signature")
self.connection.commit()
#Try with all of them
cd = tc.fixupCrashData(summaryCrashes,dt.datetime(2009,9,8,7,30),fifteen)
tc.storeFacts(cd,"all of them")
cursor.execute(countSql)
self.connection.commit()
gotCount = cursor.fetchone()[0]
assert len(summaryCrashes) == gotCount, 'but got %s'%gotCount
cursor.execute("SELECT window_end, window_size from top_crashes_by_signature")
got = cursor.fetchall()
self.connection.commit()
expect = [(stime,fifteen),] * gotCount
assert expect == got, 'but got %s'%(got)
def testChangeWindowSize(self):
"""
TestTopCrashesBySignature.testChangeWindowSize
"""
global me
cursor = self.connection.cursor()
idCache = cia.IdCache(cursor,logger=me.logger)
minStamp, maxStamp, data = self.prepareExtractDataForPeriod('date_processed',31,5) # full set of keys
bogusEnd = dt.datetime(2007,12,30)
bogusSize = dt.timedelta(minutes=20)
configBegin = minStamp - dt.timedelta(hours=1)
configEnd = maxStamp + dt.timedelta(hours=1)
keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data])
self.prepareConfigForPeriod(keySet,configBegin,configEnd)
getSql = "SELECT window_end,window_size FROM top_crashes_by_signature ORDER BY id DESC LIMIT 1"
cursor.execute(getSql)
got = cursor.fetchone()
assert None == got, 'Expect no data at this point, but %s'%(str(got))
self.connection.rollback()
inSql = "INSERT INTO top_crashes_by_signature (window_end,window_size) VALUES (%s,%s)"
cursor.execute(inSql,(bogusEnd,bogusSize))
self.connection.commit()
cursor.execute(getSql)
got = cursor.fetchone()
self.connection.rollback()
assert bogusSize == got[1],'But it was %s'%(str(got))
config = copy.copy(me.config)
config['startDate'] = dt.datetime(2008,1,2,1)
config['endDate'] = dt.datetime(2008,1,2,2)
config['startWindow'] = config['startDate']
config['deltaWindow'] = dt.timedelta(minutes=10)
tc = topcrasher.TopCrashesBySignature(config)
tc.processDateInterval()
cursor.execute(getSql)
got = cursor.fetchone()
self.connection.rollback()
assert config['deltaWindow'] == got[1], 'But it was %s'%(str(got))
config['startDate'] = config['endDate']
config['endDate'] = dt.datetime(2008,1,2,3)
config['startWindow'] = config['startDate']
config['deltaWindow'] = dt.timedelta(minutes=60)
tc = topcrasher.TopCrashesBySignature(config)
tc.processDateInterval()
cursor.execute(getSql)
got = cursor.fetchone()
self.connection.rollback()
assert config['deltaWindow'] == got[1], 'But it was %s'%(str(got))
def testProcessDateInterval(self):
"""
TestTopCrashesBySignature.testProcessDateInterval
"""
global me
cursor = self.connection.cursor()
idCache = cia.IdCache(cursor)
# test a small full set of keys. Since we have already tested each component, that should be enough
minStamp, maxStamp, data = self.prepareExtractDataForPeriod('date_processed',31,5) # full set of keys
configBegin = minStamp - dt.timedelta(hours=1)
configEnd = maxStamp + dt.timedelta(hours=1)
keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data])
self.prepareConfigForPeriod(keySet,configBegin,configEnd)
tc = topcrasher.TopCrashesBySignature(me.config)
config = copy.copy(me.config)
config['startWindow'] = dt.datetime(2008,1,2,0,0)
config['deltaWindow'] = dt.timedelta(minutes=12)
config['startDate'] = dt.datetime(2008,1,2,0,0)
config['endDate'] = dt.datetime(2008,1,6,5,0)
tc = topcrasher.TopCrashesBySignature(config)
# first assure that we have a clean playing field
countSql = "SELECT count(*) from top_crashes_by_signature"
cursor.execute(countSql)
self.connection.commit()
gotCount = cursor.fetchone()[0]
assert 0 == gotCount, "but top_crashes_by_signature isn't empty. Had %s rows"%gotCount
sDate = minStamp.replace(hour=0,minute=0,second=0,microsecond=0)
maxMin = 30
dHour = dt.timedelta(hours=0)
if maxStamp.minute >= 30:
maxMin = 0
dHour = dt.timedelta(hours=1)
eDate = maxStamp.replace(minute=maxMin,second=0,microsecond=0)+dHour
tc.dateColumnName= 'client_crash_date'
tc.processDateInterval(startDate=sDate, endDate=eDate, dateColumnName='date_processed')
assert 'client_crash_date' == tc.dateColumnName
cursor.execute(countSql)
self.connection.commit()
gotCount = cursor.fetchone()[0]
me.logger.debug("DEBUG testProcessIntervals after count top_crashes_by_signature = %s",gotCount)
expect = 137
#expect = 155 # Lars buildid
assert expect == gotCount, 'Regression test only, value not calculated. Expect %s, got %s'%(expect,gotCount)
class TestNamedCursor(TestCase):
def setUp(self):
global me
if not me:
createMe()
self.testDB = TestDB()
self.testDB.removeDB(me.config, me.logger)
self.testDB.createDB(me.config, me.logger)
#self.connection = psycopg2.connect(me.dsn)
self.connection = me.database.connection()
def tearDown(self):
global me
self.testDB.removeDB(me.config, me.logger)
self.connection.close()
def reportDataGenerator(self, sizePerDay, numDays):
idGen = dbtestutil.moreUuid()
initialDate = dt.datetime(2008, 1, 1, 1, 1, 1, 1, tzinfo=UTC)
currentDate = dt.datetime(2008, 1, 1, 1, 1, 1, 1, tzinfo=UTC)
milli5 = dt.timedelta(milliseconds=5)
milli10 = dt.timedelta(milliseconds=10)
buildStrings = ['200712312355', '200712302355', '200712292355']
buildDates = [
dt.datetime(2007, 12, 31, 23, 55, tzinfo=UTC),
dt.datetime(2007, 12, 30, 23, 55, tzinfo=UTC),
dt.datetime(2007, 12, 29, 23, 55, tzinfo=UTC)
]
osNameVersions = [
('Windows NT', '6.6.6'),
('Windows NT', '6.6.6'),
('Windows', 'v.v.v'),
('Windows', 'v.v.v'),
('Windows', 'v.v.v'),
('Windows', 'v.v.v'),
('Mac OS X', '10.5.5'),
('Mac OS X', '10.5.6'),
('Mac OS X', '10.5.6'),
('Linux', '10.10.10'),
('Linux', '10.10.11'),
]
insData = []
for dummyDays in range(numDays):
count = 0
while count < sizePerDay:
os_name, os_version = osNameVersions[count %
len(osNameVersions)]
data = {
'uuid': idGen.next(),
'client_crash_date': currentDate,
'date_processed': currentDate + milli5,
'product': 'foxy',
'version': '3.6.9b2',
'build': buildStrings[count % len(buildStrings)],
'url': 'http://www.woo.wow/weee',
'install_age': 89000,
'last_crash': 0,
'uptime': 88000,
'email': None,
'os_name': os_name,
'os_version': os_version,
#'build_date': buildDates[count%len(buildDates)],
'user_comments': 'oh help',
'app_notes': "",
'distributor': "",
'distributor_version': "",
}
insData.append(data)
if not count % (3):
currentDate += milli10
count += 1
currentDate = initialDate + dt.timedelta(days=1)
cursor = self.connection.cursor()
addReportData(cursor, insData)
def build1000(self): #testBuild1000(self):
self.reportDataGenerator(1000, 10)
ncursor = self.connection.cursor('myCursor')
ncursor.execute(
'SELECT id,uuid,client_crash_date,date_processed from reports')
try:
while True:
data = ncursor.fetchmany(512)
if data and len(data):
print data[0][0], len(data)
else:
print "Broke: %s" % (data)
break
finally:
self.connection.commit()
class TestNamedCursor(unittest.TestCase):
def setUp(self):
global me
if not me:
createMe()
self.testDB = TestDB()
self.testDB.removeDB(me.config, me.logger)
self.testDB.createDB(me.config, me.logger)
#self.connection = psycopg2.connect(me.dsn)
self.connection = me.database.connection()
def tearDown(self):
global me
self.testDB.removeDB(me.config,me.logger)
self.connection.close()
def reportDataGenerator(self,sizePerDay,numDays):
idGen = dbtestutil.moreUuid()
initialDate = dt.datetime(2008,1,1,1,1,1,1,tzinfo=UTC)
currentDate = dt.datetime(2008,1,1,1,1,1,1,tzinfo=UTC)
milli5 = dt.timedelta(milliseconds=5)
milli10 = dt.timedelta(milliseconds=10)
buildStrings = ['200712312355','200712302355','200712292355']
buildDates = [dt.datetime(2007,12,31,23,55,tzinfo=UTC),dt.datetime(2007,12,30,23,55,tzinfo=UTC),dt.datetime(2007,12,29,23,55,tzinfo=UTC)]
osNameVersions = [('Windows NT','6.6.6'),('Windows NT','6.6.6'),('Windows','v.v.v'),('Windows','v.v.v'),('Windows','v.v.v'),('Windows','v.v.v'),
('Mac OS X','10.5.5'),('Mac OS X','10.5.6'),('Mac OS X','10.5.6'),
('Linux','10.10.10'),('Linux','10.10.11'),
]
insData = []
for dummyDays in range(numDays):
count = 0
while count < sizePerDay:
os_name,os_version = osNameVersions[count % len(osNameVersions)]
data = {
'uuid':idGen.next(),
'client_crash_date':currentDate,
'date_processed': currentDate+milli5,
'product': 'foxy',
'version': '3.6.9b2',
'build': buildStrings[count%len(buildStrings)],
'url':'http://www.woo.wow/weee',
'install_age':89000,
'last_crash':0,
'uptime':88000,
'email':None,
'os_name': os_name,
'os_version': os_version,
#'build_date': buildDates[count%len(buildDates)],
'user_comments': 'oh help',
'app_notes':"",
'distributor':"",
'distributor_version':"",
}
insData.append(data)
if not count%(3):
currentDate += milli10
count += 1
currentDate = initialDate+dt.timedelta(days=1)
cursor = self.connection.cursor()
addReportData(cursor,insData)
def build1000(self): #testBuild1000(self):
self.reportDataGenerator(1000,10)
ncursor = self.connection.cursor('myCursor')
ncursor.execute('SELECT id,uuid,client_crash_date,date_processed from reports')
try:
while True:
data = ncursor.fetchmany(512)
if data and len(data):
print data[0][0],len(data)
else:
print "Broke: %s"%(data)
break
finally:
self.connection.commit()
if errno.EEXIST == x.errno: pass
else: raise
f = open(me.logFilePathname, 'w')
f.close()
fileLog = logging.FileHandler(me.logFilePathname, 'a')
fileLog.setLevel(logging.DEBUG)
fileLogFormatter = logging.Formatter(
'%(asctime)s %(levelname)s - %(message)s')
fileLog.setFormatter(fileLogFormatter)
me.logger = logging.getLogger("db_test")
me.logger.addHandler(fileLog)
me.logger.setLevel(logging.DEBUG)
me.dsn = "host=%s dbname=%s user=%s password=%s" % (
me.config.databaseHost, me.config.databaseName,
me.config.databaseUserName, me.config.databasePassword)
me.testDB = TestDB()
def teardown_module():
pass
tptCreationSql = "CREATE TABLE %s (id serial not null primary key, date_col timestamp with time zone)"
partitionSql = "CREATE table %%(partitionName)s () INHERITS (%s)"
tptPartitionCreationSqlTemplate = partitionSql % 'tpt'
tpt3PartitionCreationSqlTemplate = partitionSql % 'tpt3'
class TPT(schema.PartitionedTable):
def __init__(self, logger):
super(TPT, self).__init__(
class TestMtbf(unittest.TestCase):
def setUp(self):
global me
self.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing MTBF')
myDir = os.path.split(__file__)[0]
if not myDir: myDir = '.'
replDict = {'testDir':'%s'%myDir}
for i in self.config:
try:
self.config[i] = self.config.get(i)%(replDict)
except:
pass
self.logger = TestingLogger(me.fileLogger)
self.connection = psycopg2.connect(me.dsn)
cursor = self.connection.cursor()
self.testDB = TestDB()
self.testDB.removeDB(self.config,self.logger)
self.testDB.createDB(self.config,self.logger)
self.prods = ['zorro','vogel','lizz',]
self.oss = ['OSX','LOX','WOX',]
self.productDimData = [] # filled in by fillMtbfTables
def tearDown(self):
self.testDB.removeDB(self.config,self.logger)
self.logger.clear()
def fillMtbfTables(self,cursor):
"""
Need some data to test with. Here's where we make it out of whole cloth...
"""
# (id),product,version,os,release : what product is it, by id
self.productDimData = [ [self.prods[p],'%s.1.%s'%(p,r), self.oss[o], 'beta-%s'%r] for p in range(2) for o in range(2) for r in range(1,4) ]
cursor.executemany('INSERT into productdims (product,version,os_name,release) values (%s,%s,%s,%s)',self.productDimData)
cursor.connection.commit()
cursor.execute("SELECT id, product,version, os_name, release from productdims")
productDimData = cursor.fetchall()
cursor.connection.commit()
self.baseDate = dt.date(2008,1,1)
self.intervals = {
'0.1.1':(self.baseDate ,self.baseDate+dt.timedelta(days=30)),
'0.1.2':(self.baseDate + dt.timedelta(days=10),self.baseDate + dt.timedelta(days=40)),
'0.1.3':(self.baseDate + dt.timedelta(days=20),self.baseDate + dt.timedelta(days=50)),
'1.1.1':(self.baseDate + dt.timedelta(days=10),self.baseDate + dt.timedelta(days=40)),
'1.1.2':(self.baseDate + dt.timedelta(days=20),self.baseDate + dt.timedelta(days=50)),
'1.1.3':(self.baseDate + dt.timedelta(days=30),self.baseDate + dt.timedelta(days=60)),
}
# processing days are located at and beyond the extremes of the full range, and
# at some interior points, midway between each pair of interior points
# layout is: (a date, the day-offset from baseDate, the expected resulting [ids])
PDindexes = [-1,0,5,10,15,25,35,45,55,60,61]
productsInProcessingDay = [
[], # -1,
[1,4],# 0,
[1,4],# 5,
[1,2,4,5,7,10],# 10,
[1,2,4,5,7,10],# 15,
[1,2,3,4,5,6,7,8,10,11],# 25,
[2,3,5,6,7,8,9,10,11,12],# 35,
[3,6,8,9,11,12],# 45,
[9,12],# 55,
[9,12],# 60,
[],# 61,
]
self.processingDays = [ (self.baseDate+dt.timedelta(days=PDindexes[x]),PDindexes[x],productsInProcessingDay[x]) for x in range(len(PDindexes))]
# (id), productdims_id, start_dt, end_dt : Date-interval when product is interesting
configData =[ (x[0],self.intervals[x[2]][0],self.intervals[x[2]][1] ) for x in productDimData ]
cursor.executemany('insert into mtbfconfig (productdims_id,start_dt,end_dt) values(%s,%s,%s)',configData)
cursor.connection.commit()
self.expectedFacts = {
# key is offset from baseDate
# value is array of (productDims_id,day,avg_seconds,report_count,count(distinct(user))
# This data WAS NOT CALCULATED BY HAND: The test was run once with prints in place
# and that output was encoded here. As of 2009-Feb, count of unique users is always 0
-1: [],
0: [
(1, dt.date(2008,1,1), 5, 6, 0),
(4, dt.date(2008,1,1), 20, 6, 0),
],
5: [
(1, dt.date(2008,1,6), 5, 6, 0),
(4, dt.date(2008,1,6), 20, 6, 0),
],
10: [
(1, dt.date(2008,1,11), 5, 6, 0),
(2, dt.date(2008,1,11), 10, 6, 0),
(4, dt.date(2008,1,11), 20, 6, 0),
(5, dt.date(2008,1,11), 25, 6, 0),
(7, dt.date(2008,1,11), 35, 6, 0),
(10, dt.date(2008,1,11), 50, 6, 0),
],
15: [
(1, dt.date(2008,1,16), 5, 6, 0),
(2, dt.date(2008,1,16), 10, 6, 0),
(4, dt.date(2008,1,16), 20, 6, 0),
(5, dt.date(2008,1,16), 25, 6, 0),
(7, dt.date(2008,1,16), 35, 6, 0),
(10, dt.date(2008,1,16), 50, 6, 0),
],
25: [
(1, dt.date(2008,1,26), 5, 6, 0),
(2, dt.date(2008,1,26), 10, 6, 0),
(3, dt.date(2008,1,26), 15, 6, 0),
(4, dt.date(2008,1,26), 20, 6, 0),
(5, dt.date(2008,1,26), 25, 6, 0),
(6, dt.date(2008,1,26), 30, 6, 0),
(7, dt.date(2008,1,26), 35, 6, 0),
(8, dt.date(2008,1,26), 40, 6, 0),
(10, dt.date(2008,1,26), 50, 6, 0),
(11, dt.date(2008,1,26), 55, 6, 0),
],
35: [
(2, dt.date(2008,2,5), 10, 6, 0),
(3, dt.date(2008,2,5), 15, 6, 0),
(5, dt.date(2008,2,5), 25, 6, 0),
(6, dt.date(2008,2,5), 30, 6, 0),
(7, dt.date(2008,2,5), 35, 6, 0),
(8, dt.date(2008,2,5), 40, 6, 0),
(9, dt.date(2008,2,5), 45, 6, 0),
(10, dt.date(2008,2,5), 50, 6, 0),
(11, dt.date(2008,2,5), 55, 6, 0),
(12, dt.date(2008,2,5), 60, 6, 0),
],
45: [
(3, dt.date(2008,2,15), 15, 6, 0),
(6, dt.date(2008,2,15), 30, 6, 0),
(8, dt.date(2008,2,15), 40, 6, 0),
(9, dt.date(2008,2,15), 45, 6, 0),
(11, dt.date(2008,2,15), 55, 6, 0),
(12, dt.date(2008,2,15), 60, 6, 0),
],
55: [
(9, dt.date(2008,2,25), 45, 6, 0),
(12, dt.date(2008,2,25), 60, 6, 0),
],
60: [
(9, dt.date(2008,3,1), 45, 6, 0),
(12, dt.date(2008,3,1), 60, 6, 0),
],
61: [],
}
def fillReports(self,cursor):
"""fill enough data to test mtbf behavior:
- AVG(uptime); COUNT(date_processed); COUNT(DISTINCT(user_id))
"""
self.fillMtbfTables(cursor) # prime the pump
sql = 'insert into reports (uuid, uptime, date_processed,product,version,os_name) values(%s,%s,%s,%s,%s,%s)'
processTimes = ['00:00:00','05:00:00','10:00:00','15:00:00','20:00:00','23:59:59']
uptimes = [5*x for x in range(1,15)]
data = []
uuidGen = dbtestutil.moreUuid()
uptimeIndex = 0
for product in self.productDimData:
uptime = uptimes[uptimeIndex%len(uptimes)]
uptimeIndex += 1
for d,off,ig in self.processingDays:
for pt in processTimes:
dp = "%s %s"%(d.isoformat(),pt)
tup = (uuidGen.next(), uptime,dp,product[0],product[1],product[2])
data.append(tup)
cursor.executemany(sql,data)
cursor.connection.commit()
# ========================================================================== #
def testCalculateMtbf(self):
"""
testCalculateMtbf(self): slow(1)
check that we get the expected data. This is NOT hand-calculated, just a regression check
"""
cursor = self.connection.cursor()
self.fillReports(cursor)
sql = 'select productdims_id,day,avg_seconds,report_count,unique_users from mtbffacts WHERE day = %s'
self.connection.commit()
for pd in self.processingDays:
self.config.processingDay = pd[0].isoformat()
mtbf.calculateMtbf(self.config, self.logger)
cursor.execute(sql,(pd[0].isoformat(),))
data = cursor.fetchall()
self.connection.commit()
expected = set(self.expectedFacts[pd[1]])
got = set(data)
assert expected==got, 'Expected: %s\nGot: %s'%(expected,got)
#end of loop through processingDay
# ========================================================================== #
def testGetProductsToUpdate(self):
"""
testGetProductsToUpdate(self):
check that we get the appropriate list of products when:
- we have none (on either end of the range)
- we have only one or several
check that we correctly log when there are no products in range
"""
cursor = self.connection.cursor()
self.fillMtbfTables(cursor)
for d in self.processingDays:
self.config.processingDay = d[0].isoformat()
self.logger.clear()
products = mtbf.getProductsToUpdate(cursor,self.config,self.logger)
self.connection.commit()
if d[1] in (-1,61):
# be sure that when appropriate we log a warning about no configured products
assert 2 == len(self.logger)
assert logging.WARNING == self.logger.levels[1]
assert 'Currently there are no MTBF products configured' == self.logger.buffer[1]
else:
# ignore the expected logging: It could change. Catch unexpected logging calls
assert len(d[2])+2 == len(self.logger)
INFO = 0
DBG = 0
oth = 0
for i in self.logger.levels:
if logging.INFO == i: INFO += 1
elif logging.DEBUG == i: DBG += 1
else: oth += 1
# Don't care about DBG or INFO counts, except we expect no other
#assert len(d[2])+1 == INFO, 'expected %d, but %s\n%s'%(len(d[2])+1,INFO,str(self.logger))
#assert 1 == DBG
assert 0 == oth
pids = set([x.dimensionId for x in products])
expected = set(d[2])
# This is the meat of the test
assert expected == pids, 'Expected %s, got %s'%(expected,pids)
# ========================================================================== #
def testGetWhereClauseFor(self):
"""
testGetWhereClauseFor(self):
check that we correctly handle the 'ALL' product
check that we correctly order and truncate version,product,os_name
"""
class P:
pass
p = P()
p.product = 'ALL'
assert '' == mtbf.getWhereClauseFor(p)
p.product = 'product'
assert_raises(AttributeError,mtbf.getWhereClauseFor,p)
p.version = 'version'
assert_raises(AttributeError,mtbf.getWhereClauseFor,p)
p.os_name='os_name'
expected = " AND version = 'version' AND product = 'product' AND substr(os_name, 1, 3) = 'os_name' "
assert expected == mtbf.getWhereClauseFor(p), 'but "%s"'%(mtbf.getWhereClauseFor(p))
# ========================================================================== #
def testClassProductDimension(self):
"""
testClassProductDimension(self):
check that we handle config with correct or greater items in config
"""
assert_raises(IndexError,mtbf.ProductDimension,[])
assert_raises(IndexError,mtbf.ProductDimension,[1])
assert_raises(IndexError,mtbf.ProductDimension,[1,2])
assert_raises(IndexError,mtbf.ProductDimension,[1,2,3])
self.logger.clear()
assert_raises(IndexError,mtbf.ProductDimension,[1,2,3,4])
config = [999,'umlaut',3.14,'OX','lemme go','toomuch']
pd = mtbf.ProductDimension(config,self.logger)
assert 999 == pd.dimensionId
assert 'umlaut' == pd.product
assert 3.14 == pd.version
assert 'OX' == pd.os_name
assert 'lemme go' == pd.release
assert 5 == len(pd.__dict__), "Assure nobody adds another dimension element without updating tests"
assert 0 == len(self.logger),'but logger:\n%s'%self.logger
class TestTopCrashesByUrl:
def setUp(self):
global me
self.logger = me.logger
#self.connection = psycopg2.connect(me.dsn)
self.connection = me.database.connection()
self.testDB = TestDB()
self.testDB.removeDB(me.config,self.logger)
self.testDB.createDB(me.config,self.logger)
def tearDown(self):
self.testDB.removeDB(me.config,self.logger)
socorro_cia.clearCache()
def testConstructor(self):
"""testTopCrashesByUrl:TestTopCrashesByUrl.testConstructor(self)"""
global me
config = copy.copy(me.config)
t = tcbu.TopCrashesByUrl(config)
assert 1 == t.configContext['minimumHitsPerUrl']
assert 500 == t.configContext['maximumUrls']
assert tcbu.logger.name
assert 'date_processed' == t.configContext.get('dateColumn')
config = copy.copy(me.config)
config['minimumHitsPerUrl'] = 2
config['maximumUrls'] = 100
config['logger'] = self.logger
t = tcbu.TopCrashesByUrl(config)
assert 2 == t.configContext.get('minimumHitsPerUrl')
assert 100 == t.configContext.get('maximumUrls')
assert self.logger == t.configContext.get('logger')
halfDay = datetime.timedelta(hours=12)
config = copy.copy(me.config)
t = tcbu.TopCrashesByUrl(config, minimumHitsPerUrl=3, maximumUrls=50,deltaWindow =halfDay)
assert 3 == t.configContext.get('minimumHitsPerUrl')
assert 50 == t.configContext.get('maximumUrls')
assert halfDay == t.configContext.get('deltaWindow')
def testCountCrashesByUrlInWindow(self):
"""testTopCrashesByUrl:TestTopCrashesByUrl.testCountCrashesByUrlInWindow(self):"""
global me
cursor = self.connection.cursor()
dbtutil.fillReportsTable(cursor,createUrls=True,multiplier=2,signatureCount=83) # just some data...
self.connection.commit()
config = copy.copy(me.config)
# test /w/ 'normal' params
t = tcbu.TopCrashesByUrl(config)
startWindow = datetime.datetime(2008,1,1)
deltaWindow = datetime.timedelta(days=1)
endWindow = startWindow + deltaWindow
data = t.countCrashesByUrlInWindow(startWindow = startWindow, deltaWindow = deltaWindow)
# the following are JUST regression tests: The data has been only very lightly examined to be sure it makes sense.
assert 24 == len(data), 'This is (just) a regression test. Did you change the data somehow? (%s)'%len(data)
for d in data:
assert 1 == d[0]
# test /w/ small maximumUrls
config = copy.copy(me.config)
t = tcbu.TopCrashesByUrl(config, maximumUrls=50)
data = t.countCrashesByUrlInWindow(startWindow = datetime.datetime(2008,1,1), endWindow = endWindow)
assert 24 == len(data), 'This is (just) a regression test. Did you change the data somehow? (%s)'%len(data)
for d in data:
assert 1 == d[0]
# test /w/ minimumHitsPerUrl larger
config = copy.copy(me.config)
t = tcbu.TopCrashesByUrl(config, minimumHitsPerUrl=2)
data = t.countCrashesByUrlInWindow(startWindow = datetime.datetime(2008,1,1),endWindow = endWindow)
assert 24 == len(data), len(data)
for d in data:
assert 1 == d[0]
# test /w/ shorter window
config = copy.copy(me.config)
halfDay = datetime.timedelta(hours=12)
t = tcbu.TopCrashesByUrl(config, deltaWindow = halfDay)
data = t.countCrashesByUrlInWindow(startWindow = datetime.datetime(2008,1,1))
assert 12 == len(data), 'This is (just) a regression test. Did you change the data somehow? (%s)'%len(data)
for d in data:
assert 1 == d[0]
# test a different day, to be sure we get different data
config = copy.copy(me.config)
t = tcbu.TopCrashesByUrl(config)
data = t.countCrashesByUrlInWindow(startWindow = datetime.datetime(2008,1,11),deltaWindow=deltaWindow)
assert 57 == len(data), 'This is (just) a regression test. Did you change the data somehow? (%s)'%len(data)
for d in data[:3]:
assert 2 == d[0]
for d in data[3:]:
assert 1 == d[0]
def testSaveData(self):
"""
testTopCrashesByUrl:TestTopCrashesByUrl.testSaveData(slow=2)
This is a reasonably realistic amount of time (about 1.5 seconds) to handle about 150 reports
"""
global me
cursor = self.connection.cursor()
## Set up
dbtutil.fillReportsTable(cursor,createUrls=True,multiplier=2,signatureCount=83) # just some data...
self.connection.commit()
# ... now assure some duplicates
sqls = "SELECT uuid, client_crash_date, install_age, last_crash, uptime, date_processed, success, signature, url, product, version, os_name, os_version from reports where date_processed >= '2008-01-01' and date_processed < '2008-01-02' LIMIT 4"
cursor.execute(sqls)
self.connection.rollback() # db not altered
rows3 = cursor.fetchall()
add11 = datetime.timedelta(seconds=1,microseconds=1000)
addData = []
for i in range(3):
r = list(rows3[i])
r[0] = r[0].replace('-dead-','-f00f-')
r[1] += add11
r[2] += 1
r[3] += 1
r[7] = rows3[i+1][7]
addData.append(r)
r[0] = r[0].replace('-f00f-','-fead-')
r[1] += add11
r[2] += 1
r[3] += 1
r[7] = 'js_blatherskytes'
addData.append(r)
sqli = """INSERT INTO reports
(uuid, client_crash_date, install_age, last_crash, uptime, date_processed, success, signature, url, product, version, os_name, os_version)
VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"""
addData.extend([
['b965de73-ae90-b936-deaf-03ae20081225','2007-12-31 23:59:50',9000,110,222,'2008-01-01 11:12:13',True,'UserCallWinProcCheckWow','http://www.mozilla.org/projects/minefield/a','Firefox','3.0.9','Windows NT','5.1.2600 Service Pack 2'],
['b965de73-ae90-b935-deaf-03ae20081225','2007-12-31 23:59:40',9009,220,333,'2008-01-01 11:12:14',True,'UserCallWinProcCheckWow','http://yachats/uncwiki/LarsLocalPortal/b', 'Firefox','3.0.9','Windows NT','5.1.2600 Service Pack 2'],
])
cursor.executemany(sqli,addData)
self.connection.commit()
config = copy.copy(me.config)
startWindow = datetime.datetime(2008,1,1)
deltaWindow = datetime.timedelta(days=1)
## On your mark...
t = tcbu.TopCrashesByUrl(config)
data = t.countCrashesByUrlInWindow(startWindow = startWindow, deltaWindow = deltaWindow)
## assure we have an empty playing field
cursor.execute("SELECT COUNT(*) from top_crashes_by_url")
self.connection.rollback()
assert 0 == cursor.fetchone()[0]
## Call the method
t.saveData(startWindow,data)
# expect 99 rows
cursor.execute("SELECT COUNT(id) from top_crashes_by_url")
self.connection.rollback()
count = cursor.fetchone()[0]
assert 35 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)
# expect 80 distinct urls
cursor.execute("SELECT COUNT(distinct urldims_id) from top_crashes_by_url")
self.connection.rollback()
count = cursor.fetchone()[0]
assert 21 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)
cursor.execute("SELECT count from top_crashes_by_url where count > 1 order by count")
self.connection.rollback()
data = cursor.fetchall()
assert [(2,),(2,),(2,)] == data, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(str(data))
cursor.execute("SELECT COUNT(top_crashes_by_url_id) from top_crashes_by_url_signature")
self.connection.rollback()
count = cursor.fetchone()[0]
assert 38 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)
cursor.execute("SELECT COUNT(distinct top_crashes_by_url_id) from top_crashes_by_url_signature")
self.connection.rollback()
count = cursor.fetchone()[0]
assert 35 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)
# Expect 3 rows with sums of 2 and three rows with counts of 2, none with both
cursor.execute("SELECT count, COUNT(top_crashes_by_url_id) AS sum FROM top_crashes_by_url_signature GROUP BY top_crashes_by_url_id, count ORDER BY sum DESC, count DESC LIMIT 6")
self.connection.rollback()
data = cursor.fetchall()
assert 6 == len(data)
assert [(1, 2L), (1, 2L), (1, 2L), (1, 1L), (1, 1L), (1, 1L)] == data, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(str(data))
def testSaveTruncatedData(self):
"""
testTopCrashesByUrl:TestTopCrashesByUrl.testSaveTruncatedData(slow=2)
This is a reasonably realistic amount of time (about 1.5 seconds) to handle about 150 reports
"""
global me
cursor = self.connection.cursor()
## Set up
dbtutil.fillReportsTable(cursor,createUrls=True,multiplier=2,signatureCount=83) # just some data...
self.connection.commit()
# ... now assure some duplicates
sqls = "SELECT uuid, client_crash_date, install_age, last_crash, uptime, date_processed, success, signature, url, product, version, os_name, os_version from reports where date_processed >= '2008-01-01' and date_processed < '2008-01-02' LIMIT 4"
cursor.execute(sqls)
self.connection.rollback() # db not altered
rows3 = cursor.fetchall()
add11 = datetime.timedelta(seconds=1,microseconds=1000)
addData = []
for i in range(3):
r = list(rows3[i])
r[0] = r[0].replace('-dead-','-f00f-')
r[1] += add11
r[2] += 1
r[3] += 1
r[7] = rows3[i+1][7]
addData.append(r)
r[0] = r[0].replace('-f00f-','-fead-')
r[1] += add11
r[2] += 1
r[3] += 1
r[7] = 'js_blatherskytes'
addData.append(r)
sqli = """INSERT INTO reports
(uuid, client_crash_date, install_age, last_crash, uptime, date_processed, success, signature, url, product, version, os_name, os_version)
VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"""
addData.extend([
['b965de73-ae90-b936-deaf-03ae20081225','2007-12-31 23:59:50',9000,110,222,'2008-01-01 11:12:13',True,'UserCallWinProcCheckWow','http://www.mozilla.org/projects/minefield/a','Firefox','3.0.9','Windows NT','5.1.2600 Service Pack 2'],
['b965de73-ae90-b935-deaf-03ae20081225','2007-12-31 23:59:40',9009,220,333,'2008-01-01 11:12:14',True,'UserCallWinProcCheckWow','http://yachats/uncwiki/LarsLocalPortal/b', 'Firefox','3.0.9','Windows NT','5.1.2600 Service Pack 2'],
])
cursor.executemany(sqli,addData)
self.connection.commit()
config = copy.copy(me.config)
startWindow = datetime.datetime(2008,1,1)
deltaWindow = datetime.timedelta(days=1)
## On your mark...
t = tcbu.TopCrashesByUrl(config,truncateUrlLength=25)
data = t.countCrashesByUrlInWindow(startWindow = startWindow, deltaWindow = deltaWindow)
## assure we have an empty playing field
cursor.execute("SELECT COUNT(*) from top_crashes_by_url")
self.connection.rollback()
assert 0 == cursor.fetchone()[0]
## Call the method
t.saveData(startWindow,data)
# expect 99 rows
cursor.execute("SELECT COUNT(id) from top_crashes_by_url")
self.connection.rollback()
count = cursor.fetchone()[0]
assert 30 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)
# expect 80 distinct urls
cursor.execute("SELECT COUNT(distinct urldims_id) from top_crashes_by_url")
self.connection.rollback()
count = cursor.fetchone()[0]
assert 17 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)
cursor.execute("SELECT count from top_crashes_by_url where count > 1 order by count")
self.connection.rollback()
data = cursor.fetchall()
assert [(2,), (2,), (2,), (2,), (2,), (4,)] == data, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(str(data))
cursor.execute("SELECT COUNT(top_crashes_by_url_id) from top_crashes_by_url_signature")
self.connection.rollback()
count = cursor.fetchone()[0]
assert 38 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)
cursor.execute("SELECT COUNT(distinct top_crashes_by_url_id) from top_crashes_by_url_signature")
self.connection.rollback()
count = cursor.fetchone()[0]
assert 30 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)
# Expect 3 rows with sums of 2 and three rows with counts of 2, none with both
cursor.execute("SELECT count, COUNT(top_crashes_by_url_id) AS sum FROM top_crashes_by_url_signature GROUP BY top_crashes_by_url_id, count ORDER BY sum DESC, count DESC LIMIT 6")
self.connection.rollback()
data = cursor.fetchall()
assert 6 == len(data)
assert [(1, 4L), (1, 2L), (1, 2L), (1, 2L), (1, 2L), (1, 2L)] == data, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(str(data))
def testProcessDateInterval(self):
"""
testTopCrashesByUrl:TestTopCrashesByUrl.testProcessDateInterval(slow=7)
Takes a long time, first to set up the data (about 1.5 seconds), then to process it several times
"""
global me
cursor = self.connection.cursor()
config = copy.copy(me.config)
## Set up
dbtutil.fillReportsTable(cursor,createUrls=True,multiplier=2,signatureCount=83) # just some data...
self.connection.commit()
t = tcbu.TopCrashesByUrl(config)
## assure we have an empty playing field
cursor.execute("SELECT COUNT(*) from top_crashes_by_url")
self.connection.rollback()
assert 0 == cursor.fetchone()[0]
t.processDateInterval(startDate = datetime.datetime(2008,1,1), endDate=datetime.datetime(2008,1,6))
cursor.execute("SELECT COUNT(id) from top_crashes_by_url")
self.connection.rollback()
count = cursor.fetchone()[0]
assert 35 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)
cursor.execute("SELECT COUNT(*) from top_crashes_by_url_signature")
self.connection.rollback()
count = cursor.fetchone()[0]
assert 38 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)
cursor.execute("delete from top_crashes_by_url; delete from top_crashes_by_url_signature")
self.connection.commit()
t = tcbu.TopCrashesByUrl(copy.copy(me.config))
t.processDateInterval(startDate = datetime.datetime(2008,1,4), endDate=datetime.datetime(2008,1,8))
cursor.execute("SELECT COUNT(id) from top_crashes_by_url")
self.connection.rollback()
count = cursor.fetchone()[0]
assert 31 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)
cursor.execute("SELECT COUNT(*) from top_crashes_by_url_signature")
self.connection.rollback()
count = cursor.fetchone()[0]
assert 32 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)
cursor.execute("delete from top_crashes_by_url; delete from top_crashes_by_url_signature")
self.connection.commit()
t = tcbu.TopCrashesByUrl(copy.copy(me.config))
t.processDateInterval(startDate = datetime.datetime(2008,1,1), endDate=datetime.datetime(2008,3,3))
cursor.execute("SELECT COUNT(id) from top_crashes_by_url")
self.connection.rollback()
count = cursor.fetchone()[0]
assert 483 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)
cursor.execute("SELECT COUNT(*) from top_crashes_by_url_signature")
self.connection.rollback()
count = cursor.fetchone()[0]
assert 514 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)