def main(): logger = logging.getLogger("topcrashes_summary") logger.setLevel(logging.WARNING) stderrLog = logging.StreamHandler() stderrLog.setLevel(logging.WARNING) stderrLogFormatter = logging.Formatter('%(asctime)s %(levelname)s - %(message)s') stderrLog.setFormatter(stderrLogFormatter) logger.addHandler(stderrLog) kwargs = {} for i in sys.argv[1:]: if i.startswith('-h') or i.startswith('--he'): help() sys.exit(0) j = i if i.startswith('-'): j = i.lstrip('-') if '=' in j: name,value = (s.strip() for s in j.split('=')) kwargs[name] = value else: print >> sys.stderr,"Ignoring unkown argument '%s'"%(i) sys.argv = sys.argv[:1] config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Create Database') config.update(kwargs) testDB = TestDB() testDB.removeDB(config,logger) testDB.createDB(config,logger)
def setUp(self): global me if not me: createMe() self.testDB = TestDB() self.testDB.removeDB(me.config, me.logger) self.testDB.createDB(me.config, me.logger) #self.connection = psycopg2.connect(me.dsn) self.connection = me.database.connection()
def createData(config,logger): # Now do the work in several steps connection = sdatabase.Database(config).connection() #connection = psycopg2.connect("host=%(databaseHost)s dbname=%(databaseName)s user=%(databaseUserName)s password=%(databasePassword)s"%config) cursor = connection.cursor() testDB = TestDB() try: testDB.removeDB(config,logger) if config.get('drop-all'): print "Dropped the database tables ..." return print "Creating the database tables..." testDB.createDB(config,logger) print "populating the dimensions tables..." processingDays,ignore = dbtestutil.fillMtbfTables(cursor,limit=int(config.get('product-os-count'))) startDays = [x[0] for x in processingDays] multiplier = int(config.get('repeat-count')) print "populating the reports table (takes about %d seconds)..."%(int(1.7+1.2*multiplier)) dbtestutil.fillReportsTable(cursor,createUrls=True,doFillMtbfTables=False, numUrls=100, multiplier=multiplier,signatureCount=int(config.get('signature-count'))) connection.commit() extras = [] print "All done populating the 'raw' data" if config.get('mtbf-fill'): blurb = "" cost = 0.20 + multiplier*0.15 if cost > 1.0: blurb = ("(takes about %2.1f seconds)"%cost) print "Filling the time_before_failure table %s..."%blurb # R=1: .35 seconds; 2:0.49s; 3:.064s; 4:.9 ## = .20 + R*.15 starter = None ender = None mtbfInstance = mtbf.Mtbf(config,logger) for startDay in startDays: if not starter: starter = startDay ender = startDay mtbfInstance.processOneMtbfWindow(processingDay=startDay) extras.append(" - Time before fail: for days in %s through %s"%(starter,ender)) if config.get('sig-fill'): print "Filling the top_crashes_by_signature table (takes about %s seconds)..."%(20+11*multiplier) # R=1: 27.3 secs; 2:38.5s; 3=48.3 ## = 20 +R*11 tc = topcrasher.TopCrashesBySignature(config) tc.processDateInterval(startDate=startDays[0],endDate = startDays[-1]) extras.append(" - Top crash by sig: for days in %s through %s"%(startDays[0],startDays[-1])) if config.get('url-fill'): print "Filling the top_crashes_by_url table (takes about %s seconds)..."%(4+multiplier*2) # R=1: 4 secs; 2: 5s, 3: 7.6 ## = 4+R*2 logger.info("Filling the top_crashes_by_url table (takes about %s seconds)..."%(4+multiplier*2)) tu = topcrashbyurl.TopCrashesByUrl(config) tu.processDateInterval(startDate=startDays[0],endDate = startDays[-1]) extras.append(" - Top crash by url: for days in %s through %s"%(startDays[0],startDays[-1])) print "DONE populating the database tables" if extras: print "\n".join(extras) finally: logger.info("All done. Closing connection") connection.close()
def setup_module(): global me if me: return me = Me() me.testDB = TestDB() me.config = configurationManager.newConfiguration( configurationModule=testConfig, applicationName='TestingCachedIdAccess') myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir': '%s' % myDir} for i in me.config: try: me.config[i] = me.config.get(i) % (replDict) except: pass cia.logger.setLevel(logging.DEBUG) me.logFilePathname = me.config.logFilePathname logfileDir = os.path.split(me.config.logFilePathname)[0] try: os.makedirs(logfileDir) except OSError, x: if errno.EEXIST != x.errno: raise f = open(me.config.logFilePathname, 'w') f.close()
def setUp(self): global me self.logger = me.logger #self.connection = psycopg2.connect(me.dsn) self.connection = me.database.connection() self.testDB = TestDB() self.testDB.removeDB(me.config,self.logger) self.testDB.createDB(me.config,self.logger)
def setUp(self): global me if not me: createMe() self.connection = me.database.connection() #self.connection = psycopg2.connect(me.dsn) self.testDB = TestDB() self.testDB.removeDB(me.config, me.logger) self.testDB.createDB(me.config, me.logger) dbtestutil.fillDimsTables(self.connection.cursor()) self.connection.commit()
def main(): logger = logging.getLogger("topcrashes_summary") logger.setLevel(logging.WARNING) stderrLog = logging.StreamHandler() stderrLog.setLevel(logging.WARNING) stderrLogFormatter = logging.Formatter( '%(asctime)s %(levelname)s - %(message)s') stderrLog.setFormatter(stderrLogFormatter) logger.addHandler(stderrLog) kwargs = {} for i in sys.argv[1:]: if i.startswith('-h') or i.startswith('--he'): help() sys.exit(0) j = i if i.startswith('-'): j = i.lstrip('-') if '=' in j: name, value = (s.strip() for s in j.split('=')) kwargs[name] = value else: print >> sys.stderr, "Ignoring unkown argument '%s'" % (i) sys.argv = sys.argv[:1] config = configurationManager.newConfiguration( configurationModule=testConfig, applicationName='Create Database') config.update(kwargs) testDB = TestDB() testDB.removeDB(config, logger) testDB.createDB(config, logger)
def setUp(self): global me # config gets messed up by some tests. Use this one by preference self.config = configurationManager.newConfiguration( configurationModule=testConfig, applicationName='Testing Postgresql Utils') for i in self.config: try: self.config[i] = self.config.get(i) % (replDict) except: pass self.connection = psycopg2.connect(me.dsn) self.testDB = TestDB()
def setUp(self): global me self.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing MTBF') myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir':'%s'%myDir} for i in self.config: try: self.config[i] = self.config.get(i)%(replDict) except: pass self.logger = TestingLogger(me.fileLogger) self.connection = psycopg2.connect(me.dsn) cursor = self.connection.cursor() self.testDB = TestDB() self.testDB.removeDB(self.config,self.logger) self.testDB.createDB(self.config,self.logger) self.prods = ['zorro','vogel','lizz',] self.oss = ['OSX','LOX','WOX',] self.productDimData = [] # filled in by fillMtbfTables
class TestTopCrashesBySignature(unittest.TestCase): def setUp(self): global me if not me: createMe() self.connection = me.database.connection() #self.connection = psycopg2.connect(me.dsn) self.testDB = TestDB() self.testDB.removeDB(me.config, me.logger) self.testDB.createDB(me.config, me.logger) dbtestutil.fillDimsTables(self.connection.cursor()) self.connection.commit() def tearDown(self): global me self.testDB.removeDB(me.config,me.logger) cia.clearCache() self.connection.close() def prepareConfigForPeriod(self,idkeys,startDate,endDate): """enter a row for each idkey, start and end date. Do it repeatedly if you need different ones""" cfgKeys = set([x[0] for x in idkeys]) cfgdata = [[x,startDate,endDate] for x in cfgKeys] self.connection.cursor().execute("delete from product_visibility") sql = "INSERT INTO product_visibility (productdims_id,start_date,end_date) VALUES (%s,%s,%s)" self.connection.cursor().executemany(sql,cfgdata) self.connection.commit() def prepareExtractDataForPeriod(self, columnName, sizePerDay=2, numDays=5): """Put some crash data into reports table, return their extrema for the columnName specified""" global me cursor = self.connection.cursor() data = makeReportData(sizePerDay,numDays) self.connection.commit() # find the actual min and max dates minStamp = dt.datetime(2100,11,11) maxStamp = dt.datetime(1900,11,11) for d in data: ds = d[columnName] if ds < minStamp: minStamp = ds if ds > maxStamp: maxStamp = ds addReportData(cursor,data) self.connection.commit() return minStamp,maxStamp,data def testConstructor(self): """ TestTopCrashesBySignature.testConstructor """ global me for reqd in ["databaseHost","databaseName","databaseUserName","databasePassword",]: cc = copy.copy(me.config) del(cc[reqd]) assert_raises(SystemExit,topcrasher.TopCrashesBySignature,cc) bogusMap = {'databaseHost':me.config.databaseHost,'databaseName':me.config.databaseName,'databaseUserName':'******','databasePassword':me.config.databasePassword} assert_raises(SystemExit,topcrasher.TopCrashesBySignature,bogusMap) #check without specific config items now = dt.datetime.now() tc = topcrasher.TopCrashesBySignature(me.config) expectedStart = now.replace(hour=0,minute=0,second=0,microsecond=0) - tc.configContext.get('initialDeltaDate',dt.timedelta(days=4)) expectedEnd = now.replace(hour=0,minute=0,second=0,microsecond=0) assert expectedStart == tc.startDate assert tc.endDate <= now, 'But NOT %s <= %s'%(tc.endDate,now) assert dt.timedelta(days=4) <= tc.deltaDate, 'But got %s'%(tc.deltaDate) # Check for defaults. If the defaults are hard-code changed, this block needs equivalent changes assert 'date_processed' == tc.dateColumnName # end Check for defaults # check with solo startDate, endDate or deltaDate config = copy.copy(me.config) config['startDate'] = dt.datetime(2009,01,01,0,0,1) assert_raises(SystemExit,topcrasher.TopCrashesBySignature,config) config = copy.copy(me.config) config['endDate'] = dt.datetime(2009,01,01,0,0,1) assert_raises(SystemExit,topcrasher.TopCrashesBySignature,config) config = copy.copy(me.config) config['deltaDate'] = dt.timedelta(days=3) assert_raises(SystemExit,topcrasher.TopCrashesBySignature,config) def testExtractDataForPeriod_ByDateProcessed(self): """ TestTopCrashesBySignature.testExtractDataForPeriod_ByDateProcessed(self): - Check that we get nothing for a period outside our data - Check that we get expected count for period inside our data - Check that we get half-open interval (don't get data exactly at last moment) - Check that we get all for period that surrounds our data - Check that we get nothing for start == end - Check that we get one moment for end = start+microsecond - Check that we raise ValueError for period with dates reversed """ global me minStamp,maxStamp,data = self.prepareExtractDataForPeriod('date_processed') cursor = self.connection.cursor() idCache = cia.IdCache(cursor) keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data]) mminStamp = minStamp - dt.timedelta(days=1) mmaxStamp = maxStamp + dt.timedelta(days=1) self.prepareConfigForPeriod(keySet,mminStamp,mmaxStamp) tc = topcrasher.TopCrashesBySignature(me.config) tc.dateColumnName = 'date_processed' # we should get nothing if we are outside our data start = minStamp - dt.timedelta(days=10) end = start + dt.timedelta(days=1) crashData = {} tc.extractDataForPeriod(start,end,crashData) assert {} == crashData,'Expected {}, got %s'%crashData # We should get exactly one day's worth start = minStamp end = start + dt.timedelta(days=1) tc.extractDataForPeriod(start,end,crashData) crashDataCount = 0 # all three known for k,d in crashData.items(): crashDataCount += d['count'] assert 2 == crashDataCount, 'Expected 2, got %d'%crashDataCount # we should get all but the last 2 (or one) item: Half-open interval start = minStamp end = maxStamp crashData = {} tc.extractDataForPeriod(start,end,crashData) crashDataCount = 0 # all three known for k,d in crashData.items(): crashDataCount += d['count'] assert crashDataCount == 9, 'Expected 9. Got %s'%crashDataCount # we should get all the data start = minStamp end = maxStamp+dt.timedelta(milliseconds=1) crashData = {} tc.extractDataForPeriod(start,end,crashData) crashDataCount = 0 # all three known for k,d in crashData.items(): crashDataCount += d['count'] assert crashDataCount == 10, 'Expected 10. Got %s'%crashDataCount # we should get nothing if start == end (half open interval that stops juuuust before the start) start = minStamp end = minStamp crashData = {} tc.extractDataForPeriod(start,end,crashData) assert crashData == {}, 'But got %s'%crashData # we should get precise stamp's data if end = start+teeny-tiny-epsilon start = minStamp end = start+dt.timedelta(microseconds=1) crashData = {} tc.extractDataForPeriod(start,end,crashData) crashDataCount = 0 # all three known for k,d in crashData.items(): crashDataCount += d['count'] assert crashDataCount == 1, 'Expected 1. Got %s'%crashDataCount # We should throw if start > end assert_raises(ValueError,tc.extractDataForPeriod,maxStamp,minStamp,crashData) def testExtractDataForPeriod_ByClientCrashDate(self): """ TestTopCrashesBySignature.testExtractDataForPeriod_ByClientCrashDate(self): - Check that we get nothing for a period outside our data - Check that we get expected count for period inside our data - Check that we get half-open interval (don't get data exactly at last moment) - Check that we get all for period that surrounds our data - Check that we raise ValueError for period with dates reversed """ global me minStamp,maxStamp,data = self.prepareExtractDataForPeriod('client_crash_date') cursor = self.connection.cursor() idCache = cia.IdCache(cursor) keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data]) mminStamp = minStamp - dt.timedelta(days=1) mmaxStamp = maxStamp + dt.timedelta(days=1) self.prepareConfigForPeriod(keySet,mminStamp,mmaxStamp) tc = topcrasher.TopCrashesBySignature(me.config) tc.dateColumnName = 'client_crash_date' # we should get nothing if we are outside our data start = minStamp - dt.timedelta(days=10) end = start + dt.timedelta(days=1) crashData = {} tc.extractDataForPeriod(start,end,crashData) assert {} == crashData,'Expected {}, got %s'%crashData # We should get exactly one day's worth start = minStamp end = start + dt.timedelta(days=1) tc.extractDataForPeriod(start,end,crashData) crashDataCount0 = 0 # all three known for k,d in crashData.items(): crashDataCount0 += d['count'] assert 2 == crashDataCount0, 'Expected 2, got %d'%crashDataCount0 # we should get all but the last 2 (or one) item: Half-open interval start = minStamp end = maxStamp crashData = {} tc.extractDataForPeriod(start,end,crashData) crashDataCount0 = 0 # all three known for k,d in crashData.items(): crashDataCount0 += d['count'] assert crashDataCount0 == 9, 'Expected 9. Got %s'%crashDataCount0 # we should get all the data start = minStamp end = maxStamp+dt.timedelta(milliseconds=1) crashData = {} tc.extractDataForPeriod(start,end,crashData) crashDataCount0 = 0 # all three known for k,d in crashData.items(): crashDataCount0 += d['count'] assert crashDataCount0 == 10, 'Expected 10. Got %s'%crashDataCount0 # we should get nothing if start == end (half open interval that stops juuuust before the start) start = minStamp end = minStamp crashData = {} tc.extractDataForPeriod(start,end,crashData) assert crashData == {}, 'But got %s'%crashData # we should get precise stamp's data if end = start+teeny-tiny-epsilon start = minStamp end = start+dt.timedelta(microseconds=1) crashData = {} tc.extractDataForPeriod(start,end,crashData) crashDataCount0 = 0 # all three known for k,d in crashData.items(): crashDataCount0 += d['count'] assert crashDataCount0 == 1, 'Expected 1. Got %s'%crashDataCount0 # We should throw if start > end assert_raises(ValueError,tc.extractDataForPeriod,maxStamp,minStamp,crashData) def testFixupCrashData(self): """ TestTopCrashesBySignature.testFixupCrashData(self):(slow=1) - create a bunch of data, count it in the test, put it into the reports table, then let TopCrashesBySignature get it back out, and assert that fixup gets the same count """ global me cursor = self.connection.cursor() idCache = cia.IdCache(cursor) data = makeReportData(211,5) # 211 is prime, which means we should get a nice spread of data keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data]) #keySet = set([(d['productdims_id'],d['osdims_id']) for d in data]) expect = {} newCount = 0 oldCount = 0 for d in data: # Keys are always signature,product,os key = (d['signature'],idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) if d['version'][-3:] == 'pre': continue # Lars buildid hang_count = 0 plugin_count = 0 if d['hangid'] != '': hang_count = 1 if d['process_type'] == 'plugin': plugin_count = 1; try: expect[key]['count'] += 1 expect[key]['uptime'] += d['uptime'] expect[key]['hang_count'] += hang_count expect[key]['plugin_count'] += plugin_count oldCount += 1 except: expect[key] = {} expect[key]['count'] = 1 expect[key]['uptime'] = d['uptime'] expect[key]['hang_count'] = hang_count expect[key]['plugin_count'] = plugin_count newCount += 1 addReportData(cursor,data) self.connection.commit() tc = topcrasher.TopCrashesBySignature(me.config) tc.dateColumnName = 'date_processed' baseDate = testBaseDate start = baseDate - dt.timedelta(days=1) end = baseDate + dt.timedelta(days=6) self.prepareConfigForPeriod(keySet,start,end) summaryCrashes = {} tc.extractDataForPeriod(start,end,summaryCrashes) # assert expect == summaryCrashes, 'Oops. You will need to debug on this. Sorry. Try commenting this line and un-commenting the next bunch' # the next few lines amount to "assert expect == summaryCrashes" but with more useful error detail # check that we have a chance of equality assert len(expect) == len(summaryCrashes), 'Expected equal lengths but expected:%s versus got:%s'%(len(expect),len(summaryCrashes)) getter = itemgetter('count') # ,'uptime') #,'hang_count','plugin_count') # check that everything in what we got is expected for k,v in summaryCrashes.items(): assert getter(expect[k]) == getter(v), 'for key %s: Expected %s but got %s'%(k,getter(expect[k]),getter(v)) # check that everything we expect is in what we got for k,v in expect.items(): assert getter(v) == getter(summaryCrashes[k]), 'for key %s: Expected %s but got %s'%(k,getter(v), getter(summaryCrashes[k])) result = tc.fixupCrashData(summaryCrashes,baseDate,dt.timedelta(minutes=19)) result.sort(key=itemgetter('count'),reverse=True) resultx = tc.fixupCrashData(expect,baseDate,dt.timedelta(minutes=19)) resultx.sort(key=itemgetter('count'),reverse=True) assert result == resultx ## The next few lines show more detail if there's a problem. Comment above, uncomment below # misscount = fullcount = 0 # if result != resultx: # assert len(result) == len(resultx), 'expected len: %s, got %s'%(len(result), len(resultx)) # for i in range(len(result)): # r = result[i] # rx = resultx[i] # print "%s" % (r) # print "%s" % (rx) # assert len(r) == len(rx), 'at loop %s: expected len: %s got %s'(i,len(r),len(rx)) # for j in range(len(r)): # fullcount += 1 # if r[j] != rx[j]: # print "unequal at loop %s, item %s:\n %s\n %s"%(i,j,r[j],rx[j]) # misscount += 1 # print "Unequal for %s of %s (%3.1f%%) rows"%(misscount,fullcount,100.0*(1.0*misscount/fullcount)) # assert 0 == misscount, "They weren't equal. Look nearby for details" expectedAll = set(['xpcom_core.dll@0x31b7a', 'js_Interpret', '_PR_MD_SEND', 'nsFormFillController::SetPopupOpen', 'nsAutoCompleteController::ClosePopup','morkRowObject::CloseRowObject(morkEnv*)', 'nsContainerFrame::ReflowChild(nsIFrame*, nsPresContext*, nsHTMLReflowMetrics&, nsHTMLReflowState const&, int, int, unsigned int, unsigned int&, nsOverflowContinuationTracker*)']) gotAll = set([ x['signature'] for x in result]) assert expectedAll == gotAll , 'ex-got:%s\ngot-ex:%s'%(expectedAll-gotAll,gotAll-expectedAll) def testExtractDataForPeriod_ConfigLimitedDates(self): """ TestTopCrashesBySignature.testExtractDataForPeriod_ConfigLimitedDates(self) """ global me minStamp,maxStamp,data = self.prepareExtractDataForPeriod('date_processed',23,5) cursor = self.connection.cursor() idCache = cia.IdCache(cursor) keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data]) keySSet = set([x for x in keySet if 5 != x[0]]) mminStamp = minStamp - dt.timedelta(days=1) mmaxStamp = maxStamp + dt.timedelta(days=1) self.prepareConfigForPeriod(keySet,mminStamp,mmaxStamp) tc = topcrasher.TopCrashesBySignature(me.config) tc.dateColumnName = 'date_processed' summaryData = {} tc.extractDataForPeriod(minStamp,maxStamp,summaryData) #assert 110 == len(summaryData), 'Regression test only, value not calculated. Expect 110, got %s'%(len(summaryData)) #Lars buildid assert 99 == len(summaryData), 'Regression test only, value not calculated. Expect 99, got %s'%(len(summaryData)) cursor.execute("DELETE from product_visibility") self.connection.commit() mminStamp = minStamp + dt.timedelta(days=1) mmaxStamp = maxStamp + dt.timedelta(days=1) self.prepareConfigForPeriod(keySet,mminStamp,mmaxStamp) summaryData = {} tc.extractDataForPeriod(minStamp,maxStamp,summaryData) #assert 87 == len(summaryData), 'Regression test only, value not calculated. Expect 87, got %s'%(len(summaryData)) #Lars buildid assert 78 == len(summaryData), 'Regression test only, value not calculated. Expect 78, got %s'%(len(summaryData)) def testExtractDataForPeriod_ConfigLimitedIds(self): """ TestTopCrashesBySignature.testExtractDataForPeriod_ConfigLimitedIds """ global me cursor = self.connection.cursor() idCache = cia.IdCache(cursor) minStamp,maxStamp,data = self.prepareExtractDataForPeriod('date_processed',23,5) keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data]) keySSet = set([x for x in keySet if 5 != x[0]]) mminStamp = minStamp - dt.timedelta(days=1) mmaxStamp = maxStamp + dt.timedelta(days=1) self.prepareConfigForPeriod(keySet,mminStamp,mmaxStamp) tc = topcrasher.TopCrashesBySignature(me.config) tc.dateColumnName = 'date_processed' summaryData = {} tc.extractDataForPeriod(minStamp,maxStamp,summaryData) #assert 110 == len(summaryData), 'Regression test only, value not calculated. Expect 110, got %s'%(len(summaryData)) #Lars buildid assert 99 == len(summaryData), 'Regression test only, value not calculated. Expect 99, got %s'%(len(summaryData)) cursor.execute("DELETE from product_visibility") self.connection.commit() self.prepareConfigForPeriod(keySSet,mminStamp,mmaxStamp) summaryData = {} tc.extractDataForPeriod(minStamp,maxStamp,summaryData) #assert 96 == len(summaryData), 'Regression test only, value not calculated. Expect 96, got %s'%(len(summaryData)) #Lars buildid assert 85 == len(summaryData), 'Regression test only, value not calculated. Expect 85, got %s'%(len(summaryData)) def testStoreFacts(self): """ TestTopCrashesBySignature.testStoreFacts """ global me cursor = self.connection.cursor() idCache = cia.IdCache(cursor) minStamp, maxStamp, data = self.prepareExtractDataForPeriod('date_processed',31,5) # full set of keys keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data]) tc = topcrasher.TopCrashesBySignature(me.config) tc.dateColumnName = 'date_processed' beforeDate = minStamp-dt.timedelta(minutes=30) afterDate = maxStamp+dt.timedelta(minutes=60) self.prepareConfigForPeriod(keySet,beforeDate,afterDate) summaryCrashes = {} summaryCrashes = tc.extractDataForPeriod(beforeDate,afterDate,summaryCrashes) countSql = "SELECT count(*) from top_crashes_by_signature" cursor.execute(countSql) self.connection.commit() gotCount = cursor.fetchone()[0] assert 0 == gotCount, 'but got a count of %s'%gotCount #Try with none tc.storeFacts([],"the test interval") cursor.execute(countSql) self.connection.commit() gotCount = cursor.fetchone()[0] assert 0 == gotCount, 'but got %s'%gotCount #Try with three smKeys = summaryCrashes.keys()[:3] sum = dict((k,summaryCrashes[k]) for k in smKeys) stime = dt.datetime(2009,9,8,7,30) fifteen = dt.timedelta(minutes=15) cd = tc.fixupCrashData(sum,stime,fifteen) tc.storeFacts(cd,"interval three") cursor.execute(countSql) self.connection.commit() gotCount = cursor.fetchone()[0] assert 3 == gotCount, 'but got %s'%gotCount cursor.execute("SELECT window_end, window_size from top_crashes_by_signature") got = cursor.fetchall() self.connection.commit() expect = [(stime,fifteen),] * gotCount assert expect == got, 'but got %s'%(got) cursor.execute("DELETE FROM top_crashes_by_signature") self.connection.commit() #Try with about half smKeys = summaryCrashes.keys()[-130:] sum = dict((k,summaryCrashes[k]) for k in smKeys) cd = tc.fixupCrashData(sum,dt.datetime(2009,9,8,7,30),fifteen) tc.storeFacts(cd,"interval about half") cursor.execute(countSql) self.connection.commit() gotCount = cursor.fetchone()[0] #assert 130 == gotCount, 'but got %s'%gotCount #Lars buildid assert 129 == gotCount, 'but got %s'%gotCount cursor.execute("SELECT window_end,window_size from top_crashes_by_signature") got = cursor.fetchall() self.connection.commit() expect = [(stime,fifteen),] * gotCount assert expect == got, 'but got %s'%(got) cursor.execute("DELETE FROM top_crashes_by_signature") self.connection.commit() #Try with all of them cd = tc.fixupCrashData(summaryCrashes,dt.datetime(2009,9,8,7,30),fifteen) tc.storeFacts(cd,"all of them") cursor.execute(countSql) self.connection.commit() gotCount = cursor.fetchone()[0] assert len(summaryCrashes) == gotCount, 'but got %s'%gotCount cursor.execute("SELECT window_end, window_size from top_crashes_by_signature") got = cursor.fetchall() self.connection.commit() expect = [(stime,fifteen),] * gotCount assert expect == got, 'but got %s'%(got) def testChangeWindowSize(self): """ TestTopCrashesBySignature.testChangeWindowSize """ global me cursor = self.connection.cursor() idCache = cia.IdCache(cursor,logger=me.logger) minStamp, maxStamp, data = self.prepareExtractDataForPeriod('date_processed',31,5) # full set of keys bogusEnd = dt.datetime(2007,12,30) bogusSize = dt.timedelta(minutes=20) configBegin = minStamp - dt.timedelta(hours=1) configEnd = maxStamp + dt.timedelta(hours=1) keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data]) self.prepareConfigForPeriod(keySet,configBegin,configEnd) getSql = "SELECT window_end,window_size FROM top_crashes_by_signature ORDER BY id DESC LIMIT 1" cursor.execute(getSql) got = cursor.fetchone() assert None == got, 'Expect no data at this point, but %s'%(str(got)) self.connection.rollback() inSql = "INSERT INTO top_crashes_by_signature (window_end,window_size) VALUES (%s,%s)" cursor.execute(inSql,(bogusEnd,bogusSize)) self.connection.commit() cursor.execute(getSql) got = cursor.fetchone() self.connection.rollback() assert bogusSize == got[1],'But it was %s'%(str(got)) config = copy.copy(me.config) config['startDate'] = dt.datetime(2008,1,2,1) config['endDate'] = dt.datetime(2008,1,2,2) config['startWindow'] = config['startDate'] config['deltaWindow'] = dt.timedelta(minutes=10) tc = topcrasher.TopCrashesBySignature(config) tc.processDateInterval() cursor.execute(getSql) got = cursor.fetchone() self.connection.rollback() assert config['deltaWindow'] == got[1], 'But it was %s'%(str(got)) config['startDate'] = config['endDate'] config['endDate'] = dt.datetime(2008,1,2,3) config['startWindow'] = config['startDate'] config['deltaWindow'] = dt.timedelta(minutes=60) tc = topcrasher.TopCrashesBySignature(config) tc.processDateInterval() cursor.execute(getSql) got = cursor.fetchone() self.connection.rollback() assert config['deltaWindow'] == got[1], 'But it was %s'%(str(got)) def testProcessDateInterval(self): """ TestTopCrashesBySignature.testProcessDateInterval """ global me cursor = self.connection.cursor() idCache = cia.IdCache(cursor) # test a small full set of keys. Since we have already tested each component, that should be enough minStamp, maxStamp, data = self.prepareExtractDataForPeriod('date_processed',31,5) # full set of keys configBegin = minStamp - dt.timedelta(hours=1) configEnd = maxStamp + dt.timedelta(hours=1) keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data]) self.prepareConfigForPeriod(keySet,configBegin,configEnd) tc = topcrasher.TopCrashesBySignature(me.config) config = copy.copy(me.config) config['startWindow'] = dt.datetime(2008,1,2,0,0) config['deltaWindow'] = dt.timedelta(minutes=12) config['startDate'] = dt.datetime(2008,1,2,0,0) config['endDate'] = dt.datetime(2008,1,6,5,0) tc = topcrasher.TopCrashesBySignature(config) # first assure that we have a clean playing field countSql = "SELECT count(*) from top_crashes_by_signature" cursor.execute(countSql) self.connection.commit() gotCount = cursor.fetchone()[0] assert 0 == gotCount, "but top_crashes_by_signature isn't empty. Had %s rows"%gotCount sDate = minStamp.replace(hour=0,minute=0,second=0,microsecond=0) maxMin = 30 dHour = dt.timedelta(hours=0) if maxStamp.minute >= 30: maxMin = 0 dHour = dt.timedelta(hours=1) eDate = maxStamp.replace(minute=maxMin,second=0,microsecond=0)+dHour tc.dateColumnName= 'client_crash_date' tc.processDateInterval(startDate=sDate, endDate=eDate, dateColumnName='date_processed') assert 'client_crash_date' == tc.dateColumnName cursor.execute(countSql) self.connection.commit() gotCount = cursor.fetchone()[0] me.logger.debug("DEBUG testProcessIntervals after count top_crashes_by_signature = %s",gotCount) expect = 137 #expect = 155 # Lars buildid assert expect == gotCount, 'Regression test only, value not calculated. Expect %s, got %s'%(expect,gotCount)
class TestNamedCursor(TestCase): def setUp(self): global me if not me: createMe() self.testDB = TestDB() self.testDB.removeDB(me.config, me.logger) self.testDB.createDB(me.config, me.logger) #self.connection = psycopg2.connect(me.dsn) self.connection = me.database.connection() def tearDown(self): global me self.testDB.removeDB(me.config, me.logger) self.connection.close() def reportDataGenerator(self, sizePerDay, numDays): idGen = dbtestutil.moreUuid() initialDate = dt.datetime(2008, 1, 1, 1, 1, 1, 1, tzinfo=UTC) currentDate = dt.datetime(2008, 1, 1, 1, 1, 1, 1, tzinfo=UTC) milli5 = dt.timedelta(milliseconds=5) milli10 = dt.timedelta(milliseconds=10) buildStrings = ['200712312355', '200712302355', '200712292355'] buildDates = [ dt.datetime(2007, 12, 31, 23, 55, tzinfo=UTC), dt.datetime(2007, 12, 30, 23, 55, tzinfo=UTC), dt.datetime(2007, 12, 29, 23, 55, tzinfo=UTC) ] osNameVersions = [ ('Windows NT', '6.6.6'), ('Windows NT', '6.6.6'), ('Windows', 'v.v.v'), ('Windows', 'v.v.v'), ('Windows', 'v.v.v'), ('Windows', 'v.v.v'), ('Mac OS X', '10.5.5'), ('Mac OS X', '10.5.6'), ('Mac OS X', '10.5.6'), ('Linux', '10.10.10'), ('Linux', '10.10.11'), ] insData = [] for dummyDays in range(numDays): count = 0 while count < sizePerDay: os_name, os_version = osNameVersions[count % len(osNameVersions)] data = { 'uuid': idGen.next(), 'client_crash_date': currentDate, 'date_processed': currentDate + milli5, 'product': 'foxy', 'version': '3.6.9b2', 'build': buildStrings[count % len(buildStrings)], 'url': 'http://www.woo.wow/weee', 'install_age': 89000, 'last_crash': 0, 'uptime': 88000, 'email': None, 'os_name': os_name, 'os_version': os_version, #'build_date': buildDates[count%len(buildDates)], 'user_comments': 'oh help', 'app_notes': "", 'distributor': "", 'distributor_version': "", } insData.append(data) if not count % (3): currentDate += milli10 count += 1 currentDate = initialDate + dt.timedelta(days=1) cursor = self.connection.cursor() addReportData(cursor, insData) def build1000(self): #testBuild1000(self): self.reportDataGenerator(1000, 10) ncursor = self.connection.cursor('myCursor') ncursor.execute( 'SELECT id,uuid,client_crash_date,date_processed from reports') try: while True: data = ncursor.fetchmany(512) if data and len(data): print data[0][0], len(data) else: print "Broke: %s" % (data) break finally: self.connection.commit()
class TestNamedCursor(unittest.TestCase): def setUp(self): global me if not me: createMe() self.testDB = TestDB() self.testDB.removeDB(me.config, me.logger) self.testDB.createDB(me.config, me.logger) #self.connection = psycopg2.connect(me.dsn) self.connection = me.database.connection() def tearDown(self): global me self.testDB.removeDB(me.config,me.logger) self.connection.close() def reportDataGenerator(self,sizePerDay,numDays): idGen = dbtestutil.moreUuid() initialDate = dt.datetime(2008,1,1,1,1,1,1,tzinfo=UTC) currentDate = dt.datetime(2008,1,1,1,1,1,1,tzinfo=UTC) milli5 = dt.timedelta(milliseconds=5) milli10 = dt.timedelta(milliseconds=10) buildStrings = ['200712312355','200712302355','200712292355'] buildDates = [dt.datetime(2007,12,31,23,55,tzinfo=UTC),dt.datetime(2007,12,30,23,55,tzinfo=UTC),dt.datetime(2007,12,29,23,55,tzinfo=UTC)] osNameVersions = [('Windows NT','6.6.6'),('Windows NT','6.6.6'),('Windows','v.v.v'),('Windows','v.v.v'),('Windows','v.v.v'),('Windows','v.v.v'), ('Mac OS X','10.5.5'),('Mac OS X','10.5.6'),('Mac OS X','10.5.6'), ('Linux','10.10.10'),('Linux','10.10.11'), ] insData = [] for dummyDays in range(numDays): count = 0 while count < sizePerDay: os_name,os_version = osNameVersions[count % len(osNameVersions)] data = { 'uuid':idGen.next(), 'client_crash_date':currentDate, 'date_processed': currentDate+milli5, 'product': 'foxy', 'version': '3.6.9b2', 'build': buildStrings[count%len(buildStrings)], 'url':'http://www.woo.wow/weee', 'install_age':89000, 'last_crash':0, 'uptime':88000, 'email':None, 'os_name': os_name, 'os_version': os_version, #'build_date': buildDates[count%len(buildDates)], 'user_comments': 'oh help', 'app_notes':"", 'distributor':"", 'distributor_version':"", } insData.append(data) if not count%(3): currentDate += milli10 count += 1 currentDate = initialDate+dt.timedelta(days=1) cursor = self.connection.cursor() addReportData(cursor,insData) def build1000(self): #testBuild1000(self): self.reportDataGenerator(1000,10) ncursor = self.connection.cursor('myCursor') ncursor.execute('SELECT id,uuid,client_crash_date,date_processed from reports') try: while True: data = ncursor.fetchmany(512) if data and len(data): print data[0][0],len(data) else: print "Broke: %s"%(data) break finally: self.connection.commit()
if errno.EEXIST == x.errno: pass else: raise f = open(me.logFilePathname, 'w') f.close() fileLog = logging.FileHandler(me.logFilePathname, 'a') fileLog.setLevel(logging.DEBUG) fileLogFormatter = logging.Formatter( '%(asctime)s %(levelname)s - %(message)s') fileLog.setFormatter(fileLogFormatter) me.logger = logging.getLogger("db_test") me.logger.addHandler(fileLog) me.logger.setLevel(logging.DEBUG) me.dsn = "host=%s dbname=%s user=%s password=%s" % ( me.config.databaseHost, me.config.databaseName, me.config.databaseUserName, me.config.databasePassword) me.testDB = TestDB() def teardown_module(): pass tptCreationSql = "CREATE TABLE %s (id serial not null primary key, date_col timestamp with time zone)" partitionSql = "CREATE table %%(partitionName)s () INHERITS (%s)" tptPartitionCreationSqlTemplate = partitionSql % 'tpt' tpt3PartitionCreationSqlTemplate = partitionSql % 'tpt3' class TPT(schema.PartitionedTable): def __init__(self, logger): super(TPT, self).__init__(
class TestMtbf(unittest.TestCase): def setUp(self): global me self.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing MTBF') myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir':'%s'%myDir} for i in self.config: try: self.config[i] = self.config.get(i)%(replDict) except: pass self.logger = TestingLogger(me.fileLogger) self.connection = psycopg2.connect(me.dsn) cursor = self.connection.cursor() self.testDB = TestDB() self.testDB.removeDB(self.config,self.logger) self.testDB.createDB(self.config,self.logger) self.prods = ['zorro','vogel','lizz',] self.oss = ['OSX','LOX','WOX',] self.productDimData = [] # filled in by fillMtbfTables def tearDown(self): self.testDB.removeDB(self.config,self.logger) self.logger.clear() def fillMtbfTables(self,cursor): """ Need some data to test with. Here's where we make it out of whole cloth... """ # (id),product,version,os,release : what product is it, by id self.productDimData = [ [self.prods[p],'%s.1.%s'%(p,r), self.oss[o], 'beta-%s'%r] for p in range(2) for o in range(2) for r in range(1,4) ] cursor.executemany('INSERT into productdims (product,version,os_name,release) values (%s,%s,%s,%s)',self.productDimData) cursor.connection.commit() cursor.execute("SELECT id, product,version, os_name, release from productdims") productDimData = cursor.fetchall() cursor.connection.commit() self.baseDate = dt.date(2008,1,1) self.intervals = { '0.1.1':(self.baseDate ,self.baseDate+dt.timedelta(days=30)), '0.1.2':(self.baseDate + dt.timedelta(days=10),self.baseDate + dt.timedelta(days=40)), '0.1.3':(self.baseDate + dt.timedelta(days=20),self.baseDate + dt.timedelta(days=50)), '1.1.1':(self.baseDate + dt.timedelta(days=10),self.baseDate + dt.timedelta(days=40)), '1.1.2':(self.baseDate + dt.timedelta(days=20),self.baseDate + dt.timedelta(days=50)), '1.1.3':(self.baseDate + dt.timedelta(days=30),self.baseDate + dt.timedelta(days=60)), } # processing days are located at and beyond the extremes of the full range, and # at some interior points, midway between each pair of interior points # layout is: (a date, the day-offset from baseDate, the expected resulting [ids]) PDindexes = [-1,0,5,10,15,25,35,45,55,60,61] productsInProcessingDay = [ [], # -1, [1,4],# 0, [1,4],# 5, [1,2,4,5,7,10],# 10, [1,2,4,5,7,10],# 15, [1,2,3,4,5,6,7,8,10,11],# 25, [2,3,5,6,7,8,9,10,11,12],# 35, [3,6,8,9,11,12],# 45, [9,12],# 55, [9,12],# 60, [],# 61, ] self.processingDays = [ (self.baseDate+dt.timedelta(days=PDindexes[x]),PDindexes[x],productsInProcessingDay[x]) for x in range(len(PDindexes))] # (id), productdims_id, start_dt, end_dt : Date-interval when product is interesting configData =[ (x[0],self.intervals[x[2]][0],self.intervals[x[2]][1] ) for x in productDimData ] cursor.executemany('insert into mtbfconfig (productdims_id,start_dt,end_dt) values(%s,%s,%s)',configData) cursor.connection.commit() self.expectedFacts = { # key is offset from baseDate # value is array of (productDims_id,day,avg_seconds,report_count,count(distinct(user)) # This data WAS NOT CALCULATED BY HAND: The test was run once with prints in place # and that output was encoded here. As of 2009-Feb, count of unique users is always 0 -1: [], 0: [ (1, dt.date(2008,1,1), 5, 6, 0), (4, dt.date(2008,1,1), 20, 6, 0), ], 5: [ (1, dt.date(2008,1,6), 5, 6, 0), (4, dt.date(2008,1,6), 20, 6, 0), ], 10: [ (1, dt.date(2008,1,11), 5, 6, 0), (2, dt.date(2008,1,11), 10, 6, 0), (4, dt.date(2008,1,11), 20, 6, 0), (5, dt.date(2008,1,11), 25, 6, 0), (7, dt.date(2008,1,11), 35, 6, 0), (10, dt.date(2008,1,11), 50, 6, 0), ], 15: [ (1, dt.date(2008,1,16), 5, 6, 0), (2, dt.date(2008,1,16), 10, 6, 0), (4, dt.date(2008,1,16), 20, 6, 0), (5, dt.date(2008,1,16), 25, 6, 0), (7, dt.date(2008,1,16), 35, 6, 0), (10, dt.date(2008,1,16), 50, 6, 0), ], 25: [ (1, dt.date(2008,1,26), 5, 6, 0), (2, dt.date(2008,1,26), 10, 6, 0), (3, dt.date(2008,1,26), 15, 6, 0), (4, dt.date(2008,1,26), 20, 6, 0), (5, dt.date(2008,1,26), 25, 6, 0), (6, dt.date(2008,1,26), 30, 6, 0), (7, dt.date(2008,1,26), 35, 6, 0), (8, dt.date(2008,1,26), 40, 6, 0), (10, dt.date(2008,1,26), 50, 6, 0), (11, dt.date(2008,1,26), 55, 6, 0), ], 35: [ (2, dt.date(2008,2,5), 10, 6, 0), (3, dt.date(2008,2,5), 15, 6, 0), (5, dt.date(2008,2,5), 25, 6, 0), (6, dt.date(2008,2,5), 30, 6, 0), (7, dt.date(2008,2,5), 35, 6, 0), (8, dt.date(2008,2,5), 40, 6, 0), (9, dt.date(2008,2,5), 45, 6, 0), (10, dt.date(2008,2,5), 50, 6, 0), (11, dt.date(2008,2,5), 55, 6, 0), (12, dt.date(2008,2,5), 60, 6, 0), ], 45: [ (3, dt.date(2008,2,15), 15, 6, 0), (6, dt.date(2008,2,15), 30, 6, 0), (8, dt.date(2008,2,15), 40, 6, 0), (9, dt.date(2008,2,15), 45, 6, 0), (11, dt.date(2008,2,15), 55, 6, 0), (12, dt.date(2008,2,15), 60, 6, 0), ], 55: [ (9, dt.date(2008,2,25), 45, 6, 0), (12, dt.date(2008,2,25), 60, 6, 0), ], 60: [ (9, dt.date(2008,3,1), 45, 6, 0), (12, dt.date(2008,3,1), 60, 6, 0), ], 61: [], } def fillReports(self,cursor): """fill enough data to test mtbf behavior: - AVG(uptime); COUNT(date_processed); COUNT(DISTINCT(user_id)) """ self.fillMtbfTables(cursor) # prime the pump sql = 'insert into reports (uuid, uptime, date_processed,product,version,os_name) values(%s,%s,%s,%s,%s,%s)' processTimes = ['00:00:00','05:00:00','10:00:00','15:00:00','20:00:00','23:59:59'] uptimes = [5*x for x in range(1,15)] data = [] uuidGen = dbtestutil.moreUuid() uptimeIndex = 0 for product in self.productDimData: uptime = uptimes[uptimeIndex%len(uptimes)] uptimeIndex += 1 for d,off,ig in self.processingDays: for pt in processTimes: dp = "%s %s"%(d.isoformat(),pt) tup = (uuidGen.next(), uptime,dp,product[0],product[1],product[2]) data.append(tup) cursor.executemany(sql,data) cursor.connection.commit() # ========================================================================== # def testCalculateMtbf(self): """ testCalculateMtbf(self): slow(1) check that we get the expected data. This is NOT hand-calculated, just a regression check """ cursor = self.connection.cursor() self.fillReports(cursor) sql = 'select productdims_id,day,avg_seconds,report_count,unique_users from mtbffacts WHERE day = %s' self.connection.commit() for pd in self.processingDays: self.config.processingDay = pd[0].isoformat() mtbf.calculateMtbf(self.config, self.logger) cursor.execute(sql,(pd[0].isoformat(),)) data = cursor.fetchall() self.connection.commit() expected = set(self.expectedFacts[pd[1]]) got = set(data) assert expected==got, 'Expected: %s\nGot: %s'%(expected,got) #end of loop through processingDay # ========================================================================== # def testGetProductsToUpdate(self): """ testGetProductsToUpdate(self): check that we get the appropriate list of products when: - we have none (on either end of the range) - we have only one or several check that we correctly log when there are no products in range """ cursor = self.connection.cursor() self.fillMtbfTables(cursor) for d in self.processingDays: self.config.processingDay = d[0].isoformat() self.logger.clear() products = mtbf.getProductsToUpdate(cursor,self.config,self.logger) self.connection.commit() if d[1] in (-1,61): # be sure that when appropriate we log a warning about no configured products assert 2 == len(self.logger) assert logging.WARNING == self.logger.levels[1] assert 'Currently there are no MTBF products configured' == self.logger.buffer[1] else: # ignore the expected logging: It could change. Catch unexpected logging calls assert len(d[2])+2 == len(self.logger) INFO = 0 DBG = 0 oth = 0 for i in self.logger.levels: if logging.INFO == i: INFO += 1 elif logging.DEBUG == i: DBG += 1 else: oth += 1 # Don't care about DBG or INFO counts, except we expect no other #assert len(d[2])+1 == INFO, 'expected %d, but %s\n%s'%(len(d[2])+1,INFO,str(self.logger)) #assert 1 == DBG assert 0 == oth pids = set([x.dimensionId for x in products]) expected = set(d[2]) # This is the meat of the test assert expected == pids, 'Expected %s, got %s'%(expected,pids) # ========================================================================== # def testGetWhereClauseFor(self): """ testGetWhereClauseFor(self): check that we correctly handle the 'ALL' product check that we correctly order and truncate version,product,os_name """ class P: pass p = P() p.product = 'ALL' assert '' == mtbf.getWhereClauseFor(p) p.product = 'product' assert_raises(AttributeError,mtbf.getWhereClauseFor,p) p.version = 'version' assert_raises(AttributeError,mtbf.getWhereClauseFor,p) p.os_name='os_name' expected = " AND version = 'version' AND product = 'product' AND substr(os_name, 1, 3) = 'os_name' " assert expected == mtbf.getWhereClauseFor(p), 'but "%s"'%(mtbf.getWhereClauseFor(p)) # ========================================================================== # def testClassProductDimension(self): """ testClassProductDimension(self): check that we handle config with correct or greater items in config """ assert_raises(IndexError,mtbf.ProductDimension,[]) assert_raises(IndexError,mtbf.ProductDimension,[1]) assert_raises(IndexError,mtbf.ProductDimension,[1,2]) assert_raises(IndexError,mtbf.ProductDimension,[1,2,3]) self.logger.clear() assert_raises(IndexError,mtbf.ProductDimension,[1,2,3,4]) config = [999,'umlaut',3.14,'OX','lemme go','toomuch'] pd = mtbf.ProductDimension(config,self.logger) assert 999 == pd.dimensionId assert 'umlaut' == pd.product assert 3.14 == pd.version assert 'OX' == pd.os_name assert 'lemme go' == pd.release assert 5 == len(pd.__dict__), "Assure nobody adds another dimension element without updating tests" assert 0 == len(self.logger),'but logger:\n%s'%self.logger
class TestTopCrashesByUrl: def setUp(self): global me self.logger = me.logger #self.connection = psycopg2.connect(me.dsn) self.connection = me.database.connection() self.testDB = TestDB() self.testDB.removeDB(me.config,self.logger) self.testDB.createDB(me.config,self.logger) def tearDown(self): self.testDB.removeDB(me.config,self.logger) socorro_cia.clearCache() def testConstructor(self): """testTopCrashesByUrl:TestTopCrashesByUrl.testConstructor(self)""" global me config = copy.copy(me.config) t = tcbu.TopCrashesByUrl(config) assert 1 == t.configContext['minimumHitsPerUrl'] assert 500 == t.configContext['maximumUrls'] assert tcbu.logger.name assert 'date_processed' == t.configContext.get('dateColumn') config = copy.copy(me.config) config['minimumHitsPerUrl'] = 2 config['maximumUrls'] = 100 config['logger'] = self.logger t = tcbu.TopCrashesByUrl(config) assert 2 == t.configContext.get('minimumHitsPerUrl') assert 100 == t.configContext.get('maximumUrls') assert self.logger == t.configContext.get('logger') halfDay = datetime.timedelta(hours=12) config = copy.copy(me.config) t = tcbu.TopCrashesByUrl(config, minimumHitsPerUrl=3, maximumUrls=50,deltaWindow =halfDay) assert 3 == t.configContext.get('minimumHitsPerUrl') assert 50 == t.configContext.get('maximumUrls') assert halfDay == t.configContext.get('deltaWindow') def testCountCrashesByUrlInWindow(self): """testTopCrashesByUrl:TestTopCrashesByUrl.testCountCrashesByUrlInWindow(self):""" global me cursor = self.connection.cursor() dbtutil.fillReportsTable(cursor,createUrls=True,multiplier=2,signatureCount=83) # just some data... self.connection.commit() config = copy.copy(me.config) # test /w/ 'normal' params t = tcbu.TopCrashesByUrl(config) startWindow = datetime.datetime(2008,1,1) deltaWindow = datetime.timedelta(days=1) endWindow = startWindow + deltaWindow data = t.countCrashesByUrlInWindow(startWindow = startWindow, deltaWindow = deltaWindow) # the following are JUST regression tests: The data has been only very lightly examined to be sure it makes sense. assert 24 == len(data), 'This is (just) a regression test. Did you change the data somehow? (%s)'%len(data) for d in data: assert 1 == d[0] # test /w/ small maximumUrls config = copy.copy(me.config) t = tcbu.TopCrashesByUrl(config, maximumUrls=50) data = t.countCrashesByUrlInWindow(startWindow = datetime.datetime(2008,1,1), endWindow = endWindow) assert 24 == len(data), 'This is (just) a regression test. Did you change the data somehow? (%s)'%len(data) for d in data: assert 1 == d[0] # test /w/ minimumHitsPerUrl larger config = copy.copy(me.config) t = tcbu.TopCrashesByUrl(config, minimumHitsPerUrl=2) data = t.countCrashesByUrlInWindow(startWindow = datetime.datetime(2008,1,1),endWindow = endWindow) assert 24 == len(data), len(data) for d in data: assert 1 == d[0] # test /w/ shorter window config = copy.copy(me.config) halfDay = datetime.timedelta(hours=12) t = tcbu.TopCrashesByUrl(config, deltaWindow = halfDay) data = t.countCrashesByUrlInWindow(startWindow = datetime.datetime(2008,1,1)) assert 12 == len(data), 'This is (just) a regression test. Did you change the data somehow? (%s)'%len(data) for d in data: assert 1 == d[0] # test a different day, to be sure we get different data config = copy.copy(me.config) t = tcbu.TopCrashesByUrl(config) data = t.countCrashesByUrlInWindow(startWindow = datetime.datetime(2008,1,11),deltaWindow=deltaWindow) assert 57 == len(data), 'This is (just) a regression test. Did you change the data somehow? (%s)'%len(data) for d in data[:3]: assert 2 == d[0] for d in data[3:]: assert 1 == d[0] def testSaveData(self): """ testTopCrashesByUrl:TestTopCrashesByUrl.testSaveData(slow=2) This is a reasonably realistic amount of time (about 1.5 seconds) to handle about 150 reports """ global me cursor = self.connection.cursor() ## Set up dbtutil.fillReportsTable(cursor,createUrls=True,multiplier=2,signatureCount=83) # just some data... self.connection.commit() # ... now assure some duplicates sqls = "SELECT uuid, client_crash_date, install_age, last_crash, uptime, date_processed, success, signature, url, product, version, os_name, os_version from reports where date_processed >= '2008-01-01' and date_processed < '2008-01-02' LIMIT 4" cursor.execute(sqls) self.connection.rollback() # db not altered rows3 = cursor.fetchall() add11 = datetime.timedelta(seconds=1,microseconds=1000) addData = [] for i in range(3): r = list(rows3[i]) r[0] = r[0].replace('-dead-','-f00f-') r[1] += add11 r[2] += 1 r[3] += 1 r[7] = rows3[i+1][7] addData.append(r) r[0] = r[0].replace('-f00f-','-fead-') r[1] += add11 r[2] += 1 r[3] += 1 r[7] = 'js_blatherskytes' addData.append(r) sqli = """INSERT INTO reports (uuid, client_crash_date, install_age, last_crash, uptime, date_processed, success, signature, url, product, version, os_name, os_version) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""" addData.extend([ ['b965de73-ae90-b936-deaf-03ae20081225','2007-12-31 23:59:50',9000,110,222,'2008-01-01 11:12:13',True,'UserCallWinProcCheckWow','http://www.mozilla.org/projects/minefield/a','Firefox','3.0.9','Windows NT','5.1.2600 Service Pack 2'], ['b965de73-ae90-b935-deaf-03ae20081225','2007-12-31 23:59:40',9009,220,333,'2008-01-01 11:12:14',True,'UserCallWinProcCheckWow','http://yachats/uncwiki/LarsLocalPortal/b', 'Firefox','3.0.9','Windows NT','5.1.2600 Service Pack 2'], ]) cursor.executemany(sqli,addData) self.connection.commit() config = copy.copy(me.config) startWindow = datetime.datetime(2008,1,1) deltaWindow = datetime.timedelta(days=1) ## On your mark... t = tcbu.TopCrashesByUrl(config) data = t.countCrashesByUrlInWindow(startWindow = startWindow, deltaWindow = deltaWindow) ## assure we have an empty playing field cursor.execute("SELECT COUNT(*) from top_crashes_by_url") self.connection.rollback() assert 0 == cursor.fetchone()[0] ## Call the method t.saveData(startWindow,data) # expect 99 rows cursor.execute("SELECT COUNT(id) from top_crashes_by_url") self.connection.rollback() count = cursor.fetchone()[0] assert 35 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count) # expect 80 distinct urls cursor.execute("SELECT COUNT(distinct urldims_id) from top_crashes_by_url") self.connection.rollback() count = cursor.fetchone()[0] assert 21 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count) cursor.execute("SELECT count from top_crashes_by_url where count > 1 order by count") self.connection.rollback() data = cursor.fetchall() assert [(2,),(2,),(2,)] == data, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(str(data)) cursor.execute("SELECT COUNT(top_crashes_by_url_id) from top_crashes_by_url_signature") self.connection.rollback() count = cursor.fetchone()[0] assert 38 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count) cursor.execute("SELECT COUNT(distinct top_crashes_by_url_id) from top_crashes_by_url_signature") self.connection.rollback() count = cursor.fetchone()[0] assert 35 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count) # Expect 3 rows with sums of 2 and three rows with counts of 2, none with both cursor.execute("SELECT count, COUNT(top_crashes_by_url_id) AS sum FROM top_crashes_by_url_signature GROUP BY top_crashes_by_url_id, count ORDER BY sum DESC, count DESC LIMIT 6") self.connection.rollback() data = cursor.fetchall() assert 6 == len(data) assert [(1, 2L), (1, 2L), (1, 2L), (1, 1L), (1, 1L), (1, 1L)] == data, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(str(data)) def testSaveTruncatedData(self): """ testTopCrashesByUrl:TestTopCrashesByUrl.testSaveTruncatedData(slow=2) This is a reasonably realistic amount of time (about 1.5 seconds) to handle about 150 reports """ global me cursor = self.connection.cursor() ## Set up dbtutil.fillReportsTable(cursor,createUrls=True,multiplier=2,signatureCount=83) # just some data... self.connection.commit() # ... now assure some duplicates sqls = "SELECT uuid, client_crash_date, install_age, last_crash, uptime, date_processed, success, signature, url, product, version, os_name, os_version from reports where date_processed >= '2008-01-01' and date_processed < '2008-01-02' LIMIT 4" cursor.execute(sqls) self.connection.rollback() # db not altered rows3 = cursor.fetchall() add11 = datetime.timedelta(seconds=1,microseconds=1000) addData = [] for i in range(3): r = list(rows3[i]) r[0] = r[0].replace('-dead-','-f00f-') r[1] += add11 r[2] += 1 r[3] += 1 r[7] = rows3[i+1][7] addData.append(r) r[0] = r[0].replace('-f00f-','-fead-') r[1] += add11 r[2] += 1 r[3] += 1 r[7] = 'js_blatherskytes' addData.append(r) sqli = """INSERT INTO reports (uuid, client_crash_date, install_age, last_crash, uptime, date_processed, success, signature, url, product, version, os_name, os_version) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""" addData.extend([ ['b965de73-ae90-b936-deaf-03ae20081225','2007-12-31 23:59:50',9000,110,222,'2008-01-01 11:12:13',True,'UserCallWinProcCheckWow','http://www.mozilla.org/projects/minefield/a','Firefox','3.0.9','Windows NT','5.1.2600 Service Pack 2'], ['b965de73-ae90-b935-deaf-03ae20081225','2007-12-31 23:59:40',9009,220,333,'2008-01-01 11:12:14',True,'UserCallWinProcCheckWow','http://yachats/uncwiki/LarsLocalPortal/b', 'Firefox','3.0.9','Windows NT','5.1.2600 Service Pack 2'], ]) cursor.executemany(sqli,addData) self.connection.commit() config = copy.copy(me.config) startWindow = datetime.datetime(2008,1,1) deltaWindow = datetime.timedelta(days=1) ## On your mark... t = tcbu.TopCrashesByUrl(config,truncateUrlLength=25) data = t.countCrashesByUrlInWindow(startWindow = startWindow, deltaWindow = deltaWindow) ## assure we have an empty playing field cursor.execute("SELECT COUNT(*) from top_crashes_by_url") self.connection.rollback() assert 0 == cursor.fetchone()[0] ## Call the method t.saveData(startWindow,data) # expect 99 rows cursor.execute("SELECT COUNT(id) from top_crashes_by_url") self.connection.rollback() count = cursor.fetchone()[0] assert 30 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count) # expect 80 distinct urls cursor.execute("SELECT COUNT(distinct urldims_id) from top_crashes_by_url") self.connection.rollback() count = cursor.fetchone()[0] assert 17 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count) cursor.execute("SELECT count from top_crashes_by_url where count > 1 order by count") self.connection.rollback() data = cursor.fetchall() assert [(2,), (2,), (2,), (2,), (2,), (4,)] == data, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(str(data)) cursor.execute("SELECT COUNT(top_crashes_by_url_id) from top_crashes_by_url_signature") self.connection.rollback() count = cursor.fetchone()[0] assert 38 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count) cursor.execute("SELECT COUNT(distinct top_crashes_by_url_id) from top_crashes_by_url_signature") self.connection.rollback() count = cursor.fetchone()[0] assert 30 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count) # Expect 3 rows with sums of 2 and three rows with counts of 2, none with both cursor.execute("SELECT count, COUNT(top_crashes_by_url_id) AS sum FROM top_crashes_by_url_signature GROUP BY top_crashes_by_url_id, count ORDER BY sum DESC, count DESC LIMIT 6") self.connection.rollback() data = cursor.fetchall() assert 6 == len(data) assert [(1, 4L), (1, 2L), (1, 2L), (1, 2L), (1, 2L), (1, 2L)] == data, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(str(data)) def testProcessDateInterval(self): """ testTopCrashesByUrl:TestTopCrashesByUrl.testProcessDateInterval(slow=7) Takes a long time, first to set up the data (about 1.5 seconds), then to process it several times """ global me cursor = self.connection.cursor() config = copy.copy(me.config) ## Set up dbtutil.fillReportsTable(cursor,createUrls=True,multiplier=2,signatureCount=83) # just some data... self.connection.commit() t = tcbu.TopCrashesByUrl(config) ## assure we have an empty playing field cursor.execute("SELECT COUNT(*) from top_crashes_by_url") self.connection.rollback() assert 0 == cursor.fetchone()[0] t.processDateInterval(startDate = datetime.datetime(2008,1,1), endDate=datetime.datetime(2008,1,6)) cursor.execute("SELECT COUNT(id) from top_crashes_by_url") self.connection.rollback() count = cursor.fetchone()[0] assert 35 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count) cursor.execute("SELECT COUNT(*) from top_crashes_by_url_signature") self.connection.rollback() count = cursor.fetchone()[0] assert 38 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count) cursor.execute("delete from top_crashes_by_url; delete from top_crashes_by_url_signature") self.connection.commit() t = tcbu.TopCrashesByUrl(copy.copy(me.config)) t.processDateInterval(startDate = datetime.datetime(2008,1,4), endDate=datetime.datetime(2008,1,8)) cursor.execute("SELECT COUNT(id) from top_crashes_by_url") self.connection.rollback() count = cursor.fetchone()[0] assert 31 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count) cursor.execute("SELECT COUNT(*) from top_crashes_by_url_signature") self.connection.rollback() count = cursor.fetchone()[0] assert 32 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count) cursor.execute("delete from top_crashes_by_url; delete from top_crashes_by_url_signature") self.connection.commit() t = tcbu.TopCrashesByUrl(copy.copy(me.config)) t.processDateInterval(startDate = datetime.datetime(2008,1,1), endDate=datetime.datetime(2008,3,3)) cursor.execute("SELECT COUNT(id) from top_crashes_by_url") self.connection.rollback() count = cursor.fetchone()[0] assert 483 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count) cursor.execute("SELECT COUNT(*) from top_crashes_by_url_signature") self.connection.rollback() count = cursor.fetchone()[0] assert 514 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)