Ejemplo n.º 1
0
def main():
  logger = logging.getLogger("topcrashes_summary")
  logger.setLevel(logging.WARNING)
  
  stderrLog = logging.StreamHandler()
  stderrLog.setLevel(logging.WARNING)
  stderrLogFormatter = logging.Formatter('%(asctime)s %(levelname)s - %(message)s')
  stderrLog.setFormatter(stderrLogFormatter)
  logger.addHandler(stderrLog)

  kwargs = {}
  for i in sys.argv[1:]:
    if i.startswith('-h') or i.startswith('--he'):
      help()
      sys.exit(0)
    j = i
    if i.startswith('-'):
      j = i.lstrip('-')
    if '=' in j:
      name,value = (s.strip() for s in j.split('='))
      kwargs[name] = value
    else:
      print >> sys.stderr,"Ignoring unkown argument '%s'"%(i)
  sys.argv = sys.argv[:1]
  config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Create Database')
  config.update(kwargs)
  testDB = TestDB()
  testDB.removeDB(config,logger)
  testDB.createDB(config,logger)
Ejemplo n.º 2
0
 def setUp(self):
     global me
     if not me:
         createMe()
     self.testDB = TestDB()
     self.testDB.removeDB(me.config, me.logger)
     self.testDB.createDB(me.config, me.logger)
     #self.connection = psycopg2.connect(me.dsn)
     self.connection = me.database.connection()
Ejemplo n.º 3
0
def createData(config,logger):
  # Now do the work in several steps
  connection = sdatabase.Database(config).connection()
  #connection = psycopg2.connect("host=%(databaseHost)s dbname=%(databaseName)s user=%(databaseUserName)s password=%(databasePassword)s"%config)
  cursor = connection.cursor()
  testDB = TestDB()
  try:
    testDB.removeDB(config,logger)
    if config.get('drop-all'):
      print "Dropped the database tables ..."
      return
    print "Creating the database tables..."
    testDB.createDB(config,logger)
    print "populating the dimensions tables..."
    processingDays,ignore = dbtestutil.fillMtbfTables(cursor,limit=int(config.get('product-os-count')))
    startDays = [x[0] for x in processingDays]
    multiplier = int(config.get('repeat-count'))
    print "populating the reports table (takes about %d seconds)..."%(int(1.7+1.2*multiplier))
    dbtestutil.fillReportsTable(cursor,createUrls=True,doFillMtbfTables=False, numUrls=100, multiplier=multiplier,signatureCount=int(config.get('signature-count')))
    connection.commit()
    extras = []
    print "All done populating the 'raw' data"
    if config.get('mtbf-fill'):
      blurb = ""
      cost = 0.20 + multiplier*0.15
      if cost > 1.0:
        blurb = ("(takes about %2.1f seconds)"%cost)
      print "Filling the time_before_failure table %s..."%blurb # R=1: .35 seconds; 2:0.49s; 3:.064s; 4:.9 ## = .20 + R*.15
      starter = None
      ender = None
      mtbfInstance = mtbf.Mtbf(config,logger)
      for startDay in startDays:
        if not starter: starter = startDay
        ender = startDay
        mtbfInstance.processOneMtbfWindow(processingDay=startDay)
      extras.append(" - Time before fail: for days in %s through %s"%(starter,ender))
    if config.get('sig-fill'):
      print "Filling the top_crashes_by_signature table (takes about %s seconds)..."%(20+11*multiplier) # R=1: 27.3 secs; 2:38.5s; 3=48.3 ##  = 20 +R*11
      tc = topcrasher.TopCrashesBySignature(config)
      tc.processDateInterval(startDate=startDays[0],endDate = startDays[-1])
      extras.append(" - Top crash by sig: for days in %s through %s"%(startDays[0],startDays[-1]))
    if config.get('url-fill'):
      print "Filling the top_crashes_by_url table (takes about %s seconds)..."%(4+multiplier*2) # R=1: 4 secs; 2: 5s, 3: 7.6 ## = 4+R*2
      logger.info("Filling the top_crashes_by_url table (takes about %s seconds)..."%(4+multiplier*2))
      tu = topcrashbyurl.TopCrashesByUrl(config)
      tu.processDateInterval(startDate=startDays[0],endDate = startDays[-1])
      extras.append(" - Top crash by url: for days in %s through %s"%(startDays[0],startDays[-1]))
    print "DONE populating the database tables"
    if extras:
      print "\n".join(extras)
  finally:
    logger.info("All done. Closing connection")
    connection.close()
Ejemplo n.º 4
0
def setup_module():
    global me
    if me:
        return
    me = Me()
    me.testDB = TestDB()
    me.config = configurationManager.newConfiguration(
        configurationModule=testConfig,
        applicationName='TestingCachedIdAccess')
    myDir = os.path.split(__file__)[0]
    if not myDir: myDir = '.'
    replDict = {'testDir': '%s' % myDir}
    for i in me.config:
        try:
            me.config[i] = me.config.get(i) % (replDict)
        except:
            pass
    cia.logger.setLevel(logging.DEBUG)
    me.logFilePathname = me.config.logFilePathname
    logfileDir = os.path.split(me.config.logFilePathname)[0]
    try:
        os.makedirs(logfileDir)
    except OSError, x:
        if errno.EEXIST != x.errno: raise
        f = open(me.config.logFilePathname, 'w')
        f.close()
Ejemplo n.º 5
0
 def setUp(self):
   global me
   self.logger = me.logger
   #self.connection = psycopg2.connect(me.dsn)
   self.connection = me.database.connection()
   self.testDB = TestDB()
   self.testDB.removeDB(me.config,self.logger)
   self.testDB.createDB(me.config,self.logger)
Ejemplo n.º 6
0
 def setUp(self):
   global me
   if not me:
     createMe()
   self.connection = me.database.connection()
   #self.connection = psycopg2.connect(me.dsn)
   self.testDB = TestDB()
   self.testDB.removeDB(me.config, me.logger)
   self.testDB.createDB(me.config, me.logger)
   dbtestutil.fillDimsTables(self.connection.cursor())
   self.connection.commit()
Ejemplo n.º 7
0
def main():
    logger = logging.getLogger("topcrashes_summary")
    logger.setLevel(logging.WARNING)

    stderrLog = logging.StreamHandler()
    stderrLog.setLevel(logging.WARNING)
    stderrLogFormatter = logging.Formatter(
        '%(asctime)s %(levelname)s - %(message)s')
    stderrLog.setFormatter(stderrLogFormatter)
    logger.addHandler(stderrLog)

    kwargs = {}
    for i in sys.argv[1:]:
        if i.startswith('-h') or i.startswith('--he'):
            help()
            sys.exit(0)
        j = i
        if i.startswith('-'):
            j = i.lstrip('-')
        if '=' in j:
            name, value = (s.strip() for s in j.split('='))
            kwargs[name] = value
        else:
            print >> sys.stderr, "Ignoring unkown argument '%s'" % (i)
    sys.argv = sys.argv[:1]
    config = configurationManager.newConfiguration(
        configurationModule=testConfig, applicationName='Create Database')
    config.update(kwargs)
    testDB = TestDB()
    testDB.removeDB(config, logger)
    testDB.createDB(config, logger)
Ejemplo n.º 8
0
 def setUp(self):
     global me
     # config gets messed up by some tests. Use this one by preference
     self.config = configurationManager.newConfiguration(
         configurationModule=testConfig,
         applicationName='Testing Postgresql Utils')
     for i in self.config:
         try:
             self.config[i] = self.config.get(i) % (replDict)
         except:
             pass
     self.connection = psycopg2.connect(me.dsn)
     self.testDB = TestDB()
Ejemplo n.º 9
0
  def setUp(self):
    global me
    self.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing MTBF')
    
    myDir = os.path.split(__file__)[0]
    if not myDir: myDir = '.'
    replDict = {'testDir':'%s'%myDir}
    for i in self.config:
      try:
        self.config[i] = self.config.get(i)%(replDict)
      except:
        pass
    self.logger = TestingLogger(me.fileLogger)
    self.connection = psycopg2.connect(me.dsn)
    cursor = self.connection.cursor()
    self.testDB = TestDB()
    self.testDB.removeDB(self.config,self.logger)

    self.testDB.createDB(self.config,self.logger)
    self.prods = ['zorro','vogel','lizz',]
    self.oss = ['OSX','LOX','WOX',]
    self.productDimData = [] # filled in by fillMtbfTables
Ejemplo n.º 10
0
class TestTopCrashesBySignature(unittest.TestCase):
  def setUp(self):
    global me
    if not me:
      createMe()
    self.connection = me.database.connection()
    #self.connection = psycopg2.connect(me.dsn)
    self.testDB = TestDB()
    self.testDB.removeDB(me.config, me.logger)
    self.testDB.createDB(me.config, me.logger)
    dbtestutil.fillDimsTables(self.connection.cursor())
    self.connection.commit()

  def tearDown(self):
    global me
    self.testDB.removeDB(me.config,me.logger)
    cia.clearCache()
    self.connection.close()

  def prepareConfigForPeriod(self,idkeys,startDate,endDate):
    """enter a row for each idkey, start and end date. Do it repeatedly if you need different ones"""
    cfgKeys = set([x[0] for x in idkeys])
    cfgdata = [[x,startDate,endDate] for x in cfgKeys]
    self.connection.cursor().execute("delete from product_visibility")
    sql = "INSERT INTO product_visibility (productdims_id,start_date,end_date) VALUES (%s,%s,%s)"
    self.connection.cursor().executemany(sql,cfgdata)
    self.connection.commit()

  def prepareExtractDataForPeriod(self, columnName, sizePerDay=2, numDays=5):
    """Put some crash data into reports table, return their extrema for the columnName specified"""
    global me
    cursor = self.connection.cursor()
    data = makeReportData(sizePerDay,numDays)
    self.connection.commit()
    # find the actual min and max dates
    minStamp = dt.datetime(2100,11,11)
    maxStamp = dt.datetime(1900,11,11)
    for d in data:
      ds = d[columnName]
      if ds < minStamp: minStamp = ds
      if ds > maxStamp: maxStamp = ds
    addReportData(cursor,data)
    self.connection.commit()
    return minStamp,maxStamp,data

  def testConstructor(self):
    """
    TestTopCrashesBySignature.testConstructor
    """
    global me
    for reqd in ["databaseHost","databaseName","databaseUserName","databasePassword",]:
      cc = copy.copy(me.config)
      del(cc[reqd])
      assert_raises(SystemExit,topcrasher.TopCrashesBySignature,cc)
    bogusMap = {'databaseHost':me.config.databaseHost,'databaseName':me.config.databaseName,'databaseUserName':'******','databasePassword':me.config.databasePassword}
    assert_raises(SystemExit,topcrasher.TopCrashesBySignature,bogusMap)

    #check without specific config items
    now = dt.datetime.now()
    tc = topcrasher.TopCrashesBySignature(me.config)
    expectedStart = now.replace(hour=0,minute=0,second=0,microsecond=0) - tc.configContext.get('initialDeltaDate',dt.timedelta(days=4))
    expectedEnd = now.replace(hour=0,minute=0,second=0,microsecond=0)
    assert expectedStart == tc.startDate
    assert tc.endDate <= now, 'But NOT %s <= %s'%(tc.endDate,now)
    assert dt.timedelta(days=4) <= tc.deltaDate, 'But got %s'%(tc.deltaDate)
    # Check for defaults. If the defaults are hard-code changed, this block needs equivalent changes
    assert 'date_processed' == tc.dateColumnName
    # end Check for defaults

    # check with solo startDate, endDate or deltaDate
    config = copy.copy(me.config)
    config['startDate'] = dt.datetime(2009,01,01,0,0,1)
    assert_raises(SystemExit,topcrasher.TopCrashesBySignature,config)

    config = copy.copy(me.config)
    config['endDate'] = dt.datetime(2009,01,01,0,0,1)
    assert_raises(SystemExit,topcrasher.TopCrashesBySignature,config)

    config = copy.copy(me.config)
    config['deltaDate'] = dt.timedelta(days=3)
    assert_raises(SystemExit,topcrasher.TopCrashesBySignature,config)

  def testExtractDataForPeriod_ByDateProcessed(self):
    """
    TestTopCrashesBySignature.testExtractDataForPeriod_ByDateProcessed(self):
     - Check that we get nothing for a period outside our data
     - Check that we get expected count for period inside our data
     - Check that we get half-open interval (don't get data exactly at last moment)
     - Check that we get all for period that surrounds our data
     - Check that we get nothing for start == end
     - Check that we get one moment for end = start+microsecond
     - Check that we raise ValueError for period with dates reversed
    """
    global me
    minStamp,maxStamp,data = self.prepareExtractDataForPeriod('date_processed')
    cursor = self.connection.cursor()
    idCache = cia.IdCache(cursor)
    keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data])
    mminStamp = minStamp - dt.timedelta(days=1)
    mmaxStamp = maxStamp + dt.timedelta(days=1)
    self.prepareConfigForPeriod(keySet,mminStamp,mmaxStamp)

    tc = topcrasher.TopCrashesBySignature(me.config)
    tc.dateColumnName = 'date_processed'
    # we should get nothing if we are outside our data
    start = minStamp - dt.timedelta(days=10)
    end = start + dt.timedelta(days=1)
    crashData = {}
    tc.extractDataForPeriod(start,end,crashData)
    assert {} == crashData,'Expected {}, got %s'%crashData

    # We should get exactly one day's worth
    start = minStamp
    end = start + dt.timedelta(days=1)
    tc.extractDataForPeriod(start,end,crashData)
    crashDataCount = 0 # all three known
    for k,d in crashData.items():
      crashDataCount += d['count']
    assert 2 == crashDataCount, 'Expected 2, got %d'%crashDataCount

    # we should get all but the last 2 (or one) item: Half-open interval
    start = minStamp
    end = maxStamp
    crashData = {}
    tc.extractDataForPeriod(start,end,crashData)
    crashDataCount = 0 # all three known
    for k,d in crashData.items():
      crashDataCount += d['count']
    assert crashDataCount == 9, 'Expected 9. Got %s'%crashDataCount

    # we should get all the data
    start = minStamp
    end = maxStamp+dt.timedelta(milliseconds=1)
    crashData = {}
    tc.extractDataForPeriod(start,end,crashData)
    crashDataCount = 0 # all three known
    for k,d in crashData.items():
      crashDataCount += d['count']
    assert crashDataCount == 10, 'Expected 10. Got %s'%crashDataCount

    # we should get nothing if start == end (half open interval that stops juuuust before the start)
    start = minStamp
    end = minStamp
    crashData = {}
    tc.extractDataForPeriod(start,end,crashData)
    assert crashData == {}, 'But got %s'%crashData

    # we should get precise stamp's data if end = start+teeny-tiny-epsilon
    start = minStamp
    end = start+dt.timedelta(microseconds=1)
    crashData = {}
    tc.extractDataForPeriod(start,end,crashData)
    crashDataCount = 0 # all three known
    for k,d in crashData.items():
      crashDataCount += d['count']
    assert crashDataCount == 1, 'Expected 1. Got %s'%crashDataCount

    # We should throw if start > end
    assert_raises(ValueError,tc.extractDataForPeriod,maxStamp,minStamp,crashData)

  def testExtractDataForPeriod_ByClientCrashDate(self):
    """
    TestTopCrashesBySignature.testExtractDataForPeriod_ByClientCrashDate(self):
     - Check that we get nothing for a period outside our data
     - Check that we get expected count for period inside our data
     - Check that we get half-open interval (don't get data exactly at last moment)
     - Check that we get all for period that surrounds our data
     - Check that we raise ValueError for period with dates reversed
    """
    global me
    minStamp,maxStamp,data = self.prepareExtractDataForPeriod('client_crash_date')
    cursor = self.connection.cursor()
    idCache = cia.IdCache(cursor)
    keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data])
    mminStamp = minStamp - dt.timedelta(days=1)
    mmaxStamp = maxStamp + dt.timedelta(days=1)
    self.prepareConfigForPeriod(keySet,mminStamp,mmaxStamp)
    tc = topcrasher.TopCrashesBySignature(me.config)
    tc.dateColumnName = 'client_crash_date'
    # we should get nothing if we are outside our data
    start = minStamp - dt.timedelta(days=10)
    end = start + dt.timedelta(days=1)
    crashData = {}
    tc.extractDataForPeriod(start,end,crashData)
    assert {} == crashData,'Expected {}, got %s'%crashData

    # We should get exactly one day's worth
    start = minStamp
    end = start + dt.timedelta(days=1)
    tc.extractDataForPeriod(start,end,crashData)
    crashDataCount0 = 0 # all three known
    for k,d in crashData.items():
      crashDataCount0 += d['count']
    assert 2 == crashDataCount0, 'Expected 2, got %d'%crashDataCount0

    # we should get all but the last 2 (or one) item: Half-open interval
    start = minStamp
    end = maxStamp
    crashData = {}
    tc.extractDataForPeriod(start,end,crashData)
    crashDataCount0 = 0 # all three known
    for k,d in crashData.items():
      crashDataCount0 += d['count']
    assert crashDataCount0 == 9, 'Expected 9. Got %s'%crashDataCount0

    # we should get all the data
    start = minStamp
    end = maxStamp+dt.timedelta(milliseconds=1)
    crashData = {}
    tc.extractDataForPeriod(start,end,crashData)
    crashDataCount0 = 0 # all three known
    for k,d in crashData.items():
      crashDataCount0 += d['count']
    assert crashDataCount0 == 10, 'Expected 10. Got %s'%crashDataCount0

    # we should get nothing if start == end (half open interval that stops juuuust before the start)
    start = minStamp
    end = minStamp
    crashData = {}
    tc.extractDataForPeriod(start,end,crashData)
    assert crashData == {}, 'But got %s'%crashData

    # we should get precise stamp's data if end = start+teeny-tiny-epsilon
    start = minStamp
    end = start+dt.timedelta(microseconds=1)
    crashData = {}
    tc.extractDataForPeriod(start,end,crashData)
    crashDataCount0 = 0 # all three known
    for k,d in crashData.items():
      crashDataCount0 += d['count']
    assert crashDataCount0 == 1, 'Expected 1. Got %s'%crashDataCount0

    # We should throw if start > end
    assert_raises(ValueError,tc.extractDataForPeriod,maxStamp,minStamp,crashData)

  def testFixupCrashData(self):
    """
    TestTopCrashesBySignature.testFixupCrashData(self):(slow=1)
      - create a bunch of data, count it in the test, put it into the reports table, then
        let TopCrashesBySignature get it back out, and assert that fixup gets the same count
    """
    global me
    cursor = self.connection.cursor()
    idCache = cia.IdCache(cursor)
    data = makeReportData(211,5) # 211 is prime, which means we should get a nice spread of data
    keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data])
    #keySet = set([(d['productdims_id'],d['osdims_id']) for d in data])
    expect = {}
    newCount = 0
    oldCount = 0
    for d in data:
      # Keys are always signature,product,os
      key =   (d['signature'],idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version']))
      if d['version'][-3:] == 'pre': continue    # Lars buildid
      hang_count = 0
      plugin_count = 0
      if d['hangid'] != '':
        hang_count = 1
      if d['process_type'] == 'plugin':
        plugin_count = 1;
      try:
        expect[key]['count'] += 1
        expect[key]['uptime'] += d['uptime']
        expect[key]['hang_count'] += hang_count
        expect[key]['plugin_count'] += plugin_count
        oldCount += 1
      except:
        expect[key] = {}
        expect[key]['count'] = 1
        expect[key]['uptime'] = d['uptime']
        expect[key]['hang_count'] = hang_count
        expect[key]['plugin_count'] = plugin_count
        newCount += 1
    addReportData(cursor,data)
    self.connection.commit()
    tc = topcrasher.TopCrashesBySignature(me.config)
    tc.dateColumnName = 'date_processed'
    baseDate = testBaseDate
    start = baseDate - dt.timedelta(days=1)
    end = baseDate + dt.timedelta(days=6)
    self.prepareConfigForPeriod(keySet,start,end)

    summaryCrashes = {}
    tc.extractDataForPeriod(start,end,summaryCrashes)
    # assert expect == summaryCrashes, 'Oops. You will need to debug on this. Sorry. Try commenting this line and un-commenting the next bunch'
    # the next few lines amount to "assert expect == summaryCrashes" but with more useful error detail
    # check that we have a chance of equality
    assert len(expect) == len(summaryCrashes), 'Expected equal lengths but expected:%s versus got:%s'%(len(expect),len(summaryCrashes))
    getter = itemgetter('count') # ,'uptime') #,'hang_count','plugin_count')
    # check that everything in what we got is expected
    for k,v in summaryCrashes.items():
      assert getter(expect[k]) == getter(v), 'for key %s: Expected %s but got %s'%(k,getter(expect[k]),getter(v))
    # check that everything we expect is in what we got
    for k,v in expect.items():
      assert getter(v) == getter(summaryCrashes[k]),  'for key %s: Expected %s but got %s'%(k,getter(v), getter(summaryCrashes[k]))

    result = tc.fixupCrashData(summaryCrashes,baseDate,dt.timedelta(minutes=19))
    result.sort(key=itemgetter('count'),reverse=True)
    resultx = tc.fixupCrashData(expect,baseDate,dt.timedelta(minutes=19))
    resultx.sort(key=itemgetter('count'),reverse=True)
    assert result == resultx
    ## The next few lines show more detail if there's a problem. Comment above, uncomment below
    # misscount = fullcount = 0
    # if result != resultx:
    #   assert len(result) == len(resultx), 'expected len: %s, got %s'%(len(result), len(resultx))
    #   for i in range(len(result)):
    #     r = result[i]
    #     rx = resultx[i]
    #     print "%s" % (r)
    #     print "%s" % (rx)
    #     assert len(r) == len(rx), 'at loop %s: expected len: %s got %s'(i,len(r),len(rx))
    #     for j in range(len(r)):
    #       fullcount += 1
    #       if r[j] != rx[j]:
    #         print "unequal at loop %s, item %s:\n  %s\n  %s"%(i,j,r[j],rx[j])
    #         misscount += 1
    #   print "Unequal for %s of %s (%3.1f%%) rows"%(misscount,fullcount,100.0*(1.0*misscount/fullcount))
    # assert 0 == misscount, "They weren't equal. Look nearby for details"
    expectedAll = set(['xpcom_core.dll@0x31b7a', 'js_Interpret', '_PR_MD_SEND', 'nsFormFillController::SetPopupOpen', 'nsAutoCompleteController::ClosePopup','morkRowObject::CloseRowObject(morkEnv*)', 'nsContainerFrame::ReflowChild(nsIFrame*, nsPresContext*, nsHTMLReflowMetrics&, nsHTMLReflowState const&, int, int, unsigned int, unsigned int&, nsOverflowContinuationTracker*)'])

    gotAll = set([ x['signature'] for x in result])
    assert expectedAll == gotAll , 'ex-got:%s\ngot-ex:%s'%(expectedAll-gotAll,gotAll-expectedAll)

  def testExtractDataForPeriod_ConfigLimitedDates(self):
    """
    TestTopCrashesBySignature.testExtractDataForPeriod_ConfigLimitedDates(self)
    """
    global me
    minStamp,maxStamp,data = self.prepareExtractDataForPeriod('date_processed',23,5)
    cursor = self.connection.cursor()
    idCache = cia.IdCache(cursor)
    keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data])
    keySSet = set([x for x in keySet if 5 != x[0]])
    mminStamp = minStamp - dt.timedelta(days=1)
    mmaxStamp = maxStamp + dt.timedelta(days=1)
    self.prepareConfigForPeriod(keySet,mminStamp,mmaxStamp)
    tc = topcrasher.TopCrashesBySignature(me.config)
    tc.dateColumnName = 'date_processed'

    summaryData = {}
    tc.extractDataForPeriod(minStamp,maxStamp,summaryData)
    #assert 110 == len(summaryData), 'Regression test only, value not calculated. Expect 110, got %s'%(len(summaryData)) #Lars buildid
    assert 99 == len(summaryData), 'Regression test only, value not calculated. Expect 99, got %s'%(len(summaryData))

    cursor.execute("DELETE from product_visibility")
    self.connection.commit()

    mminStamp = minStamp + dt.timedelta(days=1)
    mmaxStamp = maxStamp + dt.timedelta(days=1)
    self.prepareConfigForPeriod(keySet,mminStamp,mmaxStamp)
    summaryData = {}
    tc.extractDataForPeriod(minStamp,maxStamp,summaryData)
    #assert 87 == len(summaryData), 'Regression test only, value not calculated. Expect 87, got %s'%(len(summaryData)) #Lars buildid
    assert 78 == len(summaryData), 'Regression test only, value not calculated. Expect 78, got %s'%(len(summaryData))


  def testExtractDataForPeriod_ConfigLimitedIds(self):
    """
    TestTopCrashesBySignature.testExtractDataForPeriod_ConfigLimitedIds
    """
    global me
    cursor = self.connection.cursor()
    idCache = cia.IdCache(cursor)
    minStamp,maxStamp,data = self.prepareExtractDataForPeriod('date_processed',23,5)
    keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data])
    keySSet = set([x for x in keySet if 5 != x[0]])
    mminStamp = minStamp - dt.timedelta(days=1)
    mmaxStamp = maxStamp + dt.timedelta(days=1)
    self.prepareConfigForPeriod(keySet,mminStamp,mmaxStamp)
    tc = topcrasher.TopCrashesBySignature(me.config)
    tc.dateColumnName = 'date_processed'

    summaryData = {}
    tc.extractDataForPeriod(minStamp,maxStamp,summaryData)
    #assert 110 == len(summaryData), 'Regression test only, value not calculated. Expect 110, got %s'%(len(summaryData)) #Lars buildid
    assert 99 == len(summaryData), 'Regression test only, value not calculated. Expect 99, got %s'%(len(summaryData))

    cursor.execute("DELETE from product_visibility")
    self.connection.commit()

    self.prepareConfigForPeriod(keySSet,mminStamp,mmaxStamp)
    summaryData = {}
    tc.extractDataForPeriod(minStamp,maxStamp,summaryData)
    #assert 96 == len(summaryData), 'Regression test only, value not calculated. Expect 96, got %s'%(len(summaryData)) #Lars buildid
    assert 85 == len(summaryData), 'Regression test only, value not calculated. Expect 85, got %s'%(len(summaryData))

  def testStoreFacts(self):
    """
    TestTopCrashesBySignature.testStoreFacts
    """
    global me
    cursor = self.connection.cursor()
    idCache = cia.IdCache(cursor)
    minStamp, maxStamp, data = self.prepareExtractDataForPeriod('date_processed',31,5) # full set of keys
    keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data])
    tc = topcrasher.TopCrashesBySignature(me.config)
    tc.dateColumnName = 'date_processed'
    beforeDate = minStamp-dt.timedelta(minutes=30)
    afterDate = maxStamp+dt.timedelta(minutes=60)
    self.prepareConfigForPeriod(keySet,beforeDate,afterDate)
    summaryCrashes = {}
    summaryCrashes = tc.extractDataForPeriod(beforeDate,afterDate,summaryCrashes)

    countSql = "SELECT count(*) from top_crashes_by_signature"
    cursor.execute(countSql)
    self.connection.commit()
    gotCount = cursor.fetchone()[0]
    assert 0 == gotCount, 'but got a count of %s'%gotCount

    #Try with none
    tc.storeFacts([],"the test interval")
    cursor.execute(countSql)
    self.connection.commit()
    gotCount = cursor.fetchone()[0]
    assert 0 == gotCount, 'but got %s'%gotCount

    #Try with three
    smKeys = summaryCrashes.keys()[:3]
    sum = dict((k,summaryCrashes[k]) for k in smKeys)
    stime = dt.datetime(2009,9,8,7,30)
    fifteen = dt.timedelta(minutes=15)
    cd = tc.fixupCrashData(sum,stime,fifteen)
    tc.storeFacts(cd,"interval three")
    cursor.execute(countSql)
    self.connection.commit()
    gotCount = cursor.fetchone()[0]
    assert 3 == gotCount, 'but got %s'%gotCount
    cursor.execute("SELECT window_end, window_size from top_crashes_by_signature")
    got = cursor.fetchall()
    self.connection.commit()
    expect = [(stime,fifteen),] * gotCount
    assert expect == got, 'but got %s'%(got)

    cursor.execute("DELETE FROM top_crashes_by_signature")
    self.connection.commit()

    #Try with about half
    smKeys = summaryCrashes.keys()[-130:]
    sum = dict((k,summaryCrashes[k]) for k in smKeys)
    cd = tc.fixupCrashData(sum,dt.datetime(2009,9,8,7,30),fifteen)
    tc.storeFacts(cd,"interval about half")
    cursor.execute(countSql)
    self.connection.commit()
    gotCount = cursor.fetchone()[0]
    #assert 130 == gotCount, 'but got %s'%gotCount #Lars buildid
    assert 129 == gotCount, 'but got %s'%gotCount
    cursor.execute("SELECT window_end,window_size from top_crashes_by_signature")
    got = cursor.fetchall()
    self.connection.commit()
    expect = [(stime,fifteen),] * gotCount
    assert expect == got, 'but got %s'%(got)

    cursor.execute("DELETE FROM top_crashes_by_signature")
    self.connection.commit()

    #Try with all of them
    cd = tc.fixupCrashData(summaryCrashes,dt.datetime(2009,9,8,7,30),fifteen)
    tc.storeFacts(cd,"all of them")
    cursor.execute(countSql)
    self.connection.commit()
    gotCount = cursor.fetchone()[0]
    assert len(summaryCrashes) == gotCount, 'but got %s'%gotCount
    cursor.execute("SELECT window_end, window_size from top_crashes_by_signature")
    got = cursor.fetchall()
    self.connection.commit()
    expect = [(stime,fifteen),] * gotCount
    assert expect == got, 'but got %s'%(got)

  def testChangeWindowSize(self):
    """
    TestTopCrashesBySignature.testChangeWindowSize
    """
    global me
    cursor = self.connection.cursor()
    idCache = cia.IdCache(cursor,logger=me.logger)
    minStamp, maxStamp, data = self.prepareExtractDataForPeriod('date_processed',31,5) # full set of keys
    bogusEnd = dt.datetime(2007,12,30)
    bogusSize = dt.timedelta(minutes=20)
    configBegin = minStamp - dt.timedelta(hours=1)
    configEnd = maxStamp + dt.timedelta(hours=1)
    keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data])
    self.prepareConfigForPeriod(keySet,configBegin,configEnd)
    getSql = "SELECT window_end,window_size FROM top_crashes_by_signature ORDER BY id DESC LIMIT 1"
    cursor.execute(getSql)
    got = cursor.fetchone()
    assert None == got, 'Expect no data at this point, but %s'%(str(got))
    self.connection.rollback()
    inSql = "INSERT INTO top_crashes_by_signature (window_end,window_size) VALUES (%s,%s)"
    cursor.execute(inSql,(bogusEnd,bogusSize))
    self.connection.commit()
    cursor.execute(getSql)
    got = cursor.fetchone()
    self.connection.rollback()
    assert bogusSize == got[1],'But it was %s'%(str(got))
    config = copy.copy(me.config)
    config['startDate'] = dt.datetime(2008,1,2,1)
    config['endDate'] = dt.datetime(2008,1,2,2)
    config['startWindow'] = config['startDate']
    config['deltaWindow'] = dt.timedelta(minutes=10)
    tc = topcrasher.TopCrashesBySignature(config)
    tc.processDateInterval()
    cursor.execute(getSql)
    got = cursor.fetchone()
    self.connection.rollback()
    assert config['deltaWindow'] == got[1], 'But it was %s'%(str(got))

    config['startDate'] = config['endDate']
    config['endDate'] = dt.datetime(2008,1,2,3)
    config['startWindow'] = config['startDate']
    config['deltaWindow'] = dt.timedelta(minutes=60)
    tc = topcrasher.TopCrashesBySignature(config)
    tc.processDateInterval()
    cursor.execute(getSql)
    got = cursor.fetchone()
    self.connection.rollback()
    assert config['deltaWindow'] == got[1], 'But it was %s'%(str(got))

  def testProcessDateInterval(self):
    """
    TestTopCrashesBySignature.testProcessDateInterval
    """
    global me
    cursor = self.connection.cursor()
    idCache = cia.IdCache(cursor)

    # test a small full set of keys. Since we have already tested each component, that should be enough
    minStamp, maxStamp, data = self.prepareExtractDataForPeriod('date_processed',31,5) # full set of keys
    configBegin = minStamp - dt.timedelta(hours=1)
    configEnd = maxStamp + dt.timedelta(hours=1)
    keySet = set([(idCache.getProductId(d['product'],d['version']),idCache.getOsId(d['os_name'],d['os_version'])) for d in data])
    self.prepareConfigForPeriod(keySet,configBegin,configEnd)
    tc = topcrasher.TopCrashesBySignature(me.config)
    config = copy.copy(me.config)
    config['startWindow'] = dt.datetime(2008,1,2,0,0)
    config['deltaWindow'] = dt.timedelta(minutes=12)
    config['startDate'] =  dt.datetime(2008,1,2,0,0)
    config['endDate'] = dt.datetime(2008,1,6,5,0)
    tc = topcrasher.TopCrashesBySignature(config)

    # first assure that we have a clean playing field
    countSql = "SELECT count(*) from top_crashes_by_signature"
    cursor.execute(countSql)
    self.connection.commit()
    gotCount = cursor.fetchone()[0]
    assert 0 == gotCount, "but top_crashes_by_signature isn't empty. Had %s rows"%gotCount

    sDate = minStamp.replace(hour=0,minute=0,second=0,microsecond=0)
    maxMin = 30
    dHour = dt.timedelta(hours=0)
    if maxStamp.minute >= 30:
      maxMin = 0
      dHour = dt.timedelta(hours=1)
    eDate = maxStamp.replace(minute=maxMin,second=0,microsecond=0)+dHour
    tc.dateColumnName= 'client_crash_date'
    tc.processDateInterval(startDate=sDate, endDate=eDate, dateColumnName='date_processed')
    assert 'client_crash_date' == tc.dateColumnName

    cursor.execute(countSql)
    self.connection.commit()
    gotCount = cursor.fetchone()[0]
    me.logger.debug("DEBUG testProcessIntervals after count top_crashes_by_signature = %s",gotCount)
    expect = 137
    #expect = 155 # Lars buildid
    assert expect == gotCount, 'Regression test only, value not calculated. Expect %s, got %s'%(expect,gotCount)
Ejemplo n.º 11
0
class TestNamedCursor(TestCase):
    def setUp(self):
        global me
        if not me:
            createMe()
        self.testDB = TestDB()
        self.testDB.removeDB(me.config, me.logger)
        self.testDB.createDB(me.config, me.logger)
        #self.connection = psycopg2.connect(me.dsn)
        self.connection = me.database.connection()

    def tearDown(self):
        global me
        self.testDB.removeDB(me.config, me.logger)
        self.connection.close()

    def reportDataGenerator(self, sizePerDay, numDays):
        idGen = dbtestutil.moreUuid()
        initialDate = dt.datetime(2008, 1, 1, 1, 1, 1, 1, tzinfo=UTC)
        currentDate = dt.datetime(2008, 1, 1, 1, 1, 1, 1, tzinfo=UTC)
        milli5 = dt.timedelta(milliseconds=5)
        milli10 = dt.timedelta(milliseconds=10)
        buildStrings = ['200712312355', '200712302355', '200712292355']
        buildDates = [
            dt.datetime(2007, 12, 31, 23, 55, tzinfo=UTC),
            dt.datetime(2007, 12, 30, 23, 55, tzinfo=UTC),
            dt.datetime(2007, 12, 29, 23, 55, tzinfo=UTC)
        ]
        osNameVersions = [
            ('Windows NT', '6.6.6'),
            ('Windows NT', '6.6.6'),
            ('Windows', 'v.v.v'),
            ('Windows', 'v.v.v'),
            ('Windows', 'v.v.v'),
            ('Windows', 'v.v.v'),
            ('Mac OS X', '10.5.5'),
            ('Mac OS X', '10.5.6'),
            ('Mac OS X', '10.5.6'),
            ('Linux', '10.10.10'),
            ('Linux', '10.10.11'),
        ]
        insData = []
        for dummyDays in range(numDays):
            count = 0
            while count < sizePerDay:
                os_name, os_version = osNameVersions[count %
                                                     len(osNameVersions)]
                data = {
                    'uuid': idGen.next(),
                    'client_crash_date': currentDate,
                    'date_processed': currentDate + milli5,
                    'product': 'foxy',
                    'version': '3.6.9b2',
                    'build': buildStrings[count % len(buildStrings)],
                    'url': 'http://www.woo.wow/weee',
                    'install_age': 89000,
                    'last_crash': 0,
                    'uptime': 88000,
                    'email': None,
                    'os_name': os_name,
                    'os_version': os_version,
                    #'build_date': buildDates[count%len(buildDates)],
                    'user_comments': 'oh help',
                    'app_notes': "",
                    'distributor': "",
                    'distributor_version': "",
                }
                insData.append(data)
                if not count % (3):
                    currentDate += milli10
                count += 1
            currentDate = initialDate + dt.timedelta(days=1)
        cursor = self.connection.cursor()
        addReportData(cursor, insData)

    def build1000(self):  #testBuild1000(self):
        self.reportDataGenerator(1000, 10)
        ncursor = self.connection.cursor('myCursor')
        ncursor.execute(
            'SELECT id,uuid,client_crash_date,date_processed from reports')
        try:
            while True:
                data = ncursor.fetchmany(512)
                if data and len(data):
                    print data[0][0], len(data)
                else:
                    print "Broke: %s" % (data)
                    break
        finally:
            self.connection.commit()
Ejemplo n.º 12
0
class TestNamedCursor(unittest.TestCase):
  def setUp(self):
    global me
    if not me:
      createMe()
    self.testDB = TestDB()
    self.testDB.removeDB(me.config, me.logger)
    self.testDB.createDB(me.config, me.logger)
    #self.connection = psycopg2.connect(me.dsn)
    self.connection = me.database.connection()

  def tearDown(self):
    global me
    self.testDB.removeDB(me.config,me.logger)
    self.connection.close()

  def reportDataGenerator(self,sizePerDay,numDays):
    idGen = dbtestutil.moreUuid()
    initialDate = dt.datetime(2008,1,1,1,1,1,1,tzinfo=UTC)
    currentDate = dt.datetime(2008,1,1,1,1,1,1,tzinfo=UTC)
    milli5 = dt.timedelta(milliseconds=5)
    milli10 = dt.timedelta(milliseconds=10)
    buildStrings = ['200712312355','200712302355','200712292355']
    buildDates =   [dt.datetime(2007,12,31,23,55,tzinfo=UTC),dt.datetime(2007,12,30,23,55,tzinfo=UTC),dt.datetime(2007,12,29,23,55,tzinfo=UTC)]
    osNameVersions = [('Windows NT','6.6.6'),('Windows NT','6.6.6'),('Windows','v.v.v'),('Windows','v.v.v'),('Windows','v.v.v'),('Windows','v.v.v'),
                      ('Mac OS X','10.5.5'),('Mac OS X','10.5.6'),('Mac OS X','10.5.6'),
                      ('Linux','10.10.10'),('Linux','10.10.11'),
                      ]
    insData = []
    for dummyDays in range(numDays):
      count = 0
      while count < sizePerDay:
        os_name,os_version = osNameVersions[count % len(osNameVersions)]
        data = {
          'uuid':idGen.next(),
          'client_crash_date':currentDate,
          'date_processed': currentDate+milli5,
          'product': 'foxy',
          'version': '3.6.9b2',
          'build': buildStrings[count%len(buildStrings)],
          'url':'http://www.woo.wow/weee',
          'install_age':89000,
          'last_crash':0,
          'uptime':88000,
          'email':None,
          'os_name': os_name,
          'os_version': os_version,
          #'build_date': buildDates[count%len(buildDates)],
          'user_comments': 'oh help',
          'app_notes':"",
          'distributor':"",
          'distributor_version':"",
          }
        insData.append(data)
        if not count%(3):
          currentDate += milli10
        count += 1
      currentDate = initialDate+dt.timedelta(days=1)
    cursor = self.connection.cursor()
    addReportData(cursor,insData)

  def build1000(self): #testBuild1000(self):
    self.reportDataGenerator(1000,10)
    ncursor = self.connection.cursor('myCursor')
    ncursor.execute('SELECT id,uuid,client_crash_date,date_processed from reports')
    try:
      while True:
        data = ncursor.fetchmany(512)
        if data and len(data):
          print data[0][0],len(data)
        else:
          print "Broke: %s"%(data)
          break
    finally:
      self.connection.commit()
        if errno.EEXIST == x.errno: pass
        else: raise
    f = open(me.logFilePathname, 'w')
    f.close()
    fileLog = logging.FileHandler(me.logFilePathname, 'a')
    fileLog.setLevel(logging.DEBUG)
    fileLogFormatter = logging.Formatter(
        '%(asctime)s %(levelname)s - %(message)s')
    fileLog.setFormatter(fileLogFormatter)
    me.logger = logging.getLogger("db_test")
    me.logger.addHandler(fileLog)
    me.logger.setLevel(logging.DEBUG)
    me.dsn = "host=%s dbname=%s user=%s password=%s" % (
        me.config.databaseHost, me.config.databaseName,
        me.config.databaseUserName, me.config.databasePassword)
    me.testDB = TestDB()


def teardown_module():
    pass


tptCreationSql = "CREATE TABLE %s (id serial not null primary key, date_col timestamp with time zone)"
partitionSql = "CREATE table %%(partitionName)s () INHERITS (%s)"
tptPartitionCreationSqlTemplate = partitionSql % 'tpt'
tpt3PartitionCreationSqlTemplate = partitionSql % 'tpt3'


class TPT(schema.PartitionedTable):
    def __init__(self, logger):
        super(TPT, self).__init__(
Ejemplo n.º 14
0
class TestMtbf(unittest.TestCase):
  def setUp(self):
    global me
    self.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing MTBF')
    
    myDir = os.path.split(__file__)[0]
    if not myDir: myDir = '.'
    replDict = {'testDir':'%s'%myDir}
    for i in self.config:
      try:
        self.config[i] = self.config.get(i)%(replDict)
      except:
        pass
    self.logger = TestingLogger(me.fileLogger)
    self.connection = psycopg2.connect(me.dsn)
    cursor = self.connection.cursor()
    self.testDB = TestDB()
    self.testDB.removeDB(self.config,self.logger)

    self.testDB.createDB(self.config,self.logger)
    self.prods = ['zorro','vogel','lizz',]
    self.oss = ['OSX','LOX','WOX',]
    self.productDimData = [] # filled in by fillMtbfTables
    
  def tearDown(self):
    self.testDB.removeDB(self.config,self.logger)
    self.logger.clear()

  def fillMtbfTables(self,cursor):
    """
    Need some data to test with. Here's where we make it out of whole cloth...
    """
    # (id),product,version,os,release : what product is it, by id
    self.productDimData = [ [self.prods[p],'%s.1.%s'%(p,r), self.oss[o], 'beta-%s'%r] for p in range(2) for o in range(2) for r in range(1,4) ]
    cursor.executemany('INSERT into productdims (product,version,os_name,release) values (%s,%s,%s,%s)',self.productDimData)
    cursor.connection.commit()
    cursor.execute("SELECT id, product,version, os_name, release from productdims")
    productDimData = cursor.fetchall()
    cursor.connection.commit()
    self.baseDate = dt.date(2008,1,1)
    self.intervals = {
      '0.1.1':(self.baseDate                        ,self.baseDate+dt.timedelta(days=30)),
      '0.1.2':(self.baseDate + dt.timedelta(days=10),self.baseDate + dt.timedelta(days=40)),
      '0.1.3':(self.baseDate + dt.timedelta(days=20),self.baseDate + dt.timedelta(days=50)),
      '1.1.1':(self.baseDate + dt.timedelta(days=10),self.baseDate + dt.timedelta(days=40)),
      '1.1.2':(self.baseDate + dt.timedelta(days=20),self.baseDate + dt.timedelta(days=50)),
      '1.1.3':(self.baseDate + dt.timedelta(days=30),self.baseDate + dt.timedelta(days=60)),
      }
    # processing days are located at and beyond the extremes of the full range, and
    # at some interior points, midway between each pair of interior points
    # layout is: (a date, the day-offset from baseDate, the expected resulting [ids])
    PDindexes = [-1,0,5,10,15,25,35,45,55,60,61]
    productsInProcessingDay = [
      [], #  -1,
      [1,4],#  0,
      [1,4],#  5,
      [1,2,4,5,7,10],#  10,
      [1,2,4,5,7,10],#  15,
      [1,2,3,4,5,6,7,8,10,11],#  25,
      [2,3,5,6,7,8,9,10,11,12],#  35,
      [3,6,8,9,11,12],#  45,
      [9,12],#  55,
      [9,12],#  60,
      [],#  61,
      ]
    self.processingDays = [ (self.baseDate+dt.timedelta(days=PDindexes[x]),PDindexes[x],productsInProcessingDay[x]) for x in range(len(PDindexes))]
    
    # (id), productdims_id, start_dt, end_dt : Date-interval when product is interesting
    configData =[ (x[0],self.intervals[x[2]][0],self.intervals[x[2]][1] ) for x in productDimData ]
    cursor.executemany('insert into mtbfconfig (productdims_id,start_dt,end_dt) values(%s,%s,%s)',configData)
    cursor.connection.commit()

    self.expectedFacts = {
      # key is offset from baseDate
      # value is array of (productDims_id,day,avg_seconds,report_count,count(distinct(user))
      # This data WAS NOT CALCULATED BY HAND: The test was run once with prints in place
      # and that output was encoded here. As of 2009-Feb, count of unique users is always 0
      -1: [],
      0: [
      (1, dt.date(2008,1,1), 5, 6, 0),
      (4, dt.date(2008,1,1), 20, 6, 0),
      ],
      5: [
      (1, dt.date(2008,1,6), 5, 6, 0),
      (4, dt.date(2008,1,6), 20, 6, 0),
      ],
      10: [
      (1, dt.date(2008,1,11), 5, 6, 0),
      (2, dt.date(2008,1,11), 10, 6, 0),
      (4, dt.date(2008,1,11), 20, 6, 0),
      (5, dt.date(2008,1,11), 25, 6, 0),
      (7, dt.date(2008,1,11), 35, 6, 0),
      (10, dt.date(2008,1,11), 50, 6, 0),
      ],
      15: [
      (1, dt.date(2008,1,16), 5, 6, 0),
      (2, dt.date(2008,1,16), 10, 6, 0),
      (4, dt.date(2008,1,16), 20, 6, 0),
      (5, dt.date(2008,1,16), 25, 6, 0),
      (7, dt.date(2008,1,16), 35, 6, 0),
      (10, dt.date(2008,1,16), 50, 6, 0),
      ],
      25: [
      (1, dt.date(2008,1,26), 5, 6, 0),
      (2, dt.date(2008,1,26), 10, 6, 0),
      (3, dt.date(2008,1,26), 15, 6, 0),
      (4, dt.date(2008,1,26), 20, 6, 0),
      (5, dt.date(2008,1,26), 25, 6, 0),
      (6, dt.date(2008,1,26), 30, 6, 0),
      (7, dt.date(2008,1,26), 35, 6, 0),
      (8, dt.date(2008,1,26), 40, 6, 0),
      (10, dt.date(2008,1,26), 50, 6, 0),
      (11, dt.date(2008,1,26), 55, 6, 0),
      ],
      35: [
      (2, dt.date(2008,2,5), 10, 6, 0),
      (3, dt.date(2008,2,5), 15, 6, 0),
      (5, dt.date(2008,2,5), 25, 6, 0),
      (6, dt.date(2008,2,5), 30, 6, 0),
      (7, dt.date(2008,2,5), 35, 6, 0),
      (8, dt.date(2008,2,5), 40, 6, 0),
      (9, dt.date(2008,2,5), 45, 6, 0),
      (10, dt.date(2008,2,5), 50, 6, 0),
      (11, dt.date(2008,2,5), 55, 6, 0),
      (12, dt.date(2008,2,5), 60, 6, 0),
      ],
      45: [
      (3, dt.date(2008,2,15), 15, 6, 0),
      (6, dt.date(2008,2,15), 30, 6, 0),
      (8, dt.date(2008,2,15), 40, 6, 0),
      (9, dt.date(2008,2,15), 45, 6, 0),
      (11, dt.date(2008,2,15), 55, 6, 0),
      (12, dt.date(2008,2,15), 60, 6, 0),
      ],
      55: [
      (9, dt.date(2008,2,25), 45, 6, 0),
      (12, dt.date(2008,2,25), 60, 6, 0),
      ],
      60: [
      (9, dt.date(2008,3,1), 45, 6, 0),
      (12, dt.date(2008,3,1), 60, 6, 0),
      ],
      61: [],
      }

  def fillReports(self,cursor):
    """fill enough data to test mtbf behavior:
       - AVG(uptime); COUNT(date_processed); COUNT(DISTINCT(user_id))
    """
    self.fillMtbfTables(cursor) # prime the pump
    sql = 'insert into reports (uuid, uptime, date_processed,product,version,os_name) values(%s,%s,%s,%s,%s,%s)'
    processTimes = ['00:00:00','05:00:00','10:00:00','15:00:00','20:00:00','23:59:59']
    uptimes = [5*x for x in range(1,15)]
    data = []
    uuidGen = dbtestutil.moreUuid()
    uptimeIndex = 0
    for product in self.productDimData:
      uptime = uptimes[uptimeIndex%len(uptimes)]
      uptimeIndex += 1
      for d,off,ig in self.processingDays:
        for pt in processTimes:
          dp = "%s %s"%(d.isoformat(),pt)
          tup = (uuidGen.next(), uptime,dp,product[0],product[1],product[2])
          data.append(tup)
    cursor.executemany(sql,data)
    cursor.connection.commit()

  # ========================================================================== #  
  def testCalculateMtbf(self):
    """
    testCalculateMtbf(self): slow(1)
      check that we get the expected data. This is NOT hand-calculated, just a regression check
    """
    cursor = self.connection.cursor()
    self.fillReports(cursor)
    sql = 'select productdims_id,day,avg_seconds,report_count,unique_users from mtbffacts WHERE day = %s'
    self.connection.commit()
    for pd in self.processingDays:
      self.config.processingDay = pd[0].isoformat()
      mtbf.calculateMtbf(self.config, self.logger)
      cursor.execute(sql,(pd[0].isoformat(),))
      data = cursor.fetchall()
      self.connection.commit()
      expected = set(self.expectedFacts[pd[1]])
      got = set(data)
      assert expected==got, 'Expected: %s\nGot: %s'%(expected,got)
    #end of loop through processingDay
  
  # ========================================================================== #  
  def testGetProductsToUpdate(self):
    """
    testGetProductsToUpdate(self):
      check that we get the appropriate list of products when:
       - we have none (on either end of the range)
       - we have only one or several
      check that we correctly log when there are no products in range
    """
    cursor = self.connection.cursor()
    self.fillMtbfTables(cursor)
    for d in self.processingDays:
      self.config.processingDay = d[0].isoformat()
      self.logger.clear()
      products = mtbf.getProductsToUpdate(cursor,self.config,self.logger)
      self.connection.commit()
      if d[1] in (-1,61):
        # be sure that when appropriate we log a warning about no configured products
        assert 2 == len(self.logger)
        assert logging.WARNING == self.logger.levels[1]
        assert 'Currently there are no MTBF products configured' == self.logger.buffer[1]
      else:
        # ignore the expected logging: It could change. Catch unexpected logging calls
        assert len(d[2])+2 == len(self.logger)
        INFO = 0
        DBG = 0
        oth = 0
        for i in self.logger.levels:
          if logging.INFO == i: INFO += 1
          elif logging.DEBUG == i: DBG += 1
          else: oth += 1
        # Don't care about DBG or INFO counts, except we expect no other
        #assert len(d[2])+1 == INFO, 'expected %d, but %s\n%s'%(len(d[2])+1,INFO,str(self.logger))
        #assert 1 == DBG
        assert 0 == oth
      pids = set([x.dimensionId for x in products])
      expected = set(d[2])
      # This is the meat of the test
      assert expected == pids, 'Expected %s, got %s'%(expected,pids)
  
  # ========================================================================== #  
  def testGetWhereClauseFor(self):
    """
    testGetWhereClauseFor(self):
      check that we correctly handle the 'ALL' product
      check that we correctly order and truncate version,product,os_name
    """
    class P:
      pass
    p = P()
    p.product = 'ALL'
    assert '' == mtbf.getWhereClauseFor(p)
    p.product = 'product'
    assert_raises(AttributeError,mtbf.getWhereClauseFor,p)
    p.version = 'version'
    assert_raises(AttributeError,mtbf.getWhereClauseFor,p)
    p.os_name='os_name'
    expected = " AND version = 'version' AND product = 'product' AND substr(os_name, 1, 3) = 'os_name' "
    assert expected == mtbf.getWhereClauseFor(p), 'but "%s"'%(mtbf.getWhereClauseFor(p))

  # ========================================================================== #  
  def testClassProductDimension(self):
    """
    testClassProductDimension(self):
       check that we handle config with correct or greater items in config
    """
    assert_raises(IndexError,mtbf.ProductDimension,[])
    assert_raises(IndexError,mtbf.ProductDimension,[1])
    assert_raises(IndexError,mtbf.ProductDimension,[1,2])
    assert_raises(IndexError,mtbf.ProductDimension,[1,2,3])
    self.logger.clear()
    assert_raises(IndexError,mtbf.ProductDimension,[1,2,3,4])
    config = [999,'umlaut',3.14,'OX','lemme go','toomuch']
    pd = mtbf.ProductDimension(config,self.logger)
    assert 999 == pd.dimensionId
    assert 'umlaut' == pd.product
    assert 3.14 == pd.version
    assert 'OX' == pd.os_name
    assert 'lemme go' == pd.release
    assert 5 == len(pd.__dict__), "Assure nobody adds another dimension element without updating tests"
    assert 0 == len(self.logger),'but logger:\n%s'%self.logger
Ejemplo n.º 15
0
class TestTopCrashesByUrl:
  def setUp(self):
    global me
    self.logger = me.logger
    #self.connection = psycopg2.connect(me.dsn)
    self.connection = me.database.connection()
    self.testDB = TestDB()
    self.testDB.removeDB(me.config,self.logger)
    self.testDB.createDB(me.config,self.logger)

  def tearDown(self):
    self.testDB.removeDB(me.config,self.logger)
    socorro_cia.clearCache()

  def testConstructor(self):
    """testTopCrashesByUrl:TestTopCrashesByUrl.testConstructor(self)"""
    global me
    config = copy.copy(me.config)
    t = tcbu.TopCrashesByUrl(config)

    assert 1 == t.configContext['minimumHitsPerUrl']
    assert 500 == t.configContext['maximumUrls']
    assert tcbu.logger.name
    assert 'date_processed' == t.configContext.get('dateColumn')
    config = copy.copy(me.config)
    config['minimumHitsPerUrl'] = 2
    config['maximumUrls'] = 100
    config['logger'] = self.logger
    t = tcbu.TopCrashesByUrl(config)
    assert 2 == t.configContext.get('minimumHitsPerUrl')
    assert 100 == t.configContext.get('maximumUrls')
    assert self.logger == t.configContext.get('logger')
    halfDay = datetime.timedelta(hours=12)
    config = copy.copy(me.config)
    t = tcbu.TopCrashesByUrl(config, minimumHitsPerUrl=3, maximumUrls=50,deltaWindow =halfDay)
    assert 3 == t.configContext.get('minimumHitsPerUrl')
    assert 50 == t.configContext.get('maximumUrls')
    assert halfDay == t.configContext.get('deltaWindow')

  def testCountCrashesByUrlInWindow(self):
    """testTopCrashesByUrl:TestTopCrashesByUrl.testCountCrashesByUrlInWindow(self):"""
    global me
    cursor = self.connection.cursor()
    dbtutil.fillReportsTable(cursor,createUrls=True,multiplier=2,signatureCount=83) # just some data...
    self.connection.commit()
    config = copy.copy(me.config)

    # test /w/ 'normal' params
    t = tcbu.TopCrashesByUrl(config)
    startWindow = datetime.datetime(2008,1,1)
    deltaWindow = datetime.timedelta(days=1)
    endWindow = startWindow + deltaWindow
    data = t.countCrashesByUrlInWindow(startWindow = startWindow, deltaWindow = deltaWindow)
    # the following are JUST regression tests: The data has been only very lightly examined to be sure it makes sense.
    assert 24 ==  len(data), 'This is (just) a regression test. Did you change the data somehow? (%s)'%len(data)
    for d in data:
      assert 1 == d[0]
    # test /w/ small maximumUrls
    config = copy.copy(me.config)
    t = tcbu.TopCrashesByUrl(config, maximumUrls=50)
    data = t.countCrashesByUrlInWindow(startWindow = datetime.datetime(2008,1,1), endWindow = endWindow)
    assert 24 == len(data), 'This is (just) a regression test. Did you change the data somehow? (%s)'%len(data)
    for d in data:
      assert 1 == d[0]
    # test /w/ minimumHitsPerUrl larger
    config = copy.copy(me.config)
    t = tcbu.TopCrashesByUrl(config, minimumHitsPerUrl=2)
    data = t.countCrashesByUrlInWindow(startWindow = datetime.datetime(2008,1,1),endWindow = endWindow)
    assert 24 == len(data), len(data)
    for d in data:
      assert 1 == d[0]

    # test /w/ shorter window
    config = copy.copy(me.config)
    halfDay = datetime.timedelta(hours=12)
    t = tcbu.TopCrashesByUrl(config, deltaWindow = halfDay)
    data = t.countCrashesByUrlInWindow(startWindow = datetime.datetime(2008,1,1))
    assert 12 == len(data), 'This is (just) a regression test. Did you change the data somehow? (%s)'%len(data)
    for d in data:
      assert 1 == d[0]

    # test a different day, to be sure we get different data
    config = copy.copy(me.config)
    t = tcbu.TopCrashesByUrl(config)
    data = t.countCrashesByUrlInWindow(startWindow = datetime.datetime(2008,1,11),deltaWindow=deltaWindow)
    assert 57 == len(data), 'This is (just) a regression test. Did you change the data somehow? (%s)'%len(data)
    for d in data[:3]:
      assert 2 == d[0]
    for d in data[3:]:
      assert 1 == d[0]

  def testSaveData(self):
    """
    testTopCrashesByUrl:TestTopCrashesByUrl.testSaveData(slow=2)
    This is a reasonably realistic amount of time (about 1.5 seconds) to handle about 150 reports
    """
    global me
    cursor = self.connection.cursor()

    ## Set up
    dbtutil.fillReportsTable(cursor,createUrls=True,multiplier=2,signatureCount=83) # just some data...
    self.connection.commit()
    # ... now assure some duplicates
    sqls = "SELECT uuid, client_crash_date, install_age, last_crash, uptime, date_processed, success, signature, url, product, version, os_name, os_version from reports where date_processed >= '2008-01-01' and date_processed < '2008-01-02' LIMIT 4"
    cursor.execute(sqls)
    self.connection.rollback() # db not altered
    rows3 = cursor.fetchall()
    add11 = datetime.timedelta(seconds=1,microseconds=1000)
    addData = []
    for i in range(3):
      r = list(rows3[i])
      r[0] = r[0].replace('-dead-','-f00f-')
      r[1] += add11
      r[2] += 1
      r[3] += 1
      r[7] = rows3[i+1][7]
      addData.append(r)
      r[0] = r[0].replace('-f00f-','-fead-')
      r[1] += add11
      r[2] += 1
      r[3] += 1
      r[7] = 'js_blatherskytes'
      addData.append(r)
    sqli = """INSERT INTO reports
                (uuid, client_crash_date, install_age, last_crash, uptime, date_processed, success, signature, url, product, version, os_name, os_version)
          VALUES(%s,   %s,                %s,          %s,         %s,     %s,             %s,      %s,        %s,  %s,      %s,      %s,      %s)"""
    addData.extend([
      ['b965de73-ae90-b936-deaf-03ae20081225','2007-12-31 23:59:50',9000,110,222,'2008-01-01 11:12:13',True,'UserCallWinProcCheckWow','http://www.mozilla.org/projects/minefield/a','Firefox','3.0.9','Windows NT','5.1.2600 Service Pack 2'],
      ['b965de73-ae90-b935-deaf-03ae20081225','2007-12-31 23:59:40',9009,220,333,'2008-01-01 11:12:14',True,'UserCallWinProcCheckWow','http://yachats/uncwiki/LarsLocalPortal/b',   'Firefox','3.0.9','Windows NT','5.1.2600 Service Pack 2'],
      ])
    cursor.executemany(sqli,addData)
    self.connection.commit()
    config = copy.copy(me.config)
    startWindow = datetime.datetime(2008,1,1)
    deltaWindow = datetime.timedelta(days=1)

    ## On your mark...
    t = tcbu.TopCrashesByUrl(config)
    data = t.countCrashesByUrlInWindow(startWindow = startWindow, deltaWindow = deltaWindow)

    ## assure we have an empty playing field
    cursor.execute("SELECT COUNT(*) from top_crashes_by_url")
    self.connection.rollback()
    assert 0 == cursor.fetchone()[0]

    ## Call the method
    t.saveData(startWindow,data)

    # expect 99 rows
    cursor.execute("SELECT COUNT(id) from top_crashes_by_url")
    self.connection.rollback()
    count = cursor.fetchone()[0]
    assert 35 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)
    # expect 80 distinct urls
    cursor.execute("SELECT COUNT(distinct urldims_id) from top_crashes_by_url")
    self.connection.rollback()
    count = cursor.fetchone()[0]
    assert 21 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)

    cursor.execute("SELECT count from top_crashes_by_url where count > 1 order by count")
    self.connection.rollback()
    data = cursor.fetchall()
    assert [(2,),(2,),(2,)] == data, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(str(data))

    cursor.execute("SELECT COUNT(top_crashes_by_url_id) from top_crashes_by_url_signature")
    self.connection.rollback()
    count = cursor.fetchone()[0]
    assert 38 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)

    cursor.execute("SELECT COUNT(distinct top_crashes_by_url_id) from top_crashes_by_url_signature")
    self.connection.rollback()
    count = cursor.fetchone()[0]
    assert 35 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)

    # Expect 3 rows with sums of 2 and three rows with counts of 2, none with both
    cursor.execute("SELECT count, COUNT(top_crashes_by_url_id) AS sum FROM top_crashes_by_url_signature GROUP BY top_crashes_by_url_id, count ORDER BY sum DESC, count DESC LIMIT 6")
    self.connection.rollback()
    data = cursor.fetchall()
    assert 6 == len(data)
    assert [(1, 2L), (1, 2L), (1, 2L), (1, 1L), (1, 1L), (1, 1L)] == data, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(str(data))

  def testSaveTruncatedData(self):
    """
    testTopCrashesByUrl:TestTopCrashesByUrl.testSaveTruncatedData(slow=2)
    This is a reasonably realistic amount of time (about 1.5 seconds) to handle about 150 reports
    """
    global me
    cursor = self.connection.cursor()

    ## Set up
    dbtutil.fillReportsTable(cursor,createUrls=True,multiplier=2,signatureCount=83) # just some data...
    self.connection.commit()
    # ... now assure some duplicates
    sqls = "SELECT uuid, client_crash_date, install_age, last_crash, uptime, date_processed, success, signature, url, product, version, os_name, os_version from reports where date_processed >= '2008-01-01' and date_processed < '2008-01-02' LIMIT 4"
    cursor.execute(sqls)
    self.connection.rollback() # db not altered
    rows3 = cursor.fetchall()
    add11 = datetime.timedelta(seconds=1,microseconds=1000)
    addData = []
    for i in range(3):
      r = list(rows3[i])
      r[0] = r[0].replace('-dead-','-f00f-')
      r[1] += add11
      r[2] += 1
      r[3] += 1
      r[7] = rows3[i+1][7]
      addData.append(r)
      r[0] = r[0].replace('-f00f-','-fead-')
      r[1] += add11
      r[2] += 1
      r[3] += 1
      r[7] = 'js_blatherskytes'
      addData.append(r)
    sqli = """INSERT INTO reports
                (uuid, client_crash_date, install_age, last_crash, uptime, date_processed, success, signature, url, product, version, os_name, os_version)
          VALUES(%s,   %s,                %s,          %s,         %s,     %s,             %s,      %s,        %s,  %s,      %s,      %s,      %s)"""
    addData.extend([
      ['b965de73-ae90-b936-deaf-03ae20081225','2007-12-31 23:59:50',9000,110,222,'2008-01-01 11:12:13',True,'UserCallWinProcCheckWow','http://www.mozilla.org/projects/minefield/a','Firefox','3.0.9','Windows NT','5.1.2600 Service Pack 2'],
      ['b965de73-ae90-b935-deaf-03ae20081225','2007-12-31 23:59:40',9009,220,333,'2008-01-01 11:12:14',True,'UserCallWinProcCheckWow','http://yachats/uncwiki/LarsLocalPortal/b',   'Firefox','3.0.9','Windows NT','5.1.2600 Service Pack 2'],
      ])
    cursor.executemany(sqli,addData)
    self.connection.commit()
    config = copy.copy(me.config)
    startWindow = datetime.datetime(2008,1,1)
    deltaWindow = datetime.timedelta(days=1)

    ## On your mark...
    t = tcbu.TopCrashesByUrl(config,truncateUrlLength=25)
    data = t.countCrashesByUrlInWindow(startWindow = startWindow, deltaWindow = deltaWindow)

    ## assure we have an empty playing field
    cursor.execute("SELECT COUNT(*) from top_crashes_by_url")
    self.connection.rollback()
    assert 0 == cursor.fetchone()[0]

    ## Call the method
    t.saveData(startWindow,data)

    # expect 99 rows
    cursor.execute("SELECT COUNT(id) from top_crashes_by_url")
    self.connection.rollback()
    count = cursor.fetchone()[0]
    assert 30 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)
    # expect 80 distinct urls
    cursor.execute("SELECT COUNT(distinct urldims_id) from top_crashes_by_url")
    self.connection.rollback()
    count = cursor.fetchone()[0]
    assert 17 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)

    cursor.execute("SELECT count from top_crashes_by_url where count > 1 order by count")
    self.connection.rollback()
    data = cursor.fetchall()
    assert [(2,), (2,), (2,), (2,), (2,), (4,)] == data, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(str(data))

    cursor.execute("SELECT COUNT(top_crashes_by_url_id) from top_crashes_by_url_signature")
    self.connection.rollback()
    count = cursor.fetchone()[0]
    assert 38 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)

    cursor.execute("SELECT COUNT(distinct top_crashes_by_url_id) from top_crashes_by_url_signature")
    self.connection.rollback()
    count = cursor.fetchone()[0]
    assert 30 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)

    # Expect 3 rows with sums of 2 and three rows with counts of 2, none with both
    cursor.execute("SELECT count, COUNT(top_crashes_by_url_id) AS sum FROM top_crashes_by_url_signature GROUP BY top_crashes_by_url_id, count ORDER BY sum DESC, count DESC LIMIT 6")
    self.connection.rollback()
    data = cursor.fetchall()
    assert 6 == len(data)
    assert [(1, 4L), (1, 2L), (1, 2L), (1, 2L), (1, 2L), (1, 2L)] == data, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(str(data))

  def testProcessDateInterval(self):
    """
    testTopCrashesByUrl:TestTopCrashesByUrl.testProcessDateInterval(slow=7)
    Takes a long time, first to set up the data (about 1.5 seconds), then to process it several times
    """
    global me
    cursor = self.connection.cursor()
    config = copy.copy(me.config)

    ## Set up
    dbtutil.fillReportsTable(cursor,createUrls=True,multiplier=2,signatureCount=83) # just some data...
    self.connection.commit()
    t = tcbu.TopCrashesByUrl(config)
    ## assure we have an empty playing field
    cursor.execute("SELECT COUNT(*) from top_crashes_by_url")
    self.connection.rollback()
    assert 0 == cursor.fetchone()[0]
    t.processDateInterval(startDate = datetime.datetime(2008,1,1), endDate=datetime.datetime(2008,1,6))

    cursor.execute("SELECT COUNT(id) from top_crashes_by_url")
    self.connection.rollback()
    count = cursor.fetchone()[0]
    assert 35 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)

    cursor.execute("SELECT COUNT(*) from top_crashes_by_url_signature")
    self.connection.rollback()
    count = cursor.fetchone()[0]
    assert 38 ==  count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)

    cursor.execute("delete from top_crashes_by_url; delete from top_crashes_by_url_signature")
    self.connection.commit()
    t = tcbu.TopCrashesByUrl(copy.copy(me.config))
    t.processDateInterval(startDate = datetime.datetime(2008,1,4), endDate=datetime.datetime(2008,1,8))

    cursor.execute("SELECT COUNT(id) from top_crashes_by_url")
    self.connection.rollback()
    count = cursor.fetchone()[0]
    assert 31 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)

    cursor.execute("SELECT COUNT(*) from top_crashes_by_url_signature")
    self.connection.rollback()
    count = cursor.fetchone()[0]
    assert 32 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)

    cursor.execute("delete from top_crashes_by_url; delete from top_crashes_by_url_signature")
    self.connection.commit()

    t = tcbu.TopCrashesByUrl(copy.copy(me.config))
    t.processDateInterval(startDate = datetime.datetime(2008,1,1), endDate=datetime.datetime(2008,3,3))
    
    cursor.execute("SELECT COUNT(id) from top_crashes_by_url")
    self.connection.rollback()
    count = cursor.fetchone()[0]
    assert 483 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)

    cursor.execute("SELECT COUNT(*) from top_crashes_by_url_signature")
    self.connection.rollback()
    count = cursor.fetchone()[0]
    assert 514 == count, 'This is (just) a regression test. Did you change the data somehow? (%s)'%(count)