def processDateInterval(self, **kwargs):
cursor = self.connection.cursor()
kwargs.setdefault('defaultDeltaWindow',defaultDeltaWindow)
startDate,deltaDate,endDate = cron_util.getProcessingDates(self.configContext, resultTable, self.productVersionRestriction, cursor, logger, **kwargs)
startWindow,deltaWindow,endWindow = cron_util.getProcessingWindow(self.configContext, resultTable,self.productVersionRestriction,cursor, logger, **kwargs)
logger.info("Starting loop from %s up to %s step (%s)",startDate.isoformat(),endDate.isoformat(),deltaWindow)
if not startWindow:
startWindow = startDate
if not deltaWindow:
deltaWindow = defaultDeltaWindow
while startWindow + deltaWindow < endDate:
data = self.countCrashesByUrlInWindow(startWindow=startWindow,deltaWindow=deltaWindow)
if data:
logger.info("Saving %s items in window starting at %s",len(data),startWindow)
self.saveData(startWindow,data)
else:
logger.info("Window starting at %s had no data",startWindow)
# whether or not we saved some data, advance to next slot
startWindow += deltaWindow
logger.info("Done processIntervals")
def processDateInterval(self, **kwargs):
"""
call processOneMtbfWindow repeatedly for each window in the range defined by at least two paramerters among (start|delta|end)Date
Other kwargs/context values are passed unchanged to processOneMtbfWindow
"""
cur = self.connection.cursor()
now = datetime.datetime.now()
startDate, deltaDate, endDate = cron_util.getProcessingDates(self.configContext,resultTable,cur,self.logger,**kwargs)
startWindow,deltaWindow,endWindow = cron_util.getProcessingWindow(self.configContext,resultTable,cur,self.logger,**kwargs)
if not startDate and not startWindow:
self.logger.warn("MTBF (%s): No startDate, no startWindow. Did not run.",now)
return 0
if not startWindow: # we are guaranteed a startDate after the test above
startWindow = startDate
if not startDate or startDate > startWindow:
startDate = startWindow
if not deltaWindow:
deltaWindow = datetime.timedelta(days=1)
thisMidnight = now.replace(hour=0,minute=0,second=0,microsecond=0)
if not endDate or endDate > thisMidnight:
endDate = thisMidnight
if startDate + deltaWindow > endDate:
self.logger.warn("MTBF (%s) startDate (%s) too close to endDate (%s). Did not run.",now, startDate,endDate)
return 0
count = 0
if self.debugging:
self.logger.debug("""mtbf.processDateInterval:
startDate: %s, deltaDate: %s, endDate: %s
startWindow: %s, deltaWindow: %s, endWindow: %s """,startDate,deltaDate,endDate,startWindow,deltaWindow,endWindow)
kwargs['deltaWindow'] = deltaWindow
startWindow = startDate
while startWindow + deltaWindow < endDate:
kwargs['startWindow'] = startWindow
self.processOneMtbfWindow(**kwargs)
startWindow += deltaWindow
count += 1
return count
def testGetProcessingDates(self):
config = {}
config['processingDelay'] = datetime.timedelta(0)
cursor = self.connection.cursor()
self.connection.rollback()
self.createBunny()
now = datetime.datetime.now()
midnight = now.replace(hour=0,minute=0,second=0,microsecond=0)
defStart = midnight - cron_util.globalInitialDeltaDate
defEnd = midnight
while defEnd + cron_util.globalDefaultDeltaWindow < now:
defEnd += cron_util.globalDefaultDeltaWindow
mm = cron_util.getProcessingDates(config,self.tableName,None,cursor,me.fileLogger)
assert (defStart,defEnd-defStart,defEnd) == mm, 'But got %s'%(str(mm))
start = datetime.datetime(2000,1,2,12,12)
delta = datetime.timedelta(days=3)
end = start+delta
# check that just one kwarg raises SystemExit
assert_raises(SystemExit,cron_util.getProcessingDates,config,self.tableName,None,cursor,me.fileLogger,startDate=start)
assert_raises(SystemExit,cron_util.getProcessingDates,config,self.tableName,None,cursor,me.fileLogger,endDate=end)
assert_raises(SystemExit,cron_util.getProcessingDates,config,self.tableName,None,cursor,me.fileLogger,deltaDate=delta)
# check that just one config raises SystemExit
assert_raises(SystemExit,cron_util.getProcessingDates,{'startDate':start},self.tableName,None,cursor,me.fileLogger)
assert_raises(SystemExit,cron_util.getProcessingDates,{'endDate':end},self.tableName,None,cursor,me.fileLogger)
assert_raises(SystemExit,cron_util.getProcessingDates,{'deltaDate':delta},self.tableName,None,cursor,me.fileLogger)
# check that two are sufficient
mm = cron_util.getProcessingDates({'startDate':start},self.tableName,None,cursor,me.fileLogger,endDate=end)
assert (start,delta,end) == mm, "But got %s"%(str(mm))
mm = cron_util.getProcessingDates({'startDate':start},self.tableName,None,cursor,me.fileLogger,deltaDate=delta)
assert (start,delta,end) == mm, "But got %s"%(str(mm))
mm = cron_util.getProcessingDates({'endDate':end},self.tableName,None,cursor,me.fileLogger,startDate=start)
assert (start,delta,end) == mm, "But got %s"%(str(mm))
mm = cron_util.getProcessingDates({'endDate':end},self.tableName,None,cursor,me.fileLogger,deltaDate=delta)
assert (start,delta,end) == mm, "But got %s"%(str(mm))
mm = cron_util.getProcessingDates({'deltaDate':delta},self.tableName,None,cursor,me.fileLogger,endDate=end)
assert (start,delta,end) == mm, "But got %s"%(str(mm))
mm = cron_util.getProcessingDates({'deltaDate':delta},self.tableName,None,cursor,me.fileLogger,startDate=start)
assert (start,delta,end) == mm, "But got %s"%(str(mm))
# check various inconsistencies
assert_raises(SystemExit,cron_util.getProcessingDates,{'startDate':start},self.tableName,None,cursor,me.fileLogger,endDate=start)
assert_raises(SystemExit,cron_util.getProcessingDates,{'startDate':start},self.tableName,None,cursor,me.fileLogger,endDate=start-delta)
assert_raises(SystemExit,cron_util.getProcessingDates,{'startDate':start},self.tableName,None,cursor,me.fileLogger,deltaDate=datetime.timedelta(0))
assert_raises(SystemExit,cron_util.getProcessingDates,{'startDate':start},self.tableName,None,cursor,me.fileLogger,deltaDate=datetime.timedelta(days=-1))
# Check that table with earlier row is ignored
early = start-datetime.timedelta(days=1)
cursor.execute("INSERT INTO %s (window_end,window_size) VALUES(%%s,%%s)"%self.tableName,(early,delta))
self.connection.commit()
mm = cron_util.getProcessingDates(config,self.tableName,None,cursor,me.fileLogger,startDate=start,endDate=end)
assert (start,delta,end) == mm, "But got %s"%(str(mm))
# Check that table with later row is used
later = start+datetime.timedelta(days=1)
cursor.execute("INSERT INTO %s (window_end,window_size) VALUES(%%s,%%s)"%self.tableName,(later,delta))
self.connection.commit()
mm = cron_util.getProcessingDates(config,self.tableName,None,cursor,me.fileLogger,startDate=start,endDate=end)
assert (later,end-later,end) == mm, 'Expected %s, got %s'%(str((later,end-later,end)),str(mm))
# Check that table with 'too late' time causes assertion
later = later+datetime.timedelta(days=4)
cursor.execute("INSERT INTO %s (window_end,window_size) VALUES(%%s,%%s)"%self.tableName,(later,delta))
self.connection.commit()
assert_raises(SystemExit,cron_util.getProcessingDates,config,self.tableName,None,cursor,me.fileLogger,startDate=start,endDate=end)