def __init__(self,*args,**kwargs): """ Passes appropriate kwargs to Config, pays local attention to these keys: updateInterval: default: '0' format: 'dd:hh:mm:ss', leading parts optional. Must be >= 0 seconds. updateFunction: default: noop(). Takes self as argument. Behavior: Updates default values in argument reEvaluateFunction: default: noop(). Takes self as argument. Behavior: Mutates values in argument signalNumber: default: SIGALRM (14). If 0, then signals will not be handled. Instances that share the same signalNumber will all be update()-ed at every signal. self.internal.updateFunction may be set after construction if desired: Avoids double-work at construction self.internal.reEvalutateFunction may be set after construction if desired, but this is not recommended. """ skwargs = dict([(x,kwargs[x]) for x in socorro_config.getDefaultedConfigOptions().keys() if x in kwargs]) for i in range(len(args)): skwargs[socorro_config.namedConfigOptions[i]] = args[i] super(DynamicConfig,self).__init__(**skwargs) self.internal.updateFunction = kwargs.get('updateFunction',noop) self.internal.reEvaluateFunction = kwargs.get('reEvaluateFunction',noop) self.internal.signalNumber = kwargs.get('signalNumber',14) self.internal.nextUpdate = None updateInterval = kwargs.get('updateInterval','0:0:0:0') self.internal.updateDelta = socorro_config.timeDeltaConverter(updateInterval) if self.internal.updateDelta: if self.internal.updateDelta < datetime.timedelta(0): raise ValueError("updateInterval must be non-negative, but %s"%self.internal.updateDelta) self.internal.nextUpdate = utc_now() + self.internal.updateDelta # finally: make sure we are current if self.internal.signalNumber: priorSignal = signal.signal(self.internal.signalNumber,DynamicConfig.handleAlarm) self.doUpdate() DynamicConfig.instances[id(self)] = self
def testAcceptConfigFile(self): # Test failure with good config file, unknown option in that file try: copt = [('c', 'config', True, self.configTstPath, "the test config file")] CM.newConfiguration(configurationOptionsList=copt,optionNameForConfigFile = 'config', configurationFileRequired = True) assert(False) except CM.NotAnOptionError, e: assert(True)
def getProcessingWindow(configContext,tableName, productVersionRestriction,cursor,logger, **kwargs): """ ProcessingWindow is a single time window over which to aggregate materialized view data. Returns (startWindow,deltaWindow,endWindow) using this heuristic: kwargs beats configContext which beats latest table row if two among startWindow, endWindow, deltaWindow in config or kwargs: they are used. if all three: assert startWindow + deltaWindow == endWindow Backward compatibility: if processingDay is present and windowXxx are not: startWindow = midnight of given day, deltaWindow = timedelta(days=1) else: try to read window_end and window_size from the given table if one is available from config/kwargs it beats the same (or calculated) one from the table On inconsistency or failure, logs the problem and aborts BEWARE: You can get inconsitency by having one item in config and the other two in kwargs: BEWARE """ config = {} config.update(configContext) config.update(kwargs) startWindow = config.get('startWindow') if type(startWindow) is str: startWindow = cm.dateTimeConverter(startWindow) deltaWindow = config.get('deltaWindow') if type(deltaWindow) is str: deltaWindow = cm.timeDeltaConverter(deltaWindow) endWindow = config.get('endWindow') if type(endWindow) is str: endWindow = cm.dateTimeConverter(endWindow) processingDay = config.get('processingDay') if type(processingDay) is str: processingDay = cm.dateTimeConverter(processingDay) try: if startWindow or deltaWindow or endWindow: if startWindow and endWindow and deltaWindow: assert startWindow + deltaWindow == endWindow,"inconsistent: %s + %s != %s"%(startWindow,deltaWindow,endWindow) elif startWindow and endWindow: deltaWindow = endWindow - startWindow elif startWindow and deltaWindow: endWindow = startWindow + deltaWindow elif deltaWindow and endWindow: startWindow = endWindow - deltaWindow else: assert not (startWindow or deltaWindow or endWindow), "insufficient: Need two of window ...Start: %s, ...Delta: %s, ...End:%s"%(startWindow,deltaWindow,endWindow) elif processingDay: dayt = datetime.datetime.fromtimestamp(time.mktime(processingDay.timetuple())) startWindow = dayt.replace(hour=0,minute=0,second=0,microsecond=0) assert startWindow == dayt,'processingDay must be some midnight, but was %s'%dayt deltaWindow = datetime.timedelta(days=1) endWindow = startWindow + deltaWindow else: # no params: try table startWindow,deltaWindow = getLastWindowAndSizeFromTable(cursor,tableName, productVersionRestriction,logger) if startWindow: endWindow = startWindow+deltaWindow return (startWindow,deltaWindow,endWindow) except: lib_util.reportExceptionAndAbort(logger)
def setup_module(): global me if me: return me = Me() tutil.nosePrintModule(__file__) # config gets messed up by some tests. Use this one during module setup and teardown me.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing Postgresql Utils') myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir':'%s'%myDir} for i in me.config: try: me.config[i] = me.config.get(i)%(replDict) except: pass me.logFilePathname = me.config.logFilePathname if not me.logFilePathname: me.logFilePathname = 'logs/db_test.log' logFileDir = os.path.split(me.logFilePathname)[0] try: os.makedirs(logFileDir) except OSError,x: if errno.EEXIST == x.errno: pass else: raise
def setup_module(): global me if me: return # else initialize # print "MODULE setup" me = Me() me.markingTemplate = "MARK %s: %s" me.startMark = 'start' me.endMark = 'end' me.testDB = TestDB() me.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing Monitor') myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir':'%s'%myDir} for i in me.config: try: me.config[i] = me.config.get(i)%(replDict) except: pass knownTests = [x for x in dir(TestMonitor) if x.startswith('test')] me.logWasExtracted = {} for t in knownTests: me.logWasExtracted[t] = False me.logger = monitor.logger me.logger.setLevel(logging.DEBUG) me.logFilePathname = me.config.logFilePathname logfileDir = os.path.split(me.config.logFilePathname)[0] try: os.makedirs(logfileDir) except OSError,x: if errno.EEXIST != x.errno: raise
def setup_module(): global me if me: return me = Me() me.testDB = TestDB() me.config = configurationManager.newConfiguration( configurationModule=testConfig, applicationName='TestingCachedIdAccess') myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir': '%s' % myDir} for i in me.config: try: me.config[i] = me.config.get(i) % (replDict) except: pass cia.logger.setLevel(logging.DEBUG) me.logFilePathname = me.config.logFilePathname logfileDir = os.path.split(me.config.logFilePathname)[0] try: os.makedirs(logfileDir) except OSError, x: if errno.EEXIST != x.errno: raise f = open(me.config.logFilePathname, 'w') f.close()
def newConfiguration(**kwargs): """ This used as an alternate constructor for class Config so that applications can be lax in defining all the required paramters in the right order. """ kw = socorro_config.getDefaultedConfigOptions() kw.update(kwargs) return DynamicConfig(**kw)
def setup_module(): global me if me: return me = Me() tutil.nosePrintModule(__file__) me.testDB = TestDB() me.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='TestingCachedIdAccess') myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir':'%s'%myDir} for i in me.config: try: me.config[i] = me.config.get(i)%(replDict) except: pass cia.logger.setLevel(logging.DEBUG) me.logFilePathname = me.config.logFilePathname logfileDir = os.path.split(me.config.logFilePathname)[0] try: os.makedirs(logfileDir) except OSError,x: if errno.EEXIST != x.errno: raise f = open(me.config.logFilePathname,'w') f.close()
def setupConfig(): try: return configurationManager.newConfiguration(configurationModule=config, applicationName="Migrate Process Type") except configurationManager.NotAnOptionError, x: print >>sys.stderr, x print >>sys.stderr, "for usage, try --help" sys.exit(1)
def main(): logger = logging.getLogger("topcrashes_summary") logger.setLevel(logging.WARNING) stderrLog = logging.StreamHandler() stderrLog.setLevel(logging.WARNING) stderrLogFormatter = logging.Formatter( '%(asctime)s %(levelname)s - %(message)s') stderrLog.setFormatter(stderrLogFormatter) logger.addHandler(stderrLog) kwargs = {} for i in sys.argv[1:]: if i.startswith('-h') or i.startswith('--he'): help() sys.exit(0) j = i if i.startswith('-'): j = i.lstrip('-') if '=' in j: name, value = (s.strip() for s in j.split('=')) kwargs[name] = value else: print >> sys.stderr, "Ignoring unkown argument '%s'" % (i) sys.argv = sys.argv[:1] config = configurationManager.newConfiguration( configurationModule=testConfig, applicationName='Create Database') config.update(kwargs) testDB = TestDB() testDB.removeDB(config, logger) testDB.createDB(config, logger)
def setup_module(): global me if me: return me = Me() me.config = cfgManager.newConfiguration( configurationModule=testConfig, applicationName='Testing ftpscraper') myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir': '%s' % myDir} for i in me.config: try: me.config[i] = me.config.get(i) % (replDict) except: pass me.logFilePathname = me.config.logFilePathname if not me.logFilePathname: me.logFilePathname = 'logs/ftpscraper_test.log' logFileDir = os.path.split(me.logFilePathname)[0] try: os.makedirs(logFileDir) except OSError, x: if errno.EEXIST == x.errno: pass else: raise
def setup_module(): global me if me: return me = Me() tutil.nosePrintModule(__file__) me.config = configurationManager.newConfiguration( configurationModule=testConfig, applicationName="Testing Postgresql Utils" ) myDir = os.path.split(__file__)[0] if not myDir: myDir = "." replDict = {"testDir": "%s" % myDir} for i in me.config: try: me.config[i] = me.config.get(i) % (replDict) except: pass me.logFilePathname = me.config.logFilePathname if not me.logFilePathname: me.logFilePathname = "logs/db_test.log" logFileDir = os.path.split(me.logFilePathname)[0] try: os.makedirs(logFileDir) except OSError, x: if errno.EEXIST == x.errno: pass else: raise
def setup_module(): global me if me: return me = Me() me.config = cfgManager.newConfiguration(configurationModule=testConfig, applicationName='Testing ftpscraper') myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir': '%s' % myDir} for i in me.config: try: me.config[i] = me.config.get(i) % (replDict) except: pass me.logFilePathname = me.config.logFilePathname if not me.logFilePathname: me.logFilePathname = 'logs/ftpscraper_test.log' logFileDir = os.path.split(me.logFilePathname)[0] try: os.makedirs(logFileDir) except OSError, x: if errno.EEXIST == x.errno: pass else: raise
def main(): logger = logging.getLogger("topcrashes_summary") logger.setLevel(logging.WARNING) stderrLog = logging.StreamHandler() stderrLog.setLevel(logging.WARNING) stderrLogFormatter = logging.Formatter('%(asctime)s %(levelname)s - %(message)s') stderrLog.setFormatter(stderrLogFormatter) logger.addHandler(stderrLog) kwargs = {} for i in sys.argv[1:]: if i.startswith('-h') or i.startswith('--he'): help() sys.exit(0) j = i if i.startswith('-'): j = i.lstrip('-') if '=' in j: name,value = (s.strip() for s in j.split('=')) kwargs[name] = value else: print >> sys.stderr,"Ignoring unkown argument '%s'"%(i) sys.argv = sys.argv[:1] config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Create Database') config.update(kwargs) testDB = TestDB() testDB.removeDB(config,logger) testDB.createDB(config,logger)
def setup_module(): global me if me: return me = Me() # config gets messed up by some tests. Use this one during module setup and teardown me.config = configurationManager.newConfiguration( configurationModule=testConfig, applicationName='Testing Postgresql Utils') myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir': '%s' % myDir} for i in me.config: try: me.config[i] = me.config.get(i) % (replDict) except: pass me.logFilePathname = me.config.logFilePathname if not me.logFilePathname: me.logFilePathname = 'logs/db_test.log' logFileDir = os.path.split(me.logFilePathname)[0] try: os.makedirs(logFileDir) except OSError, x: if errno.EEXIST == x.errno: pass else: raise
def setupConfig(): try: return configurationManager.newConfiguration( configurationModule=config, applicationName="Migrate Process Type") except configurationManager.NotAnOptionError, x: print >> sys.stderr, x print >> sys.stderr, "for usage, try --help" sys.exit(1)
def setUp(self): """Create a configuration context and a database connection. """ self.config = ConfigurationManager.newConfiguration( configurationModule=commonconfig, applicationName="PostgreSQL Tests") self.database = db.Database(self.config) self.connection = self.database.connection()
def fixupContextByProcessingDay(self,context): pday = context.get('processingDay') if pday: logger.info("Adjusting startDate and deltaDate per processingDay %s",pday) pday = cm.dateTimeConverter(pday) startDate = datetime.datetime.fromtimestamp(time.mktime(pday.timetuple())) startDate.replace(hour=0,minute=0,second=0,microsecond=0) context['startDate'] = startDate context['deltaDate'] = datetime.timedelta(days=1) context['startWindw'] = startDate
def setUp(self): global me # config gets messed up by some tests. Use this one by preference self.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing Postgresql Utils') for i in self.config: try: self.config[i] = self.config.get(i)%(replDict) except: pass self.connection = psycopg2.connect(me.dsn) self.testDB = TestDB()
def testAcceptUserCommandLineHelp(self): opts = [('h','help',False,False,'another help')] args = {} args['automaticHelp'] = False args['configurationOptionsList'] = opts hh = HelpHandler() args['helpHandler'] = hh.handleHelp sys.argv.append('--help') conf = CM.newConfiguration(**args) assert("--help" in hh.data) assert("another help" in hh.data)
def testAcceptAutoCommandLineHelp(self): opts = [] args = {} args['automaticHelp'] = True args['configurationOptionsList'] = opts hh = HelpHandler() args['helpHandler'] = hh.handleHelp sys.argv.append('--help') conf = CM.newConfiguration(**args) assert("--help" in hh.data) assert("print this list" in hh.data)
def testAcceptEnvironment(self): opts = [] args = {} opts.append(('c','chickensoup',False,False,'help for the ailing')) opts.append(('r','rabbit', True, '', 'rabbits are bunnies')) args['configurationOptionsList'] = opts os.environ['chickensoup']='' os.environ['r'] = 'bunny-rabbit' conf = CM.newConfiguration(**args) assert('chickensoup' in conf) assert('rabbit' in conf) assert('bunny-rabbit' == conf.rabbit)
def testAcceptTypePriority(self): '''Assure that commandline beats config file beats environment beats defaults''' copt = [('c', 'config', True, self.configTstPath, "the test config file"),('r','rabbit', True, 'bambi', 'rabbits are bunnies')] copt.append(('b','badger',True,'gentle','some badgers are gentle')) copt.append(('z','zeta', True, 'zebra', 'zebras ooze')) os.environ['badger'] = 'bloody' os.environ['zeta'] = 'zymurgy' sys.argv.append('--rabbit=kangaroo') conf = CM.newConfiguration(automaticHelp=False,configurationOptionsList=copt,optionNameForConfigFile = 'config', configurationFileRequired = True) assert('kangaroo' == conf.rabbit) # command line beats config file assert('this badger=awful' == conf.badger) # config file beats environment assert('zymurgy' == conf.zeta)
def setUp(self): """Create a configuration context and a database connection. """ self.config = ConfigurationManager.newConfiguration( configurationModule=commonconfig, applicationName="PostgreSQL Tests" ) try: self.database = db.Database(self.config) except (AttributeError, KeyError): raise self.connection = self.database.connection()
def setUp(self): """Create a configuration context and a database connection. """ self.config = ConfigurationManager.newConfiguration( configurationModule=commonconfig, applicationName="PostgreSQL Tests") try: self.database = db.Database(self.config) except (AttributeError, KeyError): raise self.connection = self.database.connection()
def testAcceptCommandLine(self): opts = [] args = {} opts.append(('c','chickensoup',False,False,'help for the ailing')) opts.append(('r','rabbit', True, '', 'rabbits are bunnies')) args['configurationOptionsList'] = opts sys.argv.append('-c') sys.argv.append('--rabbit=bunny') conf = CM.newConfiguration(**args) assert('chickensoup' in conf) assert('rabbit' in conf) assert('bunny' == conf.rabbit)
def setUp(self): global me # config gets messed up by some tests. Use this one by preference self.config = configurationManager.newConfiguration( configurationModule=testConfig, applicationName='Testing Postgresql Utils') for i in self.config: try: self.config[i] = self.config.get(i) % (replDict) except: pass self.connection = psycopg2.connect(me.dsn) self.testDB = TestDB()
def setUp(self): global me self.config = cfgManager.newConfiguration( configurationModule=testConfig, applicationName='Testing ftpscraper') myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir': '%s' % myDir} for i in self.config: try: self.config[i] = self.config.get(i) % (replDict) except: pass self.logger = TestingLogger(me.fileLogger) self.testConfig = cfgManager.Config([ ('t', 'testPath', True, './TEST-BUILDS', ''), ('f', 'testFileName', True, 'lastrun.pickle', '') ]) self.testConfig["persistentDataPathname"] = os.path.join( self.testConfig.testPath, self.testConfig.testFileName)
def setUp(self): global me self.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing builds') myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir':'%s'%myDir} for i in self.config: try: self.config[i] = self.config.get(i)%(replDict) except: pass self.logger = TestingLogger(me.fileLogger) self.testConfig = configurationManager.Config([('t','testPath', True, './TEST-BUILDS', ''), ('f','testFileName', True, 'lastrun.pickle', ''), ]) self.testConfig["persistentDataPathname"] = os.path.join(self.testConfig.testPath, self.testConfig.testFileName)
def createMe(): global me if not me: me = Me() me.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName = "Testing TopCrashers") myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir':'%s'%myDir} for i in me.config: try: me.config[i] = me.config.get(i)%(replDict) except: pass me.logFilePathname = me.config.logFilePathname logfileDir = os.path.split(me.config.logFilePathname)[0] try: os.makedirs(logfileDir) except OSError,x: if errno.EEXIST != x.errno: raise
def requested_processor_id(requested_id): """This method makes sure that the configuration parameter 'processorID' is in the proper form. If it is an integer, it is cast into the integer type. If it is a string, it is ensured to be one of the acceptable values. Parameters: requested_id - the value is passed in from the 'processorId' configuration parameter. Returns: an integer or the the strings 'host' or 'auto'""" try: return int(requested_id) except ValueError: if requested_id in ('auto', 'host'): return requested_id else: raise scm.OptionError("'%s' is not a valid value" % requested_id)
def testNewConfiguration(self): ''' Trick: To call f(**kwargs) with a dictionary d as the single arg, chant f(**d) ''' opts = [] args = {} args['automaticHelp'] = False # Test for empty conf = CM.newConfiguration(**args) assert(not conf.allowableOptionDictionary) # Test for autoHelp args['automaticHelp'] = True conf = CM.newConfiguration(**args) assert(2 == len(conf.allowableOptionDictionary)) # Test for another legal option opts.append(('c','chickensoup',False,False,'Help for the ailing')) args['automaticHelp'] = True args['configurationOptionsList'] = opts conf = CM.newConfiguration(**args) assert(4 == len(conf.allowableOptionDictionary)) # Test a config module conf = CM.newConfiguration(automaticHelp=False,configurationModule=optionfile) cd = conf.allowableOptionDictionary assert(5 == len(cd)) assert ['T', 'testSingleCharacter', True, None] == cd.get('T')[:-1],'but got %s' % (str(cd.get('T')[:-1])) assert 'testSingleCharacter imported from' in cd.get('T')[-1],'but got %s' % (str(cd.get('T')[-1])) assert 'optionfile' in cd.get('T')[-1],'but got %s' % (str(cd.get('T')[-1])) #assert ['T', 'testSingleCharacter', True, None, 'testSingleCharacter imported from optionfile'] == cd.get('T'),'but got %s' % (str(cd.get('T'))) assert [None, 'testDefault', True, 'default'] == cd.get('testDefault')[:-1], "but got %s" %(str(cd.get('testDefault')[:-1])) assert 'testDefault imported from' in cd.get('testDefault')[-1], "but got %s" %(cd.get('testDefault')[-1]) assert 'optionfile' in cd.get('testDefault')[-1], "but got %s" %(cd.get('testDefault')[-1]) #assert([None, 'testDefault', True, 'default', 'testDefault imported from optionfile'] == cd.get('testDefault')) assert([None, 'testDoc', True, None, 'test doc'] == cd.get('testDoc')) assert [None, 'testNil', True, None] == cd.get('testNil')[:-1], "but got %s" %(str(cd.get('testNil')[:-1])) assert 'testNil imported from' in cd.get('testNil')[-1], "but got %s" %(cd.get('testNil')[-1]) assert 'optionfile' in cd.get('testNil')[-1], "but got %s" %(cd.get('testNil')[-1]) #assert([None, 'testNil', True, None, 'testNil imported from optionfile'] == cd.get('testNil')) # Test failure with good option, bad file try: copt = [('c', 'config', True, './badone', "the badconfig file")] CM.newConfiguration(configurationOptionsList=copt,optionNameForConfigFile = 'config', configurationFileRequired = True) assert(False) except CM.ConfigFileMissingError, e: assert(True)
def setUp(self): configContext = configurationManager.newConfiguration(configurationModule=config, applicationName="Top Crash By URL Summary") logger = logging.getLogger("tcbyurl_summary") logger.setLevel(logging.DEBUG) stderrLog = logging.StreamHandler() stderrLog.setLevel(configContext.stderrErrorLoggingLevel) stderrLogFormatter = logging.Formatter(configContext.stderrLineFormatString) stderrLog.setFormatter(stderrLogFormatter) logger.addHandler(stderrLog) rotatingFileLog = logging.handlers.RotatingFileHandler(configContext.logFilePathname, "a", configContext.logFileMaximumSize, configContext.logFileMaximumBackupHistory) rotatingFileLog.setLevel(logging.DEBUG) rotatingFileLogFormatter = logging.Formatter(configContext.logFileLineFormatString) rotatingFileLog.setFormatter(rotatingFileLogFormatter) logger.addHandler(rotatingFileLog) logger.info("current configuration\n%s", str(configContext)) self.configContext = configContext self.logger = logger self.rotatingFileLog = rotatingFileLog
def createMe(): global me if not me: me = Me() me.config = configurationManager.newConfiguration( configurationModule=testConfig, applicationName="Testing TopCrashers") myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir': '%s' % myDir} for i in me.config: try: me.config[i] = me.config.get(i) % (replDict) except: pass me.logFilePathname = me.config.logFilePathname logfileDir = os.path.split(me.config.logFilePathname)[0] try: os.makedirs(logfileDir) except OSError, x: if errno.EEXIST != x.errno: raise
def main(): addedConfigOptions = [ ('h','help',False,None,'print this list',None), ('P','product-os-count',True,12,'how many product/os pairs to use [1-56]',lambda x: rangeChecker(1,56,x) ), ('S','signature-count',True,83,'how many signatures to use [1-121]',lambda x: rangeChecker(1,121,x)), ('R','repeat-count',True,2,'how many times to loop [smallish N]',None), ('m','mtbf-fill',False,True,'Fill the time_before_failure table',lambda x: True), ('u','url-fill',False,True,'Fill the top_crashes_by_url table',lambda x: True), ('s','sig-fill',False,True,'Fill the top_crashes_by_signature table',lambda x: True), ('a','all-fill',False,True,'Fill all three matrialized view tables',[('mtbf-fill',None),('url-fill',None),('sig-fill',None)]), ('D','drop-all',False,True,'Drop all the database tables. Do no other work',lambda x: True), (None,'logFileErrorLoggingLevel',True,logging.WARNING,'logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)',None), ] config = configurationManager.newConfiguration(configurationModule = commonConfig, applicationName='fillSchema.py',configurationOptionsList = addedConfigOptions) myDir = os.path.split(__file__)[0] logDir = os.path.join(myDir,'logs') logFile = os.path.join(logDir,'fillDB.log') try: os.makedirs(logDir) except OSError: pass logger = logging.getLogger("fillDB") fileLog = logging.FileHandler(logFile) fileLog.setLevel(int(config.logFileErrorLoggingLevel)) fileLogFormatter = logging.Formatter('%(asctime)s %(levelname)s - %(message)s') fileLog.setFormatter(fileLogFormatter) # now add the handler for all the logs of interest logger.addHandler(fileLog) logger.setLevel(int(config.logFileErrorLoggingLevel)) socorro_cia.logger.addHandler(fileLog) socorro_cia.logger.setLevel(int(config.logFileErrorLoggingLevel)) topcrasher.logger.addHandler(fileLog) topcrasher.logger.setLevel(int(config.logFileErrorLoggingLevel)) topcrashbyurl.logger.addHandler(fileLog) topcrashbyurl.logger.setLevel(int(config.logFileErrorLoggingLevel)) logger.info("Config is\n%s",str(config)) createData(config,logger)
def setUp(self): global me self.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing MTBF') myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir':'%s'%myDir} for i in self.config: try: self.config[i] = self.config.get(i)%(replDict) except: pass self.logger = TestingLogger(me.fileLogger) self.connection = psycopg2.connect(me.dsn) cursor = self.connection.cursor() self.testDB = TestDB() self.testDB.removeDB(self.config,self.logger) self.testDB.createDB(self.config,self.logger) self.prods = ['zorro','vogel','lizz',] self.oss = ['OSX','LOX','WOX',] self.productDimData = [] # filled in by fillMtbfTables
def setup_module(): global me tutil.nosePrintModule(__file__) if me: return me = Me() me.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing TCByUrl') tcbu.logger.setLevel(logging.DEBUG) myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir':'%s'%myDir} for i in me.config: try: me.config[i] = me.config.get(i)%(replDict) except: pass me.logFilePathname = me.config.logFilePathname logFileDir = os.path.split(me.logFilePathname)[0] try: os.makedirs(logFileDir) except OSError,x: if errno.EEXIST == x.errno: pass else: raise
def setup_module(): global me if me: return me = Me() me.config = configurationManager.newConfiguration( configurationModule=testConfig, applicationName='Testing dbtestutil') tutil.nosePrintModule(__file__) myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir': '%s' % myDir} for i in me.config: try: me.config[i] = me.config.get(i) % (replDict) except: pass logger.setLevel(logging.DEBUG) logFilePathname = me.config.logFilePathname logfileDir = os.path.split(me.config.logFilePathname)[0] try: os.makedirs(logfileDir) except OSError, x: if errno.EEXIST != x.errno: raise
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # Deliberately use base-class's Option class just to see if things break (they better not) import socorro.lib.ConfigurationManager as cm testOption0 = cm.Option() testOption0.doc = 'option 0' testOption0.default = 0 testOption1 = cm.Option() testOption1.doc = 'option one' testOption1.default = 'one' logFilePathname = cm.Option() logFilePathname.default = '/some/bogus/location'
import logging import logging.handlers import sys try: import config.fixbrokendumpsconfig as configModule except ImportError: import fixbrokendumpsconfig as configModule import socorro.lib.ConfigurationManager as configurationManager import socorro.cron.fixBrokenDumps as fixBrokenDumps import socorro.lib.util as sutil try: config = configurationManager.newConfiguration(configurationModule=configModule, applicationName="Fix Broken Dumps") except configurationManager.NotAnOptionError, x: print >>sys.stderr, x print >>sys.stderr, "for usage, try --help" sys.exit(1) logger = logging.getLogger("fix_broken_dumps") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) try: last_date_processed = fixBrokenDumps.fix(config, logger, config.brokenFirefoxLinuxQuery, config.brokenFirefoxLinuxFixer) last_date_processed = fixBrokenDumps.fix(config, logger, config.brokenFennecQuery, config.brokenFennecFixer)
import sys import logging import logging.handlers try: import config.duplicatesconfig as configModule except ImportError: import duplicatesconfig as configModule import socorro.lib.ConfigurationManager as configurationManager import socorro.cron.duplicates as duplicates import socorro.lib.util as sutil try: config = configurationManager.newConfiguration(configurationModule=configModule, applicationName="Duplicate Detector 0.1") except configurationManager.NotAnOptionError, x: print >>sys.stderr, x print >>sys.stderr, "for usage, try --help" sys.exit() logger = logging.getLogger("duplicates") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) try: duplicates.find_duplicates(config) finally: logger.info("done.")
import sys import logging import logging.handlers try: import config.dailyurlconfig as configModule except ImportError: import dailyurlconfig as configModule import socorro.lib.ConfigurationManager as configurationManager import socorro.cron.dailyUrl as url import socorro.lib.util as sutil try: config = configurationManager.newConfiguration( configurationModule=configModule, applicationName="Daily URL Dump 0.1") except configurationManager.NotAnOptionError, x: print >> sys.stderr, x print >> sys.stderr, "for usage, try --help" sys.exit() logger = logging.getLogger("dailyUrlDump") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) try: url.dailyUrlDump(config) finally: logger.info("done.")
#bigStorageRoot.doc = 'the root of the file system where big test set is found' #bigStorageRoot.default = '%(testDir)s/testdata' #dumpDirPrefix = cm.Option() #dumpDirPrefix.doc = 'dump directory names begin with this prefix' #dumpDirPrefix.default = 'tst_' #jsonFileSuffix = cm.Option() #jsonFileSuffix.doc = 'the suffix used to identify a json file' #jsonFileSuffix.default = '.json' #dumpFileSuffix = cm.Option() #dumpFileSuffix.doc = 'the suffix used to identify a dump file' #dumpFileSuffix.default = '.dump' processorCheckInTime = cm.Option() processorCheckInTime.doc = 'the time after which a processor is considered dead (HH:MM:SS)' processorCheckInTime.default = "00:05:00" processorCheckInTime.fromStringConverter = lambda x: str( cm.timeDeltaConverter(x)) standardLoopDelay = cm.Option() standardLoopDelay.doc = 'the time between scans for jobs (HHH:MM:SS)' standardLoopDelay.default = '00:05:00' standardLoopDelay.fromStringConverter = cm.timeDeltaConverter cleanupJobsLoopDelay = cm.Option() cleanupJobsLoopDelay.doc = 'the time between runs of the job clean up routines (HHH:MM:SS)' cleanupJobsLoopDelay.default = '00:05:00' cleanupJobsLoopDelay.fromStringConverter = cm.timeDeltaConverter
import sys import logging import logging.handlers try: import config.bugzillaconfig as configModule except ImportError: import bugzillaconfig as configModule import socorro.lib.ConfigurationManager as configurationManager import socorro.cron.bugzilla as bug import socorro.lib.util as sutil try: config = configurationManager.newConfiguration(configurationModule=configModule, applicationName="Bugzilla Associations 0.1") except configurationManager.NotAnOptionError, x: print >>sys.stderr, x print >>sys.stderr, "for usage, try --help" sys.exit() logger = logging.getLogger("bugzilla") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) try: bug.record_associations(config) finally: logger.info("done.")
from socorro.unittest.config.commonconfig \ import databaseHost as database_hostname try: from socorro.unittest.config.commonconfig \ import databasePort as database_port except: database_port = 5432 from socorro.unittest.config.commonconfig \ import oldDatabaseName as database_name from socorro.unittest.config.commonconfig \ import databaseUserName as database_username from socorro.unittest.config.commonconfig \ import databasePassword as database_password logFilePathname = cm.Option() logFilePathname.doc = 'full pathname for the log file' logFilePathname.default = '%(testDir)s/logs/lib_test.log' logFileMaximumSize = cm.Option() logFileMaximumSize.doc = 'maximum size in bytes of the log file' logFileMaximumSize.default = 1000000 logFileMaximumBackupHistory = cm.Option() logFileMaximumBackupHistory.doc = 'maximum number of log files to keep' logFileMaximumBackupHistory.default = 50 logFileLineFormatString = cm.Option() logFileLineFormatString.doc = 'python logging system format for log file entries' logFileLineFormatString.default = '%(asctime)s %(levelname)s - %(message)s'
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import socorro.lib.ConfigurationManager as cm testNil = cm.Option() testSingleCharacter = cm.Option() testSingleCharacter.singleCharacter = 'T' testDefault = cm.Option() testDefault.default = 'default' testDoc = cm.Option() testDoc.doc = 'test doc'
import logging import logging.handlers import sys import datetime as dt try: import config.createpartitionsconfig as configModule except ImportError: import createpartitionsconfig as configModule import socorro.lib.ConfigurationManager as configurationManager import socorro.database.schema as schema import socorro.lib.util as sutil try: config = configurationManager.newConfiguration(configurationModule=configModule, applicationName="startNextPartition") except configurationManager.NotAnOptionError, x: print >>sys.stderr, x print >>sys.stderr, "for usage, try --help" sys.exit(1) logger = logging.getLogger("nextPartition") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) try: config["endDate"] = config.startDate + dt.timedelta(config.weeksIntoFuture * 7) print schema.createPartitions(config, logger)
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import web import socorro.webapi.class_partial as cpart import socorro.lib.ConfigurationManager as cm import socorro.collector.wsgicollector as wscol import socorro.lib.util as sutil #import socorro.webapi.hello as hello import config.collectorconfig as collectorConfig #------------------------------------------------------------------------------- config = \ cm.newConfiguration(configurationModule=collectorConfig, applicationName="Socorro Collector 3.0") #------------------------------------------------------------------------------- import logging import logging.handlers logger = logging.getLogger("collector") logger.setLevel(logging.DEBUG) syslog = logging.handlers.SysLogHandler( address=(config.syslogHost, config.syslogPort), facility=config.syslogFacilityString, ) syslog.setLevel(config.syslogErrorLoggingLevel) syslogFormatter = logging.Formatter(config.syslogLineFormatString) syslog.setFormatter(syslogFormatter)
import logging import logging.handlers import sys import time try: import config.topCrashesByUrlConfig as configModule except ImportError: import topCrashesByUrlConfig as configModule import socorro.lib.ConfigurationManager as configurationManager import socorro.cron.topCrashesByUrl as tcbyurl import socorro.lib.util as sutil config = configurationManager.newConfiguration( configurationModule=configModule, applicationName="Top Crash By URL Summary") logger = logging.getLogger("topCrashesByUrl") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) try: before = time.time() tu = tcbyurl.TopCrashesByUrl(config) tu.processDateInterval() logger.info("Successfully ran in %d seconds" % (time.time() - before)) finally: logger.info("done.")
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import sys import logging import logging.handlers import config.crashmoverconfig as cmconf import socorro.lib.ConfigurationManager as configurationManager import socorro.storage.storageMover as smover import socorro.lib.util as sutil try: config = configurationManager.newConfiguration( configurationModule=cmconf, applicationName="New Crash Mover") except configurationManager.NotAnOptionError, x: print >> sys.stderr, x print >> sys.stderr, "for usage, try --help" sys.exit() logger = logging.getLogger("newCrashMover") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) config.logger = logger try: smover.move(config)
import sys import logging import logging.handlers from datetime import date, timedelta try: import config.dailyMatviewsConfig as configModule except ImportError: import dailyMatviewsConfig as configModule import socorro.lib.ConfigurationManager as configurationManager import socorro.cron.dailyMatviews as dailyMatviews import socorro.lib.util as sutil try: config = configurationManager.newConfiguration( configurationModule=configModule, applicationName="dailyMatviews 0.1") except configurationManager.NotAnOptionError, x: print >> sys.stderr, x print >> sys.stderr, "for usage, try --help" sys.exit() logger = logging.getLogger("dailyMatviews") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) exitCode = 255 try: targetDate = date.today() - timedelta(1)
import sys import logging import logging.handlers try: import config.updateadus as configModule except ImportError: import updateadus as configModule import socorro.lib.ConfigurationManager as configurationManager import socorro.cron.updateADUs as updateADUs import socorro.lib.util as sutil try: config = configurationManager.newConfiguration( configurationModule=configModule, applicationName="Update ADUs 0.1") except configurationManager.NotAnOptionError, x: print >> sys.stderr, x print >> sys.stderr, "for usage, try --help" sys.exit() logger = logging.getLogger("updateADUs") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) try: updateADUs.update_adus(config) finally: logger.info("done.")
# XXX Set to be deprecated in favor of socorro/external/postgresql/models.py import socorro.database.database as db import psycopg2 import psycopg2.extensions import logging import threading from socorro.unittest.testlib.loggerForTest import TestingLogger from createDBforTest import * import socorro.lib.util as util from socorro.unittest.testbase import TestCase import socorro.lib.ConfigurationManager as cm import dbTestconfig as testConfig config = cm.newConfiguration(configurationModule=testConfig, applicationName='Testing Psycopghelper') """ Assume that psycopg2 works, then all we need to do is assure ourselves that our simplistic wrap around a returned array is correct """ class TestMultiCursor(psycopg2.extensions.cursor): def __init__(self, numCols=4, numRows=2, **kwargs): self.result = [] for i in range(numRows): aRow = [] for j in range(numCols): aRow.append('Row %d, Column %d' % (i, j)) self.result.append(aRow) self.next = self.__next()
import web import datetime as dt import config.webapiconfig as configModule from config import revisionsconfig import socorro.lib.ConfigurationManager as configurationManager import socorro.lib.datetimeutil as dtutil import socorro.lib.productVersionCache as pvc import socorro.webapi.webapiService as webapi import socorro.lib.util as sutil import logging import logging.handlers config = configurationManager.newConfiguration( configurationModule=configModule, applicationName="Socorro Webapi") # Adding revisions of Socorro and Breakpad for the server status service. revisions = configurationManager.newConfiguration( configurationModule=revisionsconfig, applicationName="Socorro Revisions") config.update(revisions) logger = logging.getLogger("webapi") logger.setLevel(logging.DEBUG) syslog = logging.handlers.SysLogHandler(facility=config.syslogFacilityString) syslog.setLevel(config.syslogErrorLoggingLevel) syslogFormatter = logging.Formatter(config.syslogLineFormatString) syslog.setFormatter(syslogFormatter) logger.addHandler(syslog)
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. ## Test config file for database utilities import socorro.lib.ConfigurationManager as cm import datetime from socorro.unittest.config.commonconfig import databaseHost from socorro.unittest.config.commonconfig import databaseName from socorro.unittest.config.commonconfig import databaseUserName from socorro.unittest.config.commonconfig import databasePassword logFilePathname = cm.Option() logFilePathname.doc = 'full pathname for the log file' logFilePathname.default = '%(testDir)s/logs/db_test.log'
def setUp(self): """Create a configuration context.""" self.config = ConfigurationManager.newConfiguration( configurationModule=commonconfig, applicationName="RabbitMQ Tests")
import unittest import socorro.database.database as db import psycopg2 import psycopg2.extensions import logging import threading from socorro.unittest.testlib.loggerForTest import TestingLogger import socorro.unittest.testlib.util as tutil from createDBforTest import * import socorro.lib.util as util import socorro.lib.ConfigurationManager as cm import dbTestconfig as testConfig config = cm.newConfiguration(configurationModule = testConfig, applicationName='Testing Psycopghelper') def setup_module(): tutil.nosePrintModule(__file__) """ Assume that psycopg2 works, then all we need to do is assure ourselves that our simplistic wrap around a returned array is correct """ class TestMultiCursor(psycopg2.extensions.cursor): def __init__(self,numCols = 4, numRows=2, **kwargs): self.result = [] for i in range(numRows): aRow = [] for j in range(numCols): aRow.append('Row %d, Column %d' %(i,j))
import sys import logging import logging.handlers import config.submitterconfig as subconf import socorro.lib.ConfigurationManager as configurationManager import socorro.collector.submitter as sub import socorro.lib.util as sutil import poster poster.streaminghttp.register_openers() try: config = configurationManager.newConfiguration( configurationModule=subconf, applicationName="submitter 2.0") except configurationManager.NotAnOptionError, x: print >> sys.stderr, x print >> sys.stderr, "for usage, try --help" sys.exit() logger = logging.getLogger("submitter") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) config.logger = logger if config.numberOfSubmissions == 'forever': config.iteratorFunc = sub.createInfiniteFileSystemIterator