def testAcceptConfigFile(self): # Test failure with good config file, unknown option in that file try: copt = [('c', 'config', True, self.configTstPath, "the test config file")] CM.newConfiguration(configurationOptionsList=copt,optionNameForConfigFile = 'config', configurationFileRequired = True) assert(False) except CM.NotAnOptionError, e: assert(True)
def setup_module(): global me if me: return # else initialize # print "MODULE setup" me = Me() me.markingTemplate = "MARK %s: %s" me.startMark = 'start' me.endMark = 'end' me.testDB = TestDB() me.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing Monitor') myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir':'%s'%myDir} for i in me.config: try: me.config[i] = me.config.get(i)%(replDict) except: pass knownTests = [x for x in dir(TestMonitor) if x.startswith('test')] me.logWasExtracted = {} for t in knownTests: me.logWasExtracted[t] = False me.logger = monitor.logger me.logger.setLevel(logging.DEBUG) me.logFilePathname = me.config.logFilePathname logfileDir = os.path.split(me.config.logFilePathname)[0] try: os.makedirs(logfileDir) except OSError,x: if errno.EEXIST != x.errno: raise
def main(): logger = logging.getLogger("topcrashes_summary") logger.setLevel(logging.WARNING) stderrLog = logging.StreamHandler() stderrLog.setLevel(logging.WARNING) stderrLogFormatter = logging.Formatter('%(asctime)s %(levelname)s - %(message)s') stderrLog.setFormatter(stderrLogFormatter) logger.addHandler(stderrLog) kwargs = {} for i in sys.argv[1:]: if i.startswith('-h') or i.startswith('--he'): help() sys.exit(0) j = i if i.startswith('-'): j = i.lstrip('-') if '=' in j: name,value = (s.strip() for s in j.split('=')) kwargs[name] = value else: print >> sys.stderr,"Ignoring unkown argument '%s'"%(i) sys.argv = sys.argv[:1] config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Create Database') config.update(kwargs) testDB = TestDB() testDB.removeDB(config,logger) testDB.createDB(config,logger)
def setup_module(): global me if me: return me = Me() tutil.nosePrintModule(__file__) me.testDB = TestDB() me.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='TestingCachedIdAccess') myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir':'%s'%myDir} for i in me.config: try: me.config[i] = me.config.get(i)%(replDict) except: pass cia.logger.setLevel(logging.DEBUG) me.logFilePathname = me.config.logFilePathname logfileDir = os.path.split(me.config.logFilePathname)[0] try: os.makedirs(logfileDir) except OSError,x: if errno.EEXIST != x.errno: raise f = open(me.config.logFilePathname,'w') f.close()
def setup_module(): global me if me: return me = Me() me.config = cfgManager.newConfiguration( configurationModule=testConfig, applicationName='Testing ftpscraper') myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir': '%s' % myDir} for i in me.config: try: me.config[i] = me.config.get(i) % (replDict) except: pass me.logFilePathname = me.config.logFilePathname if not me.logFilePathname: me.logFilePathname = 'logs/ftpscraper_test.log' logFileDir = os.path.split(me.logFilePathname)[0] try: os.makedirs(logFileDir) except OSError, x: if errno.EEXIST == x.errno: pass else: raise
def main(): logger = logging.getLogger("topcrashes_summary") logger.setLevel(logging.WARNING) stderrLog = logging.StreamHandler() stderrLog.setLevel(logging.WARNING) stderrLogFormatter = logging.Formatter( '%(asctime)s %(levelname)s - %(message)s') stderrLog.setFormatter(stderrLogFormatter) logger.addHandler(stderrLog) kwargs = {} for i in sys.argv[1:]: if i.startswith('-h') or i.startswith('--he'): help() sys.exit(0) j = i if i.startswith('-'): j = i.lstrip('-') if '=' in j: name, value = (s.strip() for s in j.split('=')) kwargs[name] = value else: print >> sys.stderr, "Ignoring unkown argument '%s'" % (i) sys.argv = sys.argv[:1] config = configurationManager.newConfiguration( configurationModule=testConfig, applicationName='Create Database') config.update(kwargs) testDB = TestDB() testDB.removeDB(config, logger) testDB.createDB(config, logger)
def setup_module(): global me if me: return me = Me() tutil.nosePrintModule(__file__) # config gets messed up by some tests. Use this one during module setup and teardown me.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing Postgresql Utils') myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir':'%s'%myDir} for i in me.config: try: me.config[i] = me.config.get(i)%(replDict) except: pass me.logFilePathname = me.config.logFilePathname if not me.logFilePathname: me.logFilePathname = 'logs/db_test.log' logFileDir = os.path.split(me.logFilePathname)[0] try: os.makedirs(logFileDir) except OSError,x: if errno.EEXIST == x.errno: pass else: raise
def setup_module(): global me if me: return me = Me() tutil.nosePrintModule(__file__) me.config = configurationManager.newConfiguration( configurationModule=testConfig, applicationName="Testing Postgresql Utils" ) myDir = os.path.split(__file__)[0] if not myDir: myDir = "." replDict = {"testDir": "%s" % myDir} for i in me.config: try: me.config[i] = me.config.get(i) % (replDict) except: pass me.logFilePathname = me.config.logFilePathname if not me.logFilePathname: me.logFilePathname = "logs/db_test.log" logFileDir = os.path.split(me.logFilePathname)[0] try: os.makedirs(logFileDir) except OSError, x: if errno.EEXIST == x.errno: pass else: raise
def setup_module(): global me if me: return me = Me() me.config = cfgManager.newConfiguration(configurationModule=testConfig, applicationName='Testing ftpscraper') myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir': '%s' % myDir} for i in me.config: try: me.config[i] = me.config.get(i) % (replDict) except: pass me.logFilePathname = me.config.logFilePathname if not me.logFilePathname: me.logFilePathname = 'logs/ftpscraper_test.log' logFileDir = os.path.split(me.logFilePathname)[0] try: os.makedirs(logFileDir) except OSError, x: if errno.EEXIST == x.errno: pass else: raise
def setupConfig(): try: return configurationManager.newConfiguration(configurationModule=config, applicationName="Migrate Process Type") except configurationManager.NotAnOptionError, x: print >>sys.stderr, x print >>sys.stderr, "for usage, try --help" sys.exit(1)
def setupConfig(): try: return configurationManager.newConfiguration( configurationModule=config, applicationName="Migrate Process Type") except configurationManager.NotAnOptionError, x: print >> sys.stderr, x print >> sys.stderr, "for usage, try --help" sys.exit(1)
def setUp(self): """Create a configuration context and a database connection. """ self.config = ConfigurationManager.newConfiguration( configurationModule=commonconfig, applicationName="PostgreSQL Tests") self.database = db.Database(self.config) self.connection = self.database.connection()
def testAcceptUserCommandLineHelp(self): opts = [('h','help',False,False,'another help')] args = {} args['automaticHelp'] = False args['configurationOptionsList'] = opts hh = HelpHandler() args['helpHandler'] = hh.handleHelp sys.argv.append('--help') conf = CM.newConfiguration(**args) assert("--help" in hh.data) assert("another help" in hh.data)
def testAcceptAutoCommandLineHelp(self): opts = [] args = {} args['automaticHelp'] = True args['configurationOptionsList'] = opts hh = HelpHandler() args['helpHandler'] = hh.handleHelp sys.argv.append('--help') conf = CM.newConfiguration(**args) assert("--help" in hh.data) assert("print this list" in hh.data)
def setUp(self): global me # config gets messed up by some tests. Use this one by preference self.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing Postgresql Utils') for i in self.config: try: self.config[i] = self.config.get(i)%(replDict) except: pass self.connection = psycopg2.connect(me.dsn) self.testDB = TestDB()
def setUp(self): """Create a configuration context and a database connection. """ self.config = ConfigurationManager.newConfiguration( configurationModule=commonconfig, applicationName="PostgreSQL Tests") try: self.database = db.Database(self.config) except (AttributeError, KeyError): raise self.connection = self.database.connection()
def setUp(self): """Create a configuration context and a database connection. """ self.config = ConfigurationManager.newConfiguration( configurationModule=commonconfig, applicationName="PostgreSQL Tests" ) try: self.database = db.Database(self.config) except (AttributeError, KeyError): raise self.connection = self.database.connection()
def testAcceptCommandLine(self): opts = [] args = {} opts.append(('c','chickensoup',False,False,'help for the ailing')) opts.append(('r','rabbit', True, '', 'rabbits are bunnies')) args['configurationOptionsList'] = opts sys.argv.append('-c') sys.argv.append('--rabbit=bunny') conf = CM.newConfiguration(**args) assert('chickensoup' in conf) assert('rabbit' in conf) assert('bunny' == conf.rabbit)
def testAcceptEnvironment(self): opts = [] args = {} opts.append(('c','chickensoup',False,False,'help for the ailing')) opts.append(('r','rabbit', True, '', 'rabbits are bunnies')) args['configurationOptionsList'] = opts os.environ['chickensoup']='' os.environ['r'] = 'bunny-rabbit' conf = CM.newConfiguration(**args) assert('chickensoup' in conf) assert('rabbit' in conf) assert('bunny-rabbit' == conf.rabbit)
def testAcceptTypePriority(self): '''Assure that commandline beats config file beats environment beats defaults''' copt = [('c', 'config', True, self.configTstPath, "the test config file"),('r','rabbit', True, 'bambi', 'rabbits are bunnies')] copt.append(('b','badger',True,'gentle','some badgers are gentle')) copt.append(('z','zeta', True, 'zebra', 'zebras ooze')) os.environ['badger'] = 'bloody' os.environ['zeta'] = 'zymurgy' sys.argv.append('--rabbit=kangaroo') conf = CM.newConfiguration(automaticHelp=False,configurationOptionsList=copt,optionNameForConfigFile = 'config', configurationFileRequired = True) assert('kangaroo' == conf.rabbit) # command line beats config file assert('this badger=awful' == conf.badger) # config file beats environment assert('zymurgy' == conf.zeta)
def setUp(self): global me # config gets messed up by some tests. Use this one by preference self.config = configurationManager.newConfiguration( configurationModule=testConfig, applicationName='Testing Postgresql Utils') for i in self.config: try: self.config[i] = self.config.get(i) % (replDict) except: pass self.connection = psycopg2.connect(me.dsn) self.testDB = TestDB()
def setUp(self): global me self.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing builds') myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir':'%s'%myDir} for i in self.config: try: self.config[i] = self.config.get(i)%(replDict) except: pass self.logger = TestingLogger(me.fileLogger) self.testConfig = configurationManager.Config([('t','testPath', True, './TEST-BUILDS', ''), ('f','testFileName', True, 'lastrun.pickle', ''), ]) self.testConfig["persistentDataPathname"] = os.path.join(self.testConfig.testPath, self.testConfig.testFileName)
def createMe(): global me if not me: me = Me() me.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName = "Testing TopCrashers") myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir':'%s'%myDir} for i in me.config: try: me.config[i] = me.config.get(i)%(replDict) except: pass me.logFilePathname = me.config.logFilePathname logfileDir = os.path.split(me.config.logFilePathname)[0] try: os.makedirs(logfileDir) except OSError,x: if errno.EEXIST != x.errno: raise
def testNewConfiguration(self): ''' Trick: To call f(**kwargs) with a dictionary d as the single arg, chant f(**d) ''' opts = [] args = {} args['automaticHelp'] = False # Test for empty conf = CM.newConfiguration(**args) assert(not conf.allowableOptionDictionary) # Test for autoHelp args['automaticHelp'] = True conf = CM.newConfiguration(**args) assert(2 == len(conf.allowableOptionDictionary)) # Test for another legal option opts.append(('c','chickensoup',False,False,'Help for the ailing')) args['automaticHelp'] = True args['configurationOptionsList'] = opts conf = CM.newConfiguration(**args) assert(4 == len(conf.allowableOptionDictionary)) # Test a config module conf = CM.newConfiguration(automaticHelp=False,configurationModule=optionfile) cd = conf.allowableOptionDictionary assert(5 == len(cd)) assert ['T', 'testSingleCharacter', True, None] == cd.get('T')[:-1],'but got %s' % (str(cd.get('T')[:-1])) assert 'testSingleCharacter imported from' in cd.get('T')[-1],'but got %s' % (str(cd.get('T')[-1])) assert 'optionfile' in cd.get('T')[-1],'but got %s' % (str(cd.get('T')[-1])) #assert ['T', 'testSingleCharacter', True, None, 'testSingleCharacter imported from optionfile'] == cd.get('T'),'but got %s' % (str(cd.get('T'))) assert [None, 'testDefault', True, 'default'] == cd.get('testDefault')[:-1], "but got %s" %(str(cd.get('testDefault')[:-1])) assert 'testDefault imported from' in cd.get('testDefault')[-1], "but got %s" %(cd.get('testDefault')[-1]) assert 'optionfile' in cd.get('testDefault')[-1], "but got %s" %(cd.get('testDefault')[-1]) #assert([None, 'testDefault', True, 'default', 'testDefault imported from optionfile'] == cd.get('testDefault')) assert([None, 'testDoc', True, None, 'test doc'] == cd.get('testDoc')) assert [None, 'testNil', True, None] == cd.get('testNil')[:-1], "but got %s" %(str(cd.get('testNil')[:-1])) assert 'testNil imported from' in cd.get('testNil')[-1], "but got %s" %(cd.get('testNil')[-1]) assert 'optionfile' in cd.get('testNil')[-1], "but got %s" %(cd.get('testNil')[-1]) #assert([None, 'testNil', True, None, 'testNil imported from optionfile'] == cd.get('testNil')) # Test failure with good option, bad file try: copt = [('c', 'config', True, './badone', "the badconfig file")] CM.newConfiguration(configurationOptionsList=copt,optionNameForConfigFile = 'config', configurationFileRequired = True) assert(False) except CM.ConfigFileMissingError, e: assert(True)
def setUp(self): configContext = configurationManager.newConfiguration(configurationModule=config, applicationName="Top Crash By URL Summary") logger = logging.getLogger("tcbyurl_summary") logger.setLevel(logging.DEBUG) stderrLog = logging.StreamHandler() stderrLog.setLevel(configContext.stderrErrorLoggingLevel) stderrLogFormatter = logging.Formatter(configContext.stderrLineFormatString) stderrLog.setFormatter(stderrLogFormatter) logger.addHandler(stderrLog) rotatingFileLog = logging.handlers.RotatingFileHandler(configContext.logFilePathname, "a", configContext.logFileMaximumSize, configContext.logFileMaximumBackupHistory) rotatingFileLog.setLevel(logging.DEBUG) rotatingFileLogFormatter = logging.Formatter(configContext.logFileLineFormatString) rotatingFileLog.setFormatter(rotatingFileLogFormatter) logger.addHandler(rotatingFileLog) logger.info("current configuration\n%s", str(configContext)) self.configContext = configContext self.logger = logger self.rotatingFileLog = rotatingFileLog
def createMe(): global me if not me: me = Me() me.config = configurationManager.newConfiguration( configurationModule=testConfig, applicationName="Testing TopCrashers") myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir': '%s' % myDir} for i in me.config: try: me.config[i] = me.config.get(i) % (replDict) except: pass me.logFilePathname = me.config.logFilePathname logfileDir = os.path.split(me.config.logFilePathname)[0] try: os.makedirs(logfileDir) except OSError, x: if errno.EEXIST != x.errno: raise
def main(): addedConfigOptions = [ ('h','help',False,None,'print this list',None), ('P','product-os-count',True,12,'how many product/os pairs to use [1-56]',lambda x: rangeChecker(1,56,x) ), ('S','signature-count',True,83,'how many signatures to use [1-121]',lambda x: rangeChecker(1,121,x)), ('R','repeat-count',True,2,'how many times to loop [smallish N]',None), ('m','mtbf-fill',False,True,'Fill the time_before_failure table',lambda x: True), ('u','url-fill',False,True,'Fill the top_crashes_by_url table',lambda x: True), ('s','sig-fill',False,True,'Fill the top_crashes_by_signature table',lambda x: True), ('a','all-fill',False,True,'Fill all three matrialized view tables',[('mtbf-fill',None),('url-fill',None),('sig-fill',None)]), ('D','drop-all',False,True,'Drop all the database tables. Do no other work',lambda x: True), (None,'logFileErrorLoggingLevel',True,logging.WARNING,'logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)',None), ] config = configurationManager.newConfiguration(configurationModule = commonConfig, applicationName='fillSchema.py',configurationOptionsList = addedConfigOptions) myDir = os.path.split(__file__)[0] logDir = os.path.join(myDir,'logs') logFile = os.path.join(logDir,'fillDB.log') try: os.makedirs(logDir) except OSError: pass logger = logging.getLogger("fillDB") fileLog = logging.FileHandler(logFile) fileLog.setLevel(int(config.logFileErrorLoggingLevel)) fileLogFormatter = logging.Formatter('%(asctime)s %(levelname)s - %(message)s') fileLog.setFormatter(fileLogFormatter) # now add the handler for all the logs of interest logger.addHandler(fileLog) logger.setLevel(int(config.logFileErrorLoggingLevel)) socorro_cia.logger.addHandler(fileLog) socorro_cia.logger.setLevel(int(config.logFileErrorLoggingLevel)) topcrasher.logger.addHandler(fileLog) topcrasher.logger.setLevel(int(config.logFileErrorLoggingLevel)) topcrashbyurl.logger.addHandler(fileLog) topcrashbyurl.logger.setLevel(int(config.logFileErrorLoggingLevel)) logger.info("Config is\n%s",str(config)) createData(config,logger)
def setUp(self): global me self.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing MTBF') myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir':'%s'%myDir} for i in self.config: try: self.config[i] = self.config.get(i)%(replDict) except: pass self.logger = TestingLogger(me.fileLogger) self.connection = psycopg2.connect(me.dsn) cursor = self.connection.cursor() self.testDB = TestDB() self.testDB.removeDB(self.config,self.logger) self.testDB.createDB(self.config,self.logger) self.prods = ['zorro','vogel','lizz',] self.oss = ['OSX','LOX','WOX',] self.productDimData = [] # filled in by fillMtbfTables
def setup_module(): global me tutil.nosePrintModule(__file__) if me: return me = Me() me.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing TCByUrl') tcbu.logger.setLevel(logging.DEBUG) myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir':'%s'%myDir} for i in me.config: try: me.config[i] = me.config.get(i)%(replDict) except: pass me.logFilePathname = me.config.logFilePathname logFileDir = os.path.split(me.logFilePathname)[0] try: os.makedirs(logFileDir) except OSError,x: if errno.EEXIST == x.errno: pass else: raise
def setup_module(): global me if me: return me = Me() me.config = configurationManager.newConfiguration( configurationModule=testConfig, applicationName='Testing dbtestutil') myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir': '%s' % myDir} for i in me.config: try: me.config[i] = me.config.get(i) % (replDict) except: pass logger.setLevel(logging.DEBUG) logFilePathname = me.config.logFilePathname logfileDir = os.path.split(me.config.logFilePathname)[0] try: os.makedirs(logfileDir) except OSError, x: if errno.EEXIST != x.errno: raise
def setUp(self): global me self.config = cfgManager.newConfiguration( configurationModule=testConfig, applicationName='Testing ftpscraper') myDir = os.path.split(__file__)[0] if not myDir: myDir = '.' replDict = {'testDir': '%s' % myDir} for i in self.config: try: self.config[i] = self.config.get(i) % (replDict) except: pass self.logger = TestingLogger(me.fileLogger) self.testConfig = cfgManager.Config([ ('t', 'testPath', True, './TEST-BUILDS', ''), ('f', 'testFileName', True, 'lastrun.pickle', '') ]) self.testConfig["persistentDataPathname"] = os.path.join( self.testConfig.testPath, self.testConfig.testFileName)
import sys import logging import logging.handlers try: import config.duplicatesconfig as configModule except ImportError: import duplicatesconfig as configModule import socorro.lib.ConfigurationManager as configurationManager import socorro.cron.duplicates as duplicates import socorro.lib.util as sutil try: config = configurationManager.newConfiguration(configurationModule=configModule, applicationName="Duplicate Detector 0.1") except configurationManager.NotAnOptionError, x: print >>sys.stderr, x print >>sys.stderr, "for usage, try --help" sys.exit() logger = logging.getLogger("duplicates") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) try: duplicates.find_duplicates(config) finally: logger.info("done.")
import datetime import json import os import tarfile import time import sys import config.commonconfig as configModule import socorro.database.database as db import socorro.lib.ConfigurationManager as configurationManager import socorro.lib.datetimeutil as dtu import socorro.lib.httpclient as httpc config = configurationManager.newConfiguration( configurationModule=configModule, applicationName='movecrashes.py' ) def export_uuids(path, numberofdays): """Export crash report uuids from a PostgreSQL database to a CSV file path - Directory where the csv file will be created. numberofdays - Number of days of crash reports to retrieve, before the most recent crash date. """ database = db.Database(config) connection = database.connection() cur = connection.cursor()
import sys import logging import logging.handlers from datetime import date, timedelta try: import config.dailyMatviewsConfig as configModule except ImportError: import dailyMatviewsConfig as configModule import socorro.lib.ConfigurationManager as configurationManager import socorro.cron.dailyMatviews as dailyMatviews import socorro.lib.util as sutil try: config = configurationManager.newConfiguration( configurationModule=configModule, applicationName="dailyMatviews 0.1") except configurationManager.NotAnOptionError, x: print >>sys.stderr, x print >>sys.stderr, "for usage, try --help" sys.exit() logger = logging.getLogger("dailyMatviews") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) exitCode = 255 try: targetDate = date.today() - timedelta(1)
import web import datetime as dt import config.webapiconfig as configModule from config import revisionsconfig import socorro.lib.ConfigurationManager as configurationManager import socorro.lib.datetimeutil as dtutil import socorro.lib.productVersionCache as pvc import socorro.webapi.webapiService as webapi import socorro.lib.util as sutil import logging import logging.handlers config = configurationManager.newConfiguration(configurationModule=configModule, applicationName="Socorro Webapi") # Adding revisions of Socorro and Breakpad for the server status service. revisions = configurationManager.newConfiguration( configurationModule=revisionsconfig, applicationName="Socorro Revisions" ) config.update(revisions) logger = logging.getLogger("webapi") logger.setLevel(logging.DEBUG) syslog = logging.handlers.SysLogHandler(facility=config.syslogFacilityString) syslog.setLevel(config.syslogErrorLoggingLevel) syslogFormatter = logging.Formatter(config.syslogLineFormatString) syslog.setFormatter(syslogFormatter)
import sys import logging import logging.handlers try: import config.monitorconfig as configModule except ImportError: import monitorconfig as configModule import socorro.monitor.monitor as monitor import socorro.lib.ConfigurationManager as configurationManager import socorro.lib.util as sutil try: config = configurationManager.newConfiguration(configurationModule=configModule, applicationName="Socorro Monitor 2.0") except configurationManager.NotAnOptionError, x: print >>sys.stderr, x print >>sys.stderr, "for usage, try --help" sys.exit() logger = logging.getLogger("monitor") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) try: while True: m = monitor.Monitor(config) m.start()
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import web import socorro.webapi.class_partial as cpart import socorro.lib.ConfigurationManager as cm import socorro.collector.wsgicollector as wscol import socorro.lib.util as sutil #import socorro.webapi.hello as hello import config.collectorconfig as collectorConfig #------------------------------------------------------------------------------- config = \ cm.newConfiguration(configurationModule=collectorConfig, applicationName="Socorro Collector 3.0") #------------------------------------------------------------------------------- import logging import logging.handlers logger = logging.getLogger("collector") logger.setLevel(logging.DEBUG) syslog = logging.handlers.SysLogHandler( address=(config.syslogHost, config.syslogPort), facility=config.syslogFacilityString, ) syslog.setLevel(config.syslogErrorLoggingLevel) syslogFormatter = logging.Formatter(config.syslogLineFormatString) syslog.setFormatter(syslogFormatter)
import sys import logging import logging.handlers try: import config.dailyurlconfig as configModule except ImportError: import dailyurlconfig as configModule import socorro.lib.ConfigurationManager as configurationManager import socorro.cron.dailyUrl as url import socorro.lib.util as sutil try: config = configurationManager.newConfiguration( configurationModule=configModule, applicationName="Daily URL Dump 0.1") except configurationManager.NotAnOptionError, x: print >> sys.stderr, x print >> sys.stderr, "for usage, try --help" sys.exit() logger = logging.getLogger("dailyUrlDump") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) try: url.dailyUrlDump(config) finally: logger.info("done.")
import logging import logging.handlers import sys import time try: import config.topCrashesByUrlConfig as configModule except ImportError: import topCrashesByUrlConfig as configModule import socorro.lib.ConfigurationManager as configurationManager import socorro.cron.topCrashesByUrl as tcbyurl import socorro.lib.util as sutil config = configurationManager.newConfiguration( configurationModule=configModule, applicationName="Top Crash By URL Summary") logger = logging.getLogger("topCrashesByUrl") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) try: before = time.time() tu = tcbyurl.TopCrashesByUrl(config) tu.processDateInterval() logger.info("Successfully ran in %d seconds" % (time.time() - before)) finally: logger.info("done.")
def setUp(self): """Create a configuration context.""" self.config = ConfigurationManager.newConfiguration( configurationModule=commonconfig, applicationName="RabbitMQ Tests")
import web import datetime as dt import config.webapiconfig as configModule from config import revisionsconfig import socorro.lib.ConfigurationManager as configurationManager import socorro.lib.datetimeutil as dtutil import socorro.lib.productVersionCache as pvc import socorro.webapi.webapiService as webapi import socorro.lib.util as sutil import logging import logging.handlers config = configurationManager.newConfiguration( configurationModule=configModule, applicationName="Socorro Webapi") # Adding revisions of Socorro and Breakpad for the server status service. revisions = configurationManager.newConfiguration( configurationModule=revisionsconfig, applicationName="Socorro Revisions") config.update(revisions) logger = logging.getLogger("webapi") logger.setLevel(logging.DEBUG) syslog = logging.handlers.SysLogHandler(facility=config.syslogFacilityString) syslog.setLevel(config.syslogErrorLoggingLevel) syslogFormatter = logging.Formatter(config.syslogLineFormatString) syslog.setFormatter(syslogFormatter) logger.addHandler(syslog)
import logging import logging.handlers import sys try: import config.fixbrokendumpsconfig as configModule except ImportError: import fixbrokendumpsconfig as configModule import socorro.lib.ConfigurationManager as configurationManager import socorro.cron.fixBrokenDumps as fixBrokenDumps import socorro.lib.util as sutil try: config = configurationManager.newConfiguration( configurationModule=configModule, applicationName="Fix Broken Dumps") except configurationManager.NotAnOptionError, x: print >> sys.stderr, x print >> sys.stderr, "for usage, try --help" sys.exit(1) logger = logging.getLogger("fix_broken_dumps") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) try: last_date_processed = fixBrokenDumps.fix(config, logger, config.brokenFirefoxLinuxQuery, config.brokenFirefoxLinuxFixer)
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import sys import logging import logging.handlers import config.crashmoverconfig as cmconf import socorro.lib.ConfigurationManager as configurationManager import socorro.storage.storageMover as smover import socorro.lib.util as sutil try: config = configurationManager.newConfiguration( configurationModule=cmconf, applicationName="New Crash Mover") except configurationManager.NotAnOptionError, x: print >> sys.stderr, x print >> sys.stderr, "for usage, try --help" sys.exit() logger = logging.getLogger("newCrashMover") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) config.logger = logger try: smover.move(config)
import sys import logging import logging.handlers try: import config.bugzillaconfig as configModule except ImportError: import bugzillaconfig as configModule import socorro.lib.ConfigurationManager as configurationManager import socorro.cron.bugzilla as bug import socorro.lib.util as sutil try: config = configurationManager.newConfiguration( configurationModule=configModule, applicationName="Bugzilla Associations 0.1") except configurationManager.NotAnOptionError, x: print >> sys.stderr, x print >> sys.stderr, "for usage, try --help" sys.exit() logger = logging.getLogger("bugzilla") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) try: bug.record_associations(config) finally:
import logging import logging.handlers import sys try: import config.serverstatusconfig as configModule except ImportError: import serverstatusconfig as configModule import socorro.lib.ConfigurationManager as configurationManager import socorro.cron.serverstatus as serverstatus import socorro.lib.util as sutil try: config = configurationManager.newConfiguration( configurationModule=configModule, applicationName="Server Status Summary") except configurationManager.NotAnOptionError, x: print >> sys.stderr, x print >> sys.stderr, "for usage, try --help" sys.exit(1) logger = logging.getLogger("server_status_summary") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) try: serverstatus.update(config, logger) finally:
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import sys try: import config.processorconfig as configModule except ImportError: import processorconfig as configModule import socorro.lib.ConfigurationManager as configurationManager try: config = configurationManager.newConfiguration( configurationModule=configModule, applicationName="Socorro Processor 2.9") except configurationManager.NotAnOptionError, x: print >> sys.stderr, x print >> sys.stderr, "for usage, try --help" sys.exit() def processor2008(config): import sys import logging import logging.handlers import socorro.lib.util as sutil import socorro.processor.externalProcessor as processor
""" import logging import logging.handlers import datetime try: import config.ftpscraperconfig as configModule except ImportError: import ftpscraperconfig as configModule import socorro.cron.ftpscraper as ftpscraper import socorro.lib.ConfigurationManager as cfgManager import socorro.lib.util as sutil config = cfgManager.newConfiguration(configurationModule=configModule, applicationName='startFtpScraper.py') assert "databaseHost" in config, "databaseHost missing from config" assert "databaseName" in config, "databaseName missing from config" assert "databaseUserName" in config, "databaseUserName missing from config" assert "databasePassword" in config, "databasePassword missing from config" assert "base_url" in config, "base_url missing from config" assert "products" in config, "products missing from config" assert "backfillDate" in config, "backfillDate missing from config" logger = logging.getLogger("ftpscraper") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) config.logger = logger
import sys import logging import logging.handlers try: import config.signaturesconfig as configModule except ImportError: import signaturesconfig as configModule import socorro.lib.ConfigurationManager as configurationManager import socorro.cron.signatures as signatures import socorro.lib.util as sutil try: config = configurationManager.newConfiguration(configurationModule=configModule, applicationName="Signatures 0.1") except configurationManager.NotAnOptionError, x: print >>sys.stderr, x print >>sys.stderr, "for usage, try --help" sys.exit() logger = logging.getLogger("signatures") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) try: signatures.update_signatures(config) finally: logger.info("done.")
import sys import logging import logging.handlers from datetime import date, timedelta try: import config.dailyMatviewsConfig as configModule except ImportError: import dailyMatviewsConfig as configModule import socorro.lib.ConfigurationManager as configurationManager import socorro.cron.dailyMatviews as dailyMatviews import socorro.lib.util as sutil try: config = configurationManager.newConfiguration( configurationModule=configModule, applicationName="dailyMatviews 0.1") except configurationManager.NotAnOptionError, x: print >> sys.stderr, x print >> sys.stderr, "for usage, try --help" sys.exit() logger = logging.getLogger("dailyMatviews") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) exitCode = 255 try: targetDate = date.today() - timedelta(1)
import web import socorro.webapi.classPartial as cpart import socorro.lib.ConfigurationManager as cm import socorro.collector.wsgicollector as wscol import socorro.lib.util as sutil #import socorro.webapi.hello as hello import config.collectorconfig as collectorConfig #------------------------------------------------------------------------------- config = \ cm.newConfiguration(configurationModule=collectorConfig, applicationName="Socorro Collector 3.0") #------------------------------------------------------------------------------- import logging import logging.handlers logger = logging.getLogger("collector") logger.setLevel(logging.DEBUG) syslog = logging.handlers.SysLogHandler( address=(config.syslogHost, config.syslogPort), facility=config.syslogFacilityString, ) syslog.setLevel(config.syslogErrorLoggingLevel) syslogFormatter = logging.Formatter(config.syslogLineFormatString) syslog.setFormatter(syslogFormatter) logger.addHandler(syslog) sutil.echoConfig(logger, config)
import sys import logging import logging.handlers try: import config.duplicatesconfig as configModule except ImportError: import duplicatesconfig as configModule import socorro.lib.ConfigurationManager as configurationManager import socorro.cron.duplicates as duplicates import socorro.lib.util as sutil try: config = configurationManager.newConfiguration( configurationModule=configModule, applicationName="Duplicate Detector 0.1") except configurationManager.NotAnOptionError, x: print >> sys.stderr, x print >> sys.stderr, "for usage, try --help" sys.exit() logger = logging.getLogger("duplicates") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) try: duplicates.find_duplicates(config) finally:
# License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import sys import logging import logging.handlers import config.orphansubmitterconf as cmconf import socorro.lib.ConfigurationManager as configurationManager import socorro.storage.orphans as smover import socorro.lib.util as sutil try: config = configurationManager.newConfiguration(configurationModule=cmconf, applicationName="Orphan Submitter") except configurationManager.NotAnOptionError, x: print >>sys.stderr, x print >>sys.stderr, "for usage, try --help" sys.exit() logger = logging.getLogger("orphanSubmitter") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) config.logger = logger try: smover.move(config)
import sys import logging import logging.handlers import config.submitterconfig as subconf import socorro.lib.ConfigurationManager as configurationManager import socorro.collector.submitter as sub import socorro.lib.util as sutil import poster poster.streaminghttp.register_openers() try: config = configurationManager.newConfiguration( configurationModule=subconf, applicationName="submitter 2.0") except configurationManager.NotAnOptionError, x: print >> sys.stderr, x print >> sys.stderr, "for usage, try --help" sys.exit() logger = logging.getLogger("submitter") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) config.logger = logger if config.numberOfSubmissions == 'forever': config.iteratorFunc = sub.createInfiniteFileSystemIterator
import logging import logging.handlers import sys try: import config.fixbrokendumpsconfig as configModule except ImportError: import fixbrokendumpsconfig as configModule import socorro.lib.ConfigurationManager as configurationManager import socorro.cron.fixBrokenDumps as fixBrokenDumps import socorro.lib.util as sutil try: config = configurationManager.newConfiguration(configurationModule=configModule, applicationName="Fix Broken Dumps") except configurationManager.NotAnOptionError, x: print >>sys.stderr, x print >>sys.stderr, "for usage, try --help" sys.exit(1) logger = logging.getLogger("fix_broken_dumps") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) try: last_date_processed = fixBrokenDumps.fix(config, logger, config.brokenFirefoxLinuxQuery, config.brokenFirefoxLinuxFixer) last_date_processed = fixBrokenDumps.fix(config, logger, config.brokenFennecQuery, config.brokenFennecFixer)
import sys import logging import logging.handlers try: import config.updateadus as configModule except ImportError: import updateadus as configModule import socorro.lib.ConfigurationManager as configurationManager import socorro.cron.updateADUs as updateADUs import socorro.lib.util as sutil try: config = configurationManager.newConfiguration( configurationModule=configModule, applicationName="Update ADUs 0.1") except configurationManager.NotAnOptionError, x: print >> sys.stderr, x print >> sys.stderr, "for usage, try --help" sys.exit() logger = logging.getLogger("updateADUs") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) try: updateADUs.update_adus(config) finally: logger.info("done.")
# XXX Set to be deprecated in favor of socorro/external/postgresql/models.py import socorro.database.database as db import psycopg2 import psycopg2.extensions import logging import threading from socorro.unittest.testlib.loggerForTest import TestingLogger from createDBforTest import * import socorro.lib.util as util from socorro.unittest.testbase import TestCase import socorro.lib.ConfigurationManager as cm import dbTestconfig as testConfig config = cm.newConfiguration(configurationModule=testConfig, applicationName='Testing Psycopghelper') """ Assume that psycopg2 works, then all we need to do is assure ourselves that our simplistic wrap around a returned array is correct """ class TestMultiCursor(psycopg2.extensions.cursor): def __init__(self, numCols=4, numRows=2, **kwargs): self.result = [] for i in range(numRows): aRow = [] for j in range(numCols): aRow.append('Row %d, Column %d' % (i, j)) self.result.append(aRow) self.next = self.__next()
import sys import logging import logging.handlers try: import config.bugzillaconfig as configModule except ImportError: import bugzillaconfig as configModule import socorro.lib.ConfigurationManager as configurationManager import socorro.cron.bugzilla as bug import socorro.lib.util as sutil try: config = configurationManager.newConfiguration(configurationModule=configModule, applicationName="Bugzilla Associations 0.1") except configurationManager.NotAnOptionError, x: print >>sys.stderr, x print >>sys.stderr, "for usage, try --help" sys.exit() logger = logging.getLogger("bugzilla") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) try: bug.record_associations(config) finally: logger.info("done.")
import unittest import socorro.database.database as db import psycopg2 import psycopg2.extensions import logging import threading from socorro.unittest.testlib.loggerForTest import TestingLogger import socorro.unittest.testlib.util as tutil from createDBforTest import * import socorro.lib.util as util import socorro.lib.ConfigurationManager as cm import dbTestconfig as testConfig config = cm.newConfiguration(configurationModule = testConfig, applicationName='Testing Psycopghelper') def setup_module(): tutil.nosePrintModule(__file__) """ Assume that psycopg2 works, then all we need to do is assure ourselves that our simplistic wrap around a returned array is correct """ class TestMultiCursor(psycopg2.extensions.cursor): def __init__(self,numCols = 4, numRows=2, **kwargs): self.result = [] for i in range(numRows): aRow = [] for j in range(numCols): aRow.append('Row %d, Column %d' %(i,j))
import logging import logging.handlers import sys import datetime as dt try: import config.createpartitionsconfig as configModule except ImportError: import createpartitionsconfig as configModule import socorro.lib.ConfigurationManager as configurationManager import socorro.database.schema as schema import socorro.lib.util as sutil try: config = configurationManager.newConfiguration(configurationModule=configModule, applicationName="startNextPartition") except configurationManager.NotAnOptionError, x: print >>sys.stderr, x print >>sys.stderr, "for usage, try --help" sys.exit(1) logger = logging.getLogger("nextPartition") logger.setLevel(logging.DEBUG) sutil.setupLoggingHandlers(logger, config) sutil.echoConfig(logger, config) try: config["endDate"] = config.startDate + dt.timedelta(config.weeksIntoFuture * 7) print schema.createPartitions(config, logger)