def testAcceptConfigFile(self):
# Test failure with good config file, unknown option in that file
try:
copt = [('c', 'config', True, self.configTstPath, "the test config file")]
CM.newConfiguration(configurationOptionsList=copt,optionNameForConfigFile = 'config', configurationFileRequired = True)
assert(False)
except CM.NotAnOptionError, e:
assert(True)
def setup_module():
global me
if me:
return
# else initialize
# print "MODULE setup"
me = Me()
me.markingTemplate = "MARK %s: %s"
me.startMark = 'start'
me.endMark = 'end'
me.testDB = TestDB()
me.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing Monitor')
myDir = os.path.split(__file__)[0]
if not myDir: myDir = '.'
replDict = {'testDir':'%s'%myDir}
for i in me.config:
try:
me.config[i] = me.config.get(i)%(replDict)
except:
pass
knownTests = [x for x in dir(TestMonitor) if x.startswith('test')]
me.logWasExtracted = {}
for t in knownTests:
me.logWasExtracted[t] = False
me.logger = monitor.logger
me.logger.setLevel(logging.DEBUG)
me.logFilePathname = me.config.logFilePathname
logfileDir = os.path.split(me.config.logFilePathname)[0]
try:
os.makedirs(logfileDir)
except OSError,x:
if errno.EEXIST != x.errno: raise
def main():
logger = logging.getLogger("topcrashes_summary")
logger.setLevel(logging.WARNING)
stderrLog = logging.StreamHandler()
stderrLog.setLevel(logging.WARNING)
stderrLogFormatter = logging.Formatter('%(asctime)s %(levelname)s - %(message)s')
stderrLog.setFormatter(stderrLogFormatter)
logger.addHandler(stderrLog)
kwargs = {}
for i in sys.argv[1:]:
if i.startswith('-h') or i.startswith('--he'):
help()
sys.exit(0)
j = i
if i.startswith('-'):
j = i.lstrip('-')
if '=' in j:
name,value = (s.strip() for s in j.split('='))
kwargs[name] = value
else:
print >> sys.stderr,"Ignoring unkown argument '%s'"%(i)
sys.argv = sys.argv[:1]
config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Create Database')
config.update(kwargs)
testDB = TestDB()
testDB.removeDB(config,logger)
testDB.createDB(config,logger)
def setup_module():
global me
if me:
return
me = Me()
tutil.nosePrintModule(__file__)
me.testDB = TestDB()
me.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='TestingCachedIdAccess')
myDir = os.path.split(__file__)[0]
if not myDir: myDir = '.'
replDict = {'testDir':'%s'%myDir}
for i in me.config:
try:
me.config[i] = me.config.get(i)%(replDict)
except:
pass
cia.logger.setLevel(logging.DEBUG)
me.logFilePathname = me.config.logFilePathname
logfileDir = os.path.split(me.config.logFilePathname)[0]
try:
os.makedirs(logfileDir)
except OSError,x:
if errno.EEXIST != x.errno: raise
f = open(me.config.logFilePathname,'w')
f.close()
def setup_module():
global me
if me:
return
me = Me()
me.config = cfgManager.newConfiguration(
configurationModule=testConfig, applicationName='Testing ftpscraper')
myDir = os.path.split(__file__)[0]
if not myDir:
myDir = '.'
replDict = {'testDir': '%s' % myDir}
for i in me.config:
try:
me.config[i] = me.config.get(i) % (replDict)
except:
pass
me.logFilePathname = me.config.logFilePathname
if not me.logFilePathname:
me.logFilePathname = 'logs/ftpscraper_test.log'
logFileDir = os.path.split(me.logFilePathname)[0]
try:
os.makedirs(logFileDir)
except OSError, x:
if errno.EEXIST == x.errno:
pass
else:
raise
def main():
logger = logging.getLogger("topcrashes_summary")
logger.setLevel(logging.WARNING)
stderrLog = logging.StreamHandler()
stderrLog.setLevel(logging.WARNING)
stderrLogFormatter = logging.Formatter(
'%(asctime)s %(levelname)s - %(message)s')
stderrLog.setFormatter(stderrLogFormatter)
logger.addHandler(stderrLog)
kwargs = {}
for i in sys.argv[1:]:
if i.startswith('-h') or i.startswith('--he'):
help()
sys.exit(0)
j = i
if i.startswith('-'):
j = i.lstrip('-')
if '=' in j:
name, value = (s.strip() for s in j.split('='))
kwargs[name] = value
else:
print >> sys.stderr, "Ignoring unkown argument '%s'" % (i)
sys.argv = sys.argv[:1]
config = configurationManager.newConfiguration(
configurationModule=testConfig, applicationName='Create Database')
config.update(kwargs)
testDB = TestDB()
testDB.removeDB(config, logger)
testDB.createDB(config, logger)
def setup_module():
global me
if me:
return
me = Me()
tutil.nosePrintModule(__file__)
# config gets messed up by some tests. Use this one during module setup and teardown
me.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing Postgresql Utils')
myDir = os.path.split(__file__)[0]
if not myDir: myDir = '.'
replDict = {'testDir':'%s'%myDir}
for i in me.config:
try:
me.config[i] = me.config.get(i)%(replDict)
except:
pass
me.logFilePathname = me.config.logFilePathname
if not me.logFilePathname:
me.logFilePathname = 'logs/db_test.log'
logFileDir = os.path.split(me.logFilePathname)[0]
try:
os.makedirs(logFileDir)
except OSError,x:
if errno.EEXIST == x.errno: pass
else: raise
def setup_module():
global me
if me:
return
me = Me()
tutil.nosePrintModule(__file__)
me.config = configurationManager.newConfiguration(
configurationModule=testConfig, applicationName="Testing Postgresql Utils"
)
myDir = os.path.split(__file__)[0]
if not myDir:
myDir = "."
replDict = {"testDir": "%s" % myDir}
for i in me.config:
try:
me.config[i] = me.config.get(i) % (replDict)
except:
pass
me.logFilePathname = me.config.logFilePathname
if not me.logFilePathname:
me.logFilePathname = "logs/db_test.log"
logFileDir = os.path.split(me.logFilePathname)[0]
try:
os.makedirs(logFileDir)
except OSError, x:
if errno.EEXIST == x.errno:
pass
else:
raise
def setup_module():
global me
if me:
return
me = Me()
me.config = cfgManager.newConfiguration(configurationModule=testConfig,
applicationName='Testing ftpscraper')
myDir = os.path.split(__file__)[0]
if not myDir:
myDir = '.'
replDict = {'testDir': '%s' % myDir}
for i in me.config:
try:
me.config[i] = me.config.get(i) % (replDict)
except:
pass
me.logFilePathname = me.config.logFilePathname
if not me.logFilePathname:
me.logFilePathname = 'logs/ftpscraper_test.log'
logFileDir = os.path.split(me.logFilePathname)[0]
try:
os.makedirs(logFileDir)
except OSError, x:
if errno.EEXIST == x.errno:
pass
else:
raise
def setupConfig():
try:
return configurationManager.newConfiguration(configurationModule=config, applicationName="Migrate Process Type")
except configurationManager.NotAnOptionError, x:
print >>sys.stderr, x
print >>sys.stderr, "for usage, try --help"
sys.exit(1)
def setupConfig():
try:
return configurationManager.newConfiguration(
configurationModule=config, applicationName="Migrate Process Type")
except configurationManager.NotAnOptionError, x:
print >> sys.stderr, x
print >> sys.stderr, "for usage, try --help"
sys.exit(1)
def setUp(self):
"""Create a configuration context and a database connection. """
self.config = ConfigurationManager.newConfiguration(
configurationModule=commonconfig,
applicationName="PostgreSQL Tests")
self.database = db.Database(self.config)
self.connection = self.database.connection()
def testAcceptUserCommandLineHelp(self):
opts = [('h','help',False,False,'another help')]
args = {}
args['automaticHelp'] = False
args['configurationOptionsList'] = opts
hh = HelpHandler()
args['helpHandler'] = hh.handleHelp
sys.argv.append('--help')
conf = CM.newConfiguration(**args)
assert("--help" in hh.data)
assert("another help" in hh.data)
def testAcceptAutoCommandLineHelp(self):
opts = []
args = {}
args['automaticHelp'] = True
args['configurationOptionsList'] = opts
hh = HelpHandler()
args['helpHandler'] = hh.handleHelp
sys.argv.append('--help')
conf = CM.newConfiguration(**args)
assert("--help" in hh.data)
assert("print this list" in hh.data)
def setUp(self):
global me
# config gets messed up by some tests. Use this one by preference
self.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing Postgresql Utils')
for i in self.config:
try:
self.config[i] = self.config.get(i)%(replDict)
except:
pass
self.connection = psycopg2.connect(me.dsn)
self.testDB = TestDB()
def setUp(self):
"""Create a configuration context and a database connection. """
self.config = ConfigurationManager.newConfiguration(
configurationModule=commonconfig,
applicationName="PostgreSQL Tests")
try:
self.database = db.Database(self.config)
except (AttributeError, KeyError):
raise
self.connection = self.database.connection()
def setUp(self):
"""Create a configuration context and a database connection. """
self.config = ConfigurationManager.newConfiguration(
configurationModule=commonconfig, applicationName="PostgreSQL Tests"
)
try:
self.database = db.Database(self.config)
except (AttributeError, KeyError):
raise
self.connection = self.database.connection()
def testAcceptCommandLine(self):
opts = []
args = {}
opts.append(('c','chickensoup',False,False,'help for the ailing'))
opts.append(('r','rabbit', True, '', 'rabbits are bunnies'))
args['configurationOptionsList'] = opts
sys.argv.append('-c')
sys.argv.append('--rabbit=bunny')
conf = CM.newConfiguration(**args)
assert('chickensoup' in conf)
assert('rabbit' in conf)
assert('bunny' == conf.rabbit)
def testAcceptEnvironment(self):
opts = []
args = {}
opts.append(('c','chickensoup',False,False,'help for the ailing'))
opts.append(('r','rabbit', True, '', 'rabbits are bunnies'))
args['configurationOptionsList'] = opts
os.environ['chickensoup']=''
os.environ['r'] = 'bunny-rabbit'
conf = CM.newConfiguration(**args)
assert('chickensoup' in conf)
assert('rabbit' in conf)
assert('bunny-rabbit' == conf.rabbit)
def testAcceptTypePriority(self):
'''Assure that commandline beats config file beats environment beats defaults'''
copt = [('c', 'config', True, self.configTstPath, "the test config file"),('r','rabbit', True, 'bambi', 'rabbits are bunnies')]
copt.append(('b','badger',True,'gentle','some badgers are gentle'))
copt.append(('z','zeta', True, 'zebra', 'zebras ooze'))
os.environ['badger'] = 'bloody'
os.environ['zeta'] = 'zymurgy'
sys.argv.append('--rabbit=kangaroo')
conf = CM.newConfiguration(automaticHelp=False,configurationOptionsList=copt,optionNameForConfigFile = 'config', configurationFileRequired = True)
assert('kangaroo' == conf.rabbit) # command line beats config file
assert('this badger=awful' == conf.badger) # config file beats environment
assert('zymurgy' == conf.zeta)
def setUp(self):
global me
# config gets messed up by some tests. Use this one by preference
self.config = configurationManager.newConfiguration(
configurationModule=testConfig,
applicationName='Testing Postgresql Utils')
for i in self.config:
try:
self.config[i] = self.config.get(i) % (replDict)
except:
pass
self.connection = psycopg2.connect(me.dsn)
self.testDB = TestDB()
def setUp(self):
global me
self.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing builds')
myDir = os.path.split(__file__)[0]
if not myDir: myDir = '.'
replDict = {'testDir':'%s'%myDir}
for i in self.config:
try:
self.config[i] = self.config.get(i)%(replDict)
except:
pass
self.logger = TestingLogger(me.fileLogger)
self.testConfig = configurationManager.Config([('t','testPath', True, './TEST-BUILDS', ''),
('f','testFileName', True, 'lastrun.pickle', ''),
])
self.testConfig["persistentDataPathname"] = os.path.join(self.testConfig.testPath, self.testConfig.testFileName)
def createMe():
global me
if not me:
me = Me()
me.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName = "Testing TopCrashers")
myDir = os.path.split(__file__)[0]
if not myDir: myDir = '.'
replDict = {'testDir':'%s'%myDir}
for i in me.config:
try:
me.config[i] = me.config.get(i)%(replDict)
except:
pass
me.logFilePathname = me.config.logFilePathname
logfileDir = os.path.split(me.config.logFilePathname)[0]
try:
os.makedirs(logfileDir)
except OSError,x:
if errno.EEXIST != x.errno: raise
def testNewConfiguration(self):
'''
Trick: To call f(**kwargs) with a dictionary d as the single arg, chant f(**d)
'''
opts = []
args = {}
args['automaticHelp'] = False
# Test for empty
conf = CM.newConfiguration(**args)
assert(not conf.allowableOptionDictionary)
# Test for autoHelp
args['automaticHelp'] = True
conf = CM.newConfiguration(**args)
assert(2 == len(conf.allowableOptionDictionary))
# Test for another legal option
opts.append(('c','chickensoup',False,False,'Help for the ailing'))
args['automaticHelp'] = True
args['configurationOptionsList'] = opts
conf = CM.newConfiguration(**args)
assert(4 == len(conf.allowableOptionDictionary))
# Test a config module
conf = CM.newConfiguration(automaticHelp=False,configurationModule=optionfile)
cd = conf.allowableOptionDictionary
assert(5 == len(cd))
assert ['T', 'testSingleCharacter', True, None] == cd.get('T')[:-1],'but got %s' % (str(cd.get('T')[:-1]))
assert 'testSingleCharacter imported from' in cd.get('T')[-1],'but got %s' % (str(cd.get('T')[-1]))
assert 'optionfile' in cd.get('T')[-1],'but got %s' % (str(cd.get('T')[-1]))
#assert ['T', 'testSingleCharacter', True, None, 'testSingleCharacter imported from optionfile'] == cd.get('T'),'but got %s' % (str(cd.get('T')))
assert [None, 'testDefault', True, 'default'] == cd.get('testDefault')[:-1], "but got %s" %(str(cd.get('testDefault')[:-1]))
assert 'testDefault imported from' in cd.get('testDefault')[-1], "but got %s" %(cd.get('testDefault')[-1])
assert 'optionfile' in cd.get('testDefault')[-1], "but got %s" %(cd.get('testDefault')[-1])
#assert([None, 'testDefault', True, 'default', 'testDefault imported from optionfile'] == cd.get('testDefault'))
assert([None, 'testDoc', True, None, 'test doc'] == cd.get('testDoc'))
assert [None, 'testNil', True, None] == cd.get('testNil')[:-1], "but got %s" %(str(cd.get('testNil')[:-1]))
assert 'testNil imported from' in cd.get('testNil')[-1], "but got %s" %(cd.get('testNil')[-1])
assert 'optionfile' in cd.get('testNil')[-1], "but got %s" %(cd.get('testNil')[-1])
#assert([None, 'testNil', True, None, 'testNil imported from optionfile'] == cd.get('testNil'))
# Test failure with good option, bad file
try:
copt = [('c', 'config', True, './badone', "the badconfig file")]
CM.newConfiguration(configurationOptionsList=copt,optionNameForConfigFile = 'config', configurationFileRequired = True)
assert(False)
except CM.ConfigFileMissingError, e:
assert(True)
def setUp(self):
configContext = configurationManager.newConfiguration(configurationModule=config, applicationName="Top Crash By URL Summary")
logger = logging.getLogger("tcbyurl_summary")
logger.setLevel(logging.DEBUG)
stderrLog = logging.StreamHandler()
stderrLog.setLevel(configContext.stderrErrorLoggingLevel)
stderrLogFormatter = logging.Formatter(configContext.stderrLineFormatString)
stderrLog.setFormatter(stderrLogFormatter)
logger.addHandler(stderrLog)
rotatingFileLog = logging.handlers.RotatingFileHandler(configContext.logFilePathname, "a", configContext.logFileMaximumSize, configContext.logFileMaximumBackupHistory)
rotatingFileLog.setLevel(logging.DEBUG)
rotatingFileLogFormatter = logging.Formatter(configContext.logFileLineFormatString)
rotatingFileLog.setFormatter(rotatingFileLogFormatter)
logger.addHandler(rotatingFileLog)
logger.info("current configuration\n%s", str(configContext))
self.configContext = configContext
self.logger = logger
self.rotatingFileLog = rotatingFileLog
def createMe():
global me
if not me:
me = Me()
me.config = configurationManager.newConfiguration(
configurationModule=testConfig, applicationName="Testing TopCrashers")
myDir = os.path.split(__file__)[0]
if not myDir: myDir = '.'
replDict = {'testDir': '%s' % myDir}
for i in me.config:
try:
me.config[i] = me.config.get(i) % (replDict)
except:
pass
me.logFilePathname = me.config.logFilePathname
logfileDir = os.path.split(me.config.logFilePathname)[0]
try:
os.makedirs(logfileDir)
except OSError, x:
if errno.EEXIST != x.errno: raise
def main():
addedConfigOptions = [
('h','help',False,None,'print this list',None),
('P','product-os-count',True,12,'how many product/os pairs to use [1-56]',lambda x: rangeChecker(1,56,x) ),
('S','signature-count',True,83,'how many signatures to use [1-121]',lambda x: rangeChecker(1,121,x)),
('R','repeat-count',True,2,'how many times to loop [smallish N]',None),
('m','mtbf-fill',False,True,'Fill the time_before_failure table',lambda x: True),
('u','url-fill',False,True,'Fill the top_crashes_by_url table',lambda x: True),
('s','sig-fill',False,True,'Fill the top_crashes_by_signature table',lambda x: True),
('a','all-fill',False,True,'Fill all three matrialized view tables',[('mtbf-fill',None),('url-fill',None),('sig-fill',None)]),
('D','drop-all',False,True,'Drop all the database tables. Do no other work',lambda x: True),
(None,'logFileErrorLoggingLevel',True,logging.WARNING,'logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)',None),
]
config = configurationManager.newConfiguration(configurationModule = commonConfig, applicationName='fillSchema.py',configurationOptionsList = addedConfigOptions)
myDir = os.path.split(__file__)[0]
logDir = os.path.join(myDir,'logs')
logFile = os.path.join(logDir,'fillDB.log')
try:
os.makedirs(logDir)
except OSError:
pass
logger = logging.getLogger("fillDB")
fileLog = logging.FileHandler(logFile)
fileLog.setLevel(int(config.logFileErrorLoggingLevel))
fileLogFormatter = logging.Formatter('%(asctime)s %(levelname)s - %(message)s')
fileLog.setFormatter(fileLogFormatter)
# now add the handler for all the logs of interest
logger.addHandler(fileLog)
logger.setLevel(int(config.logFileErrorLoggingLevel))
socorro_cia.logger.addHandler(fileLog)
socorro_cia.logger.setLevel(int(config.logFileErrorLoggingLevel))
topcrasher.logger.addHandler(fileLog)
topcrasher.logger.setLevel(int(config.logFileErrorLoggingLevel))
topcrashbyurl.logger.addHandler(fileLog)
topcrashbyurl.logger.setLevel(int(config.logFileErrorLoggingLevel))
logger.info("Config is\n%s",str(config))
createData(config,logger)
def setUp(self):
global me
self.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing MTBF')
myDir = os.path.split(__file__)[0]
if not myDir: myDir = '.'
replDict = {'testDir':'%s'%myDir}
for i in self.config:
try:
self.config[i] = self.config.get(i)%(replDict)
except:
pass
self.logger = TestingLogger(me.fileLogger)
self.connection = psycopg2.connect(me.dsn)
cursor = self.connection.cursor()
self.testDB = TestDB()
self.testDB.removeDB(self.config,self.logger)
self.testDB.createDB(self.config,self.logger)
self.prods = ['zorro','vogel','lizz',]
self.oss = ['OSX','LOX','WOX',]
self.productDimData = [] # filled in by fillMtbfTables
def setup_module():
global me
tutil.nosePrintModule(__file__)
if me:
return
me = Me()
me.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing TCByUrl')
tcbu.logger.setLevel(logging.DEBUG)
myDir = os.path.split(__file__)[0]
if not myDir: myDir = '.'
replDict = {'testDir':'%s'%myDir}
for i in me.config:
try:
me.config[i] = me.config.get(i)%(replDict)
except:
pass
me.logFilePathname = me.config.logFilePathname
logFileDir = os.path.split(me.logFilePathname)[0]
try:
os.makedirs(logFileDir)
except OSError,x:
if errno.EEXIST == x.errno: pass
else: raise
def setup_module():
global me
if me:
return
me = Me()
me.config = configurationManager.newConfiguration(
configurationModule=testConfig, applicationName='Testing dbtestutil')
myDir = os.path.split(__file__)[0]
if not myDir: myDir = '.'
replDict = {'testDir': '%s' % myDir}
for i in me.config:
try:
me.config[i] = me.config.get(i) % (replDict)
except:
pass
logger.setLevel(logging.DEBUG)
logFilePathname = me.config.logFilePathname
logfileDir = os.path.split(me.config.logFilePathname)[0]
try:
os.makedirs(logfileDir)
except OSError, x:
if errno.EEXIST != x.errno: raise
def setUp(self):
global me
self.config = cfgManager.newConfiguration(
configurationModule=testConfig,
applicationName='Testing ftpscraper')
myDir = os.path.split(__file__)[0]
if not myDir:
myDir = '.'
replDict = {'testDir': '%s' % myDir}
for i in self.config:
try:
self.config[i] = self.config.get(i) % (replDict)
except:
pass
self.logger = TestingLogger(me.fileLogger)
self.testConfig = cfgManager.Config([
('t', 'testPath', True, './TEST-BUILDS', ''),
('f', 'testFileName', True, 'lastrun.pickle', '')
])
self.testConfig["persistentDataPathname"] = os.path.join(
self.testConfig.testPath, self.testConfig.testFileName)
import sys
import logging
import logging.handlers
try:
import config.duplicatesconfig as configModule
except ImportError:
import duplicatesconfig as configModule
import socorro.lib.ConfigurationManager as configurationManager
import socorro.cron.duplicates as duplicates
import socorro.lib.util as sutil
try:
config = configurationManager.newConfiguration(configurationModule=configModule, applicationName="Duplicate Detector 0.1")
except configurationManager.NotAnOptionError, x:
print >>sys.stderr, x
print >>sys.stderr, "for usage, try --help"
sys.exit()
logger = logging.getLogger("duplicates")
logger.setLevel(logging.DEBUG)
sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)
try:
duplicates.find_duplicates(config)
finally:
logger.info("done.")
import datetime
import json
import os
import tarfile
import time
import sys
import config.commonconfig as configModule
import socorro.database.database as db
import socorro.lib.ConfigurationManager as configurationManager
import socorro.lib.datetimeutil as dtu
import socorro.lib.httpclient as httpc
config = configurationManager.newConfiguration(
configurationModule=configModule,
applicationName='movecrashes.py'
)
def export_uuids(path, numberofdays):
"""Export crash report uuids from a PostgreSQL database to a CSV file
path - Directory where the csv file will be created.
numberofdays - Number of days of crash reports to retrieve, before the most
recent crash date.
"""
database = db.Database(config)
connection = database.connection()
cur = connection.cursor()
import sys
import logging
import logging.handlers
from datetime import date, timedelta
try:
import config.dailyMatviewsConfig as configModule
except ImportError:
import dailyMatviewsConfig as configModule
import socorro.lib.ConfigurationManager as configurationManager
import socorro.cron.dailyMatviews as dailyMatviews
import socorro.lib.util as sutil
try:
config = configurationManager.newConfiguration(
configurationModule=configModule, applicationName="dailyMatviews 0.1")
except configurationManager.NotAnOptionError, x:
print >>sys.stderr, x
print >>sys.stderr, "for usage, try --help"
sys.exit()
logger = logging.getLogger("dailyMatviews")
logger.setLevel(logging.DEBUG)
sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)
exitCode = 255
try:
targetDate = date.today() - timedelta(1)
import web
import datetime as dt
import config.webapiconfig as configModule
from config import revisionsconfig
import socorro.lib.ConfigurationManager as configurationManager
import socorro.lib.datetimeutil as dtutil
import socorro.lib.productVersionCache as pvc
import socorro.webapi.webapiService as webapi
import socorro.lib.util as sutil
import logging
import logging.handlers
config = configurationManager.newConfiguration(configurationModule=configModule, applicationName="Socorro Webapi")
# Adding revisions of Socorro and Breakpad for the server status service.
revisions = configurationManager.newConfiguration(
configurationModule=revisionsconfig,
applicationName="Socorro Revisions"
)
config.update(revisions)
logger = logging.getLogger("webapi")
logger.setLevel(logging.DEBUG)
syslog = logging.handlers.SysLogHandler(facility=config.syslogFacilityString)
syslog.setLevel(config.syslogErrorLoggingLevel)
syslogFormatter = logging.Formatter(config.syslogLineFormatString)
syslog.setFormatter(syslogFormatter)
import sys
import logging
import logging.handlers
try:
import config.monitorconfig as configModule
except ImportError:
import monitorconfig as configModule
import socorro.monitor.monitor as monitor
import socorro.lib.ConfigurationManager as configurationManager
import socorro.lib.util as sutil
try:
config = configurationManager.newConfiguration(configurationModule=configModule, applicationName="Socorro Monitor 2.0")
except configurationManager.NotAnOptionError, x:
print >>sys.stderr, x
print >>sys.stderr, "for usage, try --help"
sys.exit()
logger = logging.getLogger("monitor")
logger.setLevel(logging.DEBUG)
sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)
try:
while True:
m = monitor.Monitor(config)
m.start()
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import web
import socorro.webapi.class_partial as cpart
import socorro.lib.ConfigurationManager as cm
import socorro.collector.wsgicollector as wscol
import socorro.lib.util as sutil
#import socorro.webapi.hello as hello
import config.collectorconfig as collectorConfig
#-------------------------------------------------------------------------------
config = \
cm.newConfiguration(configurationModule=collectorConfig,
applicationName="Socorro Collector 3.0")
#-------------------------------------------------------------------------------
import logging
import logging.handlers
logger = logging.getLogger("collector")
logger.setLevel(logging.DEBUG)
syslog = logging.handlers.SysLogHandler(
address=(config.syslogHost, config.syslogPort),
facility=config.syslogFacilityString,
)
syslog.setLevel(config.syslogErrorLoggingLevel)
syslogFormatter = logging.Formatter(config.syslogLineFormatString)
syslog.setFormatter(syslogFormatter)
import sys
import logging
import logging.handlers
try:
import config.dailyurlconfig as configModule
except ImportError:
import dailyurlconfig as configModule
import socorro.lib.ConfigurationManager as configurationManager
import socorro.cron.dailyUrl as url
import socorro.lib.util as sutil
try:
config = configurationManager.newConfiguration(
configurationModule=configModule, applicationName="Daily URL Dump 0.1")
except configurationManager.NotAnOptionError, x:
print >> sys.stderr, x
print >> sys.stderr, "for usage, try --help"
sys.exit()
logger = logging.getLogger("dailyUrlDump")
logger.setLevel(logging.DEBUG)
sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)
try:
url.dailyUrlDump(config)
finally:
logger.info("done.")
import logging
import logging.handlers
import sys
import time
try:
import config.topCrashesByUrlConfig as configModule
except ImportError:
import topCrashesByUrlConfig as configModule
import socorro.lib.ConfigurationManager as configurationManager
import socorro.cron.topCrashesByUrl as tcbyurl
import socorro.lib.util as sutil
config = configurationManager.newConfiguration(
configurationModule=configModule,
applicationName="Top Crash By URL Summary")
logger = logging.getLogger("topCrashesByUrl")
logger.setLevel(logging.DEBUG)
sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)
try:
before = time.time()
tu = tcbyurl.TopCrashesByUrl(config)
tu.processDateInterval()
logger.info("Successfully ran in %d seconds" % (time.time() - before))
finally:
logger.info("done.")
def setUp(self):
"""Create a configuration context."""
self.config = ConfigurationManager.newConfiguration(
configurationModule=commonconfig, applicationName="RabbitMQ Tests")
import web
import datetime as dt
import config.webapiconfig as configModule
from config import revisionsconfig
import socorro.lib.ConfigurationManager as configurationManager
import socorro.lib.datetimeutil as dtutil
import socorro.lib.productVersionCache as pvc
import socorro.webapi.webapiService as webapi
import socorro.lib.util as sutil
import logging
import logging.handlers
config = configurationManager.newConfiguration(
configurationModule=configModule, applicationName="Socorro Webapi")
# Adding revisions of Socorro and Breakpad for the server status service.
revisions = configurationManager.newConfiguration(
configurationModule=revisionsconfig, applicationName="Socorro Revisions")
config.update(revisions)
logger = logging.getLogger("webapi")
logger.setLevel(logging.DEBUG)
syslog = logging.handlers.SysLogHandler(facility=config.syslogFacilityString)
syslog.setLevel(config.syslogErrorLoggingLevel)
syslogFormatter = logging.Formatter(config.syslogLineFormatString)
syslog.setFormatter(syslogFormatter)
logger.addHandler(syslog)
import logging
import logging.handlers
import sys
try:
import config.fixbrokendumpsconfig as configModule
except ImportError:
import fixbrokendumpsconfig as configModule
import socorro.lib.ConfigurationManager as configurationManager
import socorro.cron.fixBrokenDumps as fixBrokenDumps
import socorro.lib.util as sutil
try:
config = configurationManager.newConfiguration(
configurationModule=configModule, applicationName="Fix Broken Dumps")
except configurationManager.NotAnOptionError, x:
print >> sys.stderr, x
print >> sys.stderr, "for usage, try --help"
sys.exit(1)
logger = logging.getLogger("fix_broken_dumps")
logger.setLevel(logging.DEBUG)
sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)
try:
last_date_processed = fixBrokenDumps.fix(config, logger,
config.brokenFirefoxLinuxQuery,
config.brokenFirefoxLinuxFixer)
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import sys
import logging
import logging.handlers
import config.crashmoverconfig as cmconf
import socorro.lib.ConfigurationManager as configurationManager
import socorro.storage.storageMover as smover
import socorro.lib.util as sutil
try:
config = configurationManager.newConfiguration(
configurationModule=cmconf, applicationName="New Crash Mover")
except configurationManager.NotAnOptionError, x:
print >> sys.stderr, x
print >> sys.stderr, "for usage, try --help"
sys.exit()
logger = logging.getLogger("newCrashMover")
logger.setLevel(logging.DEBUG)
sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)
config.logger = logger
try:
smover.move(config)
import sys
import logging
import logging.handlers
try:
import config.bugzillaconfig as configModule
except ImportError:
import bugzillaconfig as configModule
import socorro.lib.ConfigurationManager as configurationManager
import socorro.cron.bugzilla as bug
import socorro.lib.util as sutil
try:
config = configurationManager.newConfiguration(
configurationModule=configModule,
applicationName="Bugzilla Associations 0.1")
except configurationManager.NotAnOptionError, x:
print >> sys.stderr, x
print >> sys.stderr, "for usage, try --help"
sys.exit()
logger = logging.getLogger("bugzilla")
logger.setLevel(logging.DEBUG)
sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)
try:
bug.record_associations(config)
finally:
import logging
import logging.handlers
import sys
try:
import config.serverstatusconfig as configModule
except ImportError:
import serverstatusconfig as configModule
import socorro.lib.ConfigurationManager as configurationManager
import socorro.cron.serverstatus as serverstatus
import socorro.lib.util as sutil
try:
config = configurationManager.newConfiguration(
configurationModule=configModule,
applicationName="Server Status Summary")
except configurationManager.NotAnOptionError, x:
print >> sys.stderr, x
print >> sys.stderr, "for usage, try --help"
sys.exit(1)
logger = logging.getLogger("server_status_summary")
logger.setLevel(logging.DEBUG)
sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)
try:
serverstatus.update(config, logger)
finally:
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import sys
try:
import config.processorconfig as configModule
except ImportError:
import processorconfig as configModule
import socorro.lib.ConfigurationManager as configurationManager
try:
config = configurationManager.newConfiguration(
configurationModule=configModule,
applicationName="Socorro Processor 2.9")
except configurationManager.NotAnOptionError, x:
print >> sys.stderr, x
print >> sys.stderr, "for usage, try --help"
sys.exit()
def processor2008(config):
import sys
import logging
import logging.handlers
import socorro.lib.util as sutil
import socorro.processor.externalProcessor as processor
"""
import logging
import logging.handlers
import datetime
try:
import config.ftpscraperconfig as configModule
except ImportError:
import ftpscraperconfig as configModule
import socorro.cron.ftpscraper as ftpscraper
import socorro.lib.ConfigurationManager as cfgManager
import socorro.lib.util as sutil
config = cfgManager.newConfiguration(configurationModule=configModule,
applicationName='startFtpScraper.py')
assert "databaseHost" in config, "databaseHost missing from config"
assert "databaseName" in config, "databaseName missing from config"
assert "databaseUserName" in config, "databaseUserName missing from config"
assert "databasePassword" in config, "databasePassword missing from config"
assert "base_url" in config, "base_url missing from config"
assert "products" in config, "products missing from config"
assert "backfillDate" in config, "backfillDate missing from config"
logger = logging.getLogger("ftpscraper")
logger.setLevel(logging.DEBUG)
sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)
config.logger = logger
import sys
import logging
import logging.handlers
try:
import config.signaturesconfig as configModule
except ImportError:
import signaturesconfig as configModule
import socorro.lib.ConfigurationManager as configurationManager
import socorro.cron.signatures as signatures
import socorro.lib.util as sutil
try:
config = configurationManager.newConfiguration(configurationModule=configModule, applicationName="Signatures 0.1")
except configurationManager.NotAnOptionError, x:
print >>sys.stderr, x
print >>sys.stderr, "for usage, try --help"
sys.exit()
logger = logging.getLogger("signatures")
logger.setLevel(logging.DEBUG)
sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)
try:
signatures.update_signatures(config)
finally:
logger.info("done.")
import sys
import logging
import logging.handlers
from datetime import date, timedelta
try:
import config.dailyMatviewsConfig as configModule
except ImportError:
import dailyMatviewsConfig as configModule
import socorro.lib.ConfigurationManager as configurationManager
import socorro.cron.dailyMatviews as dailyMatviews
import socorro.lib.util as sutil
try:
config = configurationManager.newConfiguration(
configurationModule=configModule, applicationName="dailyMatviews 0.1")
except configurationManager.NotAnOptionError, x:
print >> sys.stderr, x
print >> sys.stderr, "for usage, try --help"
sys.exit()
logger = logging.getLogger("dailyMatviews")
logger.setLevel(logging.DEBUG)
sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)
exitCode = 255
try:
targetDate = date.today() - timedelta(1)
import web
import socorro.webapi.classPartial as cpart
import socorro.lib.ConfigurationManager as cm
import socorro.collector.wsgicollector as wscol
import socorro.lib.util as sutil
#import socorro.webapi.hello as hello
import config.collectorconfig as collectorConfig
#-------------------------------------------------------------------------------
config = \
cm.newConfiguration(configurationModule=collectorConfig,
applicationName="Socorro Collector 3.0")
#-------------------------------------------------------------------------------
import logging
import logging.handlers
logger = logging.getLogger("collector")
logger.setLevel(logging.DEBUG)
syslog = logging.handlers.SysLogHandler(
address=(config.syslogHost, config.syslogPort),
facility=config.syslogFacilityString,
)
syslog.setLevel(config.syslogErrorLoggingLevel)
syslogFormatter = logging.Formatter(config.syslogLineFormatString)
syslog.setFormatter(syslogFormatter)
logger.addHandler(syslog)
sutil.echoConfig(logger, config)
import sys
import logging
import logging.handlers
try:
import config.duplicatesconfig as configModule
except ImportError:
import duplicatesconfig as configModule
import socorro.lib.ConfigurationManager as configurationManager
import socorro.cron.duplicates as duplicates
import socorro.lib.util as sutil
try:
config = configurationManager.newConfiguration(
configurationModule=configModule,
applicationName="Duplicate Detector 0.1")
except configurationManager.NotAnOptionError, x:
print >> sys.stderr, x
print >> sys.stderr, "for usage, try --help"
sys.exit()
logger = logging.getLogger("duplicates")
logger.setLevel(logging.DEBUG)
sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)
try:
duplicates.find_duplicates(config)
finally:
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import sys
import logging
import logging.handlers
import config.orphansubmitterconf as cmconf
import socorro.lib.ConfigurationManager as configurationManager
import socorro.storage.orphans as smover
import socorro.lib.util as sutil
try:
config = configurationManager.newConfiguration(configurationModule=cmconf,
applicationName="Orphan Submitter")
except configurationManager.NotAnOptionError, x:
print >>sys.stderr, x
print >>sys.stderr, "for usage, try --help"
sys.exit()
logger = logging.getLogger("orphanSubmitter")
logger.setLevel(logging.DEBUG)
sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)
config.logger = logger
try:
smover.move(config)
import sys
import logging
import logging.handlers
import config.submitterconfig as subconf
import socorro.lib.ConfigurationManager as configurationManager
import socorro.collector.submitter as sub
import socorro.lib.util as sutil
import poster
poster.streaminghttp.register_openers()
try:
config = configurationManager.newConfiguration(
configurationModule=subconf, applicationName="submitter 2.0")
except configurationManager.NotAnOptionError, x:
print >> sys.stderr, x
print >> sys.stderr, "for usage, try --help"
sys.exit()
logger = logging.getLogger("submitter")
logger.setLevel(logging.DEBUG)
sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)
config.logger = logger
if config.numberOfSubmissions == 'forever':
config.iteratorFunc = sub.createInfiniteFileSystemIterator
import logging
import logging.handlers
import sys
try:
import config.fixbrokendumpsconfig as configModule
except ImportError:
import fixbrokendumpsconfig as configModule
import socorro.lib.ConfigurationManager as configurationManager
import socorro.cron.fixBrokenDumps as fixBrokenDumps
import socorro.lib.util as sutil
try:
config = configurationManager.newConfiguration(configurationModule=configModule, applicationName="Fix Broken Dumps")
except configurationManager.NotAnOptionError, x:
print >>sys.stderr, x
print >>sys.stderr, "for usage, try --help"
sys.exit(1)
logger = logging.getLogger("fix_broken_dumps")
logger.setLevel(logging.DEBUG)
sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)
try:
last_date_processed = fixBrokenDumps.fix(config, logger, config.brokenFirefoxLinuxQuery, config.brokenFirefoxLinuxFixer)
last_date_processed = fixBrokenDumps.fix(config, logger, config.brokenFennecQuery, config.brokenFennecFixer)
import sys
import logging
import logging.handlers
try:
import config.updateadus as configModule
except ImportError:
import updateadus as configModule
import socorro.lib.ConfigurationManager as configurationManager
import socorro.cron.updateADUs as updateADUs
import socorro.lib.util as sutil
try:
config = configurationManager.newConfiguration(
configurationModule=configModule, applicationName="Update ADUs 0.1")
except configurationManager.NotAnOptionError, x:
print >> sys.stderr, x
print >> sys.stderr, "for usage, try --help"
sys.exit()
logger = logging.getLogger("updateADUs")
logger.setLevel(logging.DEBUG)
sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)
try:
updateADUs.update_adus(config)
finally:
logger.info("done.")
# XXX Set to be deprecated in favor of socorro/external/postgresql/models.py
import socorro.database.database as db
import psycopg2
import psycopg2.extensions
import logging
import threading
from socorro.unittest.testlib.loggerForTest import TestingLogger
from createDBforTest import *
import socorro.lib.util as util
from socorro.unittest.testbase import TestCase
import socorro.lib.ConfigurationManager as cm
import dbTestconfig as testConfig
config = cm.newConfiguration(configurationModule=testConfig,
applicationName='Testing Psycopghelper')
"""
Assume that psycopg2 works, then all we need to do is assure ourselves
that our simplistic wrap around a returned array is correct
"""
class TestMultiCursor(psycopg2.extensions.cursor):
def __init__(self, numCols=4, numRows=2, **kwargs):
self.result = []
for i in range(numRows):
aRow = []
for j in range(numCols):
aRow.append('Row %d, Column %d' % (i, j))
self.result.append(aRow)
self.next = self.__next()
import sys
import logging
import logging.handlers
try:
import config.bugzillaconfig as configModule
except ImportError:
import bugzillaconfig as configModule
import socorro.lib.ConfigurationManager as configurationManager
import socorro.cron.bugzilla as bug
import socorro.lib.util as sutil
try:
config = configurationManager.newConfiguration(configurationModule=configModule, applicationName="Bugzilla Associations 0.1")
except configurationManager.NotAnOptionError, x:
print >>sys.stderr, x
print >>sys.stderr, "for usage, try --help"
sys.exit()
logger = logging.getLogger("bugzilla")
logger.setLevel(logging.DEBUG)
sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)
try:
bug.record_associations(config)
finally:
logger.info("done.")
import unittest
import socorro.database.database as db
import psycopg2
import psycopg2.extensions
import logging
import threading
from socorro.unittest.testlib.loggerForTest import TestingLogger
import socorro.unittest.testlib.util as tutil
from createDBforTest import *
import socorro.lib.util as util
import socorro.lib.ConfigurationManager as cm
import dbTestconfig as testConfig
config = cm.newConfiguration(configurationModule = testConfig, applicationName='Testing Psycopghelper')
def setup_module():
tutil.nosePrintModule(__file__)
"""
Assume that psycopg2 works, then all we need to do is assure ourselves
that our simplistic wrap around a returned array is correct
"""
class TestMultiCursor(psycopg2.extensions.cursor):
def __init__(self,numCols = 4, numRows=2, **kwargs):
self.result = []
for i in range(numRows):
aRow = []
for j in range(numCols):
aRow.append('Row %d, Column %d' %(i,j))
import logging
import logging.handlers
import sys
import datetime as dt
try:
import config.createpartitionsconfig as configModule
except ImportError:
import createpartitionsconfig as configModule
import socorro.lib.ConfigurationManager as configurationManager
import socorro.database.schema as schema
import socorro.lib.util as sutil
try:
config = configurationManager.newConfiguration(configurationModule=configModule, applicationName="startNextPartition")
except configurationManager.NotAnOptionError, x:
print >>sys.stderr, x
print >>sys.stderr, "for usage, try --help"
sys.exit(1)
logger = logging.getLogger("nextPartition")
logger.setLevel(logging.DEBUG)
sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)
try:
config["endDate"] = config.startDate + dt.timedelta(config.weeksIntoFuture * 7)
print
schema.createPartitions(config, logger)
