Exemple #1
0
  def __init__(self,*args,**kwargs):
    """
    Passes appropriate kwargs to Config, pays local attention to these keys:
    updateInterval: default: '0' format: 'dd:hh:mm:ss', leading parts optional. Must be >= 0 seconds.
    updateFunction: default: noop(). Takes self as argument. Behavior: Updates default values in argument
    reEvaluateFunction: default: noop(). Takes self as argument. Behavior: Mutates values in argument
    signalNumber: default: SIGALRM (14). If 0, then signals will not be handled.
      Instances that share the same signalNumber will all be update()-ed at every signal.

    self.internal.updateFunction may be set after construction if desired: Avoids double-work at construction
    self.internal.reEvalutateFunction may be set after construction if desired, but this is not recommended.
    """
    skwargs = dict([(x,kwargs[x]) for x in socorro_config.getDefaultedConfigOptions().keys() if x in kwargs])
    for i in range(len(args)):
      skwargs[socorro_config.namedConfigOptions[i]] = args[i]
    super(DynamicConfig,self).__init__(**skwargs)
    self.internal.updateFunction = kwargs.get('updateFunction',noop)
    self.internal.reEvaluateFunction = kwargs.get('reEvaluateFunction',noop)
    self.internal.signalNumber = kwargs.get('signalNumber',14)
    self.internal.nextUpdate = None
    updateInterval = kwargs.get('updateInterval','0:0:0:0')
    self.internal.updateDelta = socorro_config.timeDeltaConverter(updateInterval)
    if self.internal.updateDelta:
      if self.internal.updateDelta < datetime.timedelta(0):
        raise ValueError("updateInterval must be non-negative, but %s"%self.internal.updateDelta)
      self.internal.nextUpdate = utc_now() + self.internal.updateDelta

    # finally: make sure we are current
    if self.internal.signalNumber:
      priorSignal = signal.signal(self.internal.signalNumber,DynamicConfig.handleAlarm)
    self.doUpdate()
    DynamicConfig.instances[id(self)] = self
 def testAcceptConfigFile(self):
   # Test failure with good config file, unknown option in that file
   try:
     copt = [('c',  'config', True, self.configTstPath, "the test config file")]
     CM.newConfiguration(configurationOptionsList=copt,optionNameForConfigFile = 'config', configurationFileRequired = True)
     assert(False)
   except CM.NotAnOptionError, e:
     assert(True)
Exemple #3
0
def getProcessingWindow(configContext,tableName, productVersionRestriction,cursor,logger, **kwargs):
  """
  ProcessingWindow is a single time window over which to aggregate materialized view data.

  Returns (startWindow,deltaWindow,endWindow) using this heuristic:
  kwargs beats configContext which beats latest table row
  if two among startWindow, endWindow, deltaWindow in config or kwargs: they are used.
    if all three: assert startWindow + deltaWindow == endWindow
  Backward compatibility: if processingDay is present and windowXxx are not:
    startWindow = midnight of given day, deltaWindow = timedelta(days=1)
  else: try to read window_end and window_size from the given table
  if one is available from config/kwargs it beats the same (or calculated) one from the table
  On inconsistency or failure, logs the problem and aborts
  BEWARE: You can get inconsitency by having one item in config and the other two in kwargs: BEWARE
  """
  config = {}
  config.update(configContext)
  config.update(kwargs)
  startWindow = config.get('startWindow')
  if type(startWindow) is str:
    startWindow = cm.dateTimeConverter(startWindow)
  deltaWindow = config.get('deltaWindow')
  if type(deltaWindow) is str:
    deltaWindow = cm.timeDeltaConverter(deltaWindow)
  endWindow = config.get('endWindow')
  if type(endWindow) is str:
    endWindow = cm.dateTimeConverter(endWindow)
  processingDay = config.get('processingDay')
  if type(processingDay) is str:
    processingDay = cm.dateTimeConverter(processingDay)
  try:
    if startWindow or deltaWindow or endWindow:
      if startWindow and endWindow and deltaWindow:
        assert startWindow + deltaWindow == endWindow,"inconsistent: %s + %s != %s"%(startWindow,deltaWindow,endWindow)
      elif startWindow and endWindow:
        deltaWindow = endWindow - startWindow
      elif startWindow and deltaWindow:
        endWindow = startWindow + deltaWindow
      elif deltaWindow and endWindow:
        startWindow = endWindow - deltaWindow
      else:
        assert not (startWindow or deltaWindow or endWindow), "insufficient: Need two of window ...Start: %s, ...Delta: %s, ...End:%s"%(startWindow,deltaWindow,endWindow)
    elif processingDay:
      dayt = datetime.datetime.fromtimestamp(time.mktime(processingDay.timetuple()))
      startWindow = dayt.replace(hour=0,minute=0,second=0,microsecond=0)
      assert startWindow == dayt,'processingDay must be some midnight, but was %s'%dayt
      deltaWindow = datetime.timedelta(days=1)
      endWindow = startWindow + deltaWindow
    else: # no params: try table
      startWindow,deltaWindow = getLastWindowAndSizeFromTable(cursor,tableName, productVersionRestriction,logger)
      if startWindow:
        endWindow = startWindow+deltaWindow
    return (startWindow,deltaWindow,endWindow)
  except:
    lib_util.reportExceptionAndAbort(logger)
def setup_module():
  global me
  if me:
    return
  me = Me()
  tutil.nosePrintModule(__file__)
  # config gets messed up by some tests. Use this one during module setup and teardown
  me.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing Postgresql Utils')
  myDir = os.path.split(__file__)[0]
  if not myDir: myDir = '.'
  replDict = {'testDir':'%s'%myDir}
  for i in me.config:
    try:
      me.config[i] = me.config.get(i)%(replDict)
    except:
      pass
  me.logFilePathname = me.config.logFilePathname
  if not me.logFilePathname:
    me.logFilePathname = 'logs/db_test.log'
  logFileDir = os.path.split(me.logFilePathname)[0]
  try:
    os.makedirs(logFileDir)
  except OSError,x:
    if errno.EEXIST == x.errno: pass
    else: raise
def setup_module():
  global me
  if me:
    return
  # else initialize
  # print "MODULE setup"
  me = Me()
  me.markingTemplate = "MARK %s: %s"
  me.startMark = 'start'
  me.endMark = 'end'
  me.testDB = TestDB()
  me.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing Monitor')
  myDir = os.path.split(__file__)[0]
  if not myDir: myDir = '.'
  replDict = {'testDir':'%s'%myDir}
  for i in me.config:
    try:
      me.config[i] = me.config.get(i)%(replDict)
    except:
      pass
  knownTests = [x for x in dir(TestMonitor) if x.startswith('test')]
  me.logWasExtracted = {}
  for t in knownTests:
    me.logWasExtracted[t] = False
  me.logger = monitor.logger
  me.logger.setLevel(logging.DEBUG)
  me.logFilePathname = me.config.logFilePathname

  logfileDir = os.path.split(me.config.logFilePathname)[0]
  try:
    os.makedirs(logfileDir)
  except OSError,x:
    if errno.EEXIST != x.errno: raise
def setup_module():
    global me
    if me:
        return
    me = Me()
    me.testDB = TestDB()
    me.config = configurationManager.newConfiguration(
        configurationModule=testConfig,
        applicationName='TestingCachedIdAccess')
    myDir = os.path.split(__file__)[0]
    if not myDir: myDir = '.'
    replDict = {'testDir': '%s' % myDir}
    for i in me.config:
        try:
            me.config[i] = me.config.get(i) % (replDict)
        except:
            pass
    cia.logger.setLevel(logging.DEBUG)
    me.logFilePathname = me.config.logFilePathname
    logfileDir = os.path.split(me.config.logFilePathname)[0]
    try:
        os.makedirs(logfileDir)
    except OSError, x:
        if errno.EEXIST != x.errno: raise
        f = open(me.config.logFilePathname, 'w')
        f.close()
Exemple #7
0
def newConfiguration(**kwargs):
  """ This used as an alternate constructor for class Config so that applications can
      be lax in defining all the required paramters in the right order.
  """
  kw = socorro_config.getDefaultedConfigOptions()
  kw.update(kwargs)
  return DynamicConfig(**kw)
Exemple #8
0
def setup_module():
  global me
  if me:
    return
  me = Me()
  tutil.nosePrintModule(__file__)
  me.testDB = TestDB()
  me.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='TestingCachedIdAccess')
  myDir = os.path.split(__file__)[0]
  if not myDir: myDir = '.'
  replDict = {'testDir':'%s'%myDir}
  for i in me.config:
    try:
      me.config[i] = me.config.get(i)%(replDict)
    except:
      pass
  cia.logger.setLevel(logging.DEBUG)
  me.logFilePathname = me.config.logFilePathname
  logfileDir = os.path.split(me.config.logFilePathname)[0]
  try:
    os.makedirs(logfileDir)
  except OSError,x:
    if errno.EEXIST != x.errno: raise
    f = open(me.config.logFilePathname,'w')
    f.close()
Exemple #9
0
def setupConfig():
  try:
    return configurationManager.newConfiguration(configurationModule=config, applicationName="Migrate Process Type")
  except configurationManager.NotAnOptionError, x:
    print >>sys.stderr, x
    print >>sys.stderr, "for usage, try --help"
    sys.exit(1)
Exemple #10
0
def main():
    logger = logging.getLogger("topcrashes_summary")
    logger.setLevel(logging.WARNING)

    stderrLog = logging.StreamHandler()
    stderrLog.setLevel(logging.WARNING)
    stderrLogFormatter = logging.Formatter(
        '%(asctime)s %(levelname)s - %(message)s')
    stderrLog.setFormatter(stderrLogFormatter)
    logger.addHandler(stderrLog)

    kwargs = {}
    for i in sys.argv[1:]:
        if i.startswith('-h') or i.startswith('--he'):
            help()
            sys.exit(0)
        j = i
        if i.startswith('-'):
            j = i.lstrip('-')
        if '=' in j:
            name, value = (s.strip() for s in j.split('='))
            kwargs[name] = value
        else:
            print >> sys.stderr, "Ignoring unkown argument '%s'" % (i)
    sys.argv = sys.argv[:1]
    config = configurationManager.newConfiguration(
        configurationModule=testConfig, applicationName='Create Database')
    config.update(kwargs)
    testDB = TestDB()
    testDB.removeDB(config, logger)
    testDB.createDB(config, logger)
Exemple #11
0
def setup_module():
    global me
    if me:
        return
    me = Me()
    me.config = cfgManager.newConfiguration(
        configurationModule=testConfig, applicationName='Testing ftpscraper')
    myDir = os.path.split(__file__)[0]
    if not myDir:
        myDir = '.'
    replDict = {'testDir': '%s' % myDir}
    for i in me.config:
        try:
            me.config[i] = me.config.get(i) % (replDict)
        except:
            pass
    me.logFilePathname = me.config.logFilePathname
    if not me.logFilePathname:
        me.logFilePathname = 'logs/ftpscraper_test.log'
    logFileDir = os.path.split(me.logFilePathname)[0]
    try:
        os.makedirs(logFileDir)
    except OSError, x:
        if errno.EEXIST == x.errno:
            pass
        else:
            raise
Exemple #12
0
def setup_module():
    global me
    if me:
        return
    me = Me()
    tutil.nosePrintModule(__file__)
    me.config = configurationManager.newConfiguration(
        configurationModule=testConfig, applicationName="Testing Postgresql Utils"
    )
    myDir = os.path.split(__file__)[0]
    if not myDir:
        myDir = "."
    replDict = {"testDir": "%s" % myDir}
    for i in me.config:
        try:
            me.config[i] = me.config.get(i) % (replDict)
        except:
            pass
    me.logFilePathname = me.config.logFilePathname
    if not me.logFilePathname:
        me.logFilePathname = "logs/db_test.log"
    logFileDir = os.path.split(me.logFilePathname)[0]
    try:
        os.makedirs(logFileDir)
    except OSError, x:
        if errno.EEXIST == x.errno:
            pass
        else:
            raise
Exemple #13
0
def setup_module():
    global me
    if me:
        return
    me = Me()
    me.config = cfgManager.newConfiguration(configurationModule=testConfig,
          applicationName='Testing ftpscraper')
    myDir = os.path.split(__file__)[0]
    if not myDir:
        myDir = '.'
    replDict = {'testDir': '%s' % myDir}
    for i in me.config:
        try:
            me.config[i] = me.config.get(i) % (replDict)
        except:
            pass
    me.logFilePathname = me.config.logFilePathname
    if not me.logFilePathname:
        me.logFilePathname = 'logs/ftpscraper_test.log'
    logFileDir = os.path.split(me.logFilePathname)[0]
    try:
        os.makedirs(logFileDir)
    except OSError, x:
        if errno.EEXIST == x.errno:
            pass
        else:
            raise
Exemple #14
0
def main():
  logger = logging.getLogger("topcrashes_summary")
  logger.setLevel(logging.WARNING)
  
  stderrLog = logging.StreamHandler()
  stderrLog.setLevel(logging.WARNING)
  stderrLogFormatter = logging.Formatter('%(asctime)s %(levelname)s - %(message)s')
  stderrLog.setFormatter(stderrLogFormatter)
  logger.addHandler(stderrLog)

  kwargs = {}
  for i in sys.argv[1:]:
    if i.startswith('-h') or i.startswith('--he'):
      help()
      sys.exit(0)
    j = i
    if i.startswith('-'):
      j = i.lstrip('-')
    if '=' in j:
      name,value = (s.strip() for s in j.split('='))
      kwargs[name] = value
    else:
      print >> sys.stderr,"Ignoring unkown argument '%s'"%(i)
  sys.argv = sys.argv[:1]
  config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Create Database')
  config.update(kwargs)
  testDB = TestDB()
  testDB.removeDB(config,logger)
  testDB.createDB(config,logger)
Exemple #15
0
def setup_module():
    global me
    if me:
        return
    me = Me()
    # config gets messed up by some tests. Use this one during module setup and teardown
    me.config = configurationManager.newConfiguration(
        configurationModule=testConfig,
        applicationName='Testing Postgresql Utils')
    myDir = os.path.split(__file__)[0]
    if not myDir: myDir = '.'
    replDict = {'testDir': '%s' % myDir}
    for i in me.config:
        try:
            me.config[i] = me.config.get(i) % (replDict)
        except:
            pass
    me.logFilePathname = me.config.logFilePathname
    if not me.logFilePathname:
        me.logFilePathname = 'logs/db_test.log'
    logFileDir = os.path.split(me.logFilePathname)[0]
    try:
        os.makedirs(logFileDir)
    except OSError, x:
        if errno.EEXIST == x.errno: pass
        else: raise
def setupConfig():
    try:
        return configurationManager.newConfiguration(
            configurationModule=config, applicationName="Migrate Process Type")
    except configurationManager.NotAnOptionError, x:
        print >> sys.stderr, x
        print >> sys.stderr, "for usage, try --help"
        sys.exit(1)
Exemple #17
0
    def setUp(self):
        """Create a configuration context and a database connection. """
        self.config = ConfigurationManager.newConfiguration(
            configurationModule=commonconfig,
            applicationName="PostgreSQL Tests")

        self.database = db.Database(self.config)

        self.connection = self.database.connection()
 def fixupContextByProcessingDay(self,context):
   pday = context.get('processingDay')
   if pday:
     logger.info("Adjusting startDate and deltaDate per processingDay %s",pday)
     pday = cm.dateTimeConverter(pday)
     startDate = datetime.datetime.fromtimestamp(time.mktime(pday.timetuple()))
     startDate.replace(hour=0,minute=0,second=0,microsecond=0)
     context['startDate'] = startDate
     context['deltaDate'] = datetime.timedelta(days=1)
     context['startWindw'] = startDate
Exemple #19
0
 def setUp(self):
   global me
   # config gets messed up by some tests. Use this one by preference
   self.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing Postgresql Utils')
   for i in self.config:
     try:
       self.config[i] = self.config.get(i)%(replDict)
     except:
       pass
   self.connection = psycopg2.connect(me.dsn)
   self.testDB = TestDB()
 def testAcceptUserCommandLineHelp(self):
   opts = [('h','help',False,False,'another help')]
   args = {}
   args['automaticHelp'] = False
   args['configurationOptionsList'] = opts
   hh = HelpHandler()
   args['helpHandler'] = hh.handleHelp
   sys.argv.append('--help')
   conf = CM.newConfiguration(**args)
   assert("--help" in hh.data)
   assert("another help" in hh.data)
 def testAcceptAutoCommandLineHelp(self):
   opts = []
   args = {}
   args['automaticHelp'] = True
   args['configurationOptionsList'] = opts
   hh = HelpHandler()
   args['helpHandler'] = hh.handleHelp
   sys.argv.append('--help')
   conf = CM.newConfiguration(**args)
   assert("--help" in hh.data)
   assert("print this list" in hh.data)
 def testAcceptEnvironment(self):
   opts = []
   args = {}
   opts.append(('c','chickensoup',False,False,'help for the ailing'))
   opts.append(('r','rabbit', True, '', 'rabbits are bunnies'))
   args['configurationOptionsList'] = opts
   os.environ['chickensoup']=''
   os.environ['r'] = 'bunny-rabbit'
   conf = CM.newConfiguration(**args)
   assert('chickensoup' in conf)
   assert('rabbit' in conf)
   assert('bunny-rabbit' == conf.rabbit)
 def testAcceptTypePriority(self):
   '''Assure that commandline beats config file beats environment beats defaults'''
   copt = [('c',  'config', True, self.configTstPath, "the test config file"),('r','rabbit', True, 'bambi', 'rabbits are bunnies')]
   copt.append(('b','badger',True,'gentle','some badgers are gentle'))
   copt.append(('z','zeta', True, 'zebra', 'zebras ooze'))
   os.environ['badger'] = 'bloody'
   os.environ['zeta'] = 'zymurgy'
   sys.argv.append('--rabbit=kangaroo')
   conf = CM.newConfiguration(automaticHelp=False,configurationOptionsList=copt,optionNameForConfigFile = 'config', configurationFileRequired = True)
   assert('kangaroo' == conf.rabbit) # command line beats config file
   assert('this badger=awful' == conf.badger) # config file beats environment
   assert('zymurgy' == conf.zeta)
Exemple #24
0
    def setUp(self):
        """Create a configuration context and a database connection. """
        self.config = ConfigurationManager.newConfiguration(
            configurationModule=commonconfig, applicationName="PostgreSQL Tests"
        )

        try:
            self.database = db.Database(self.config)
        except (AttributeError, KeyError):
            raise

        self.connection = self.database.connection()
Exemple #25
0
    def setUp(self):
        """Create a configuration context and a database connection. """
        self.config = ConfigurationManager.newConfiguration(
            configurationModule=commonconfig,
            applicationName="PostgreSQL Tests")

        try:
            self.database = db.Database(self.config)
        except (AttributeError, KeyError):
            raise

        self.connection = self.database.connection()
 def testAcceptCommandLine(self):
   opts = []
   args = {}
   opts.append(('c','chickensoup',False,False,'help for the ailing'))
   opts.append(('r','rabbit', True, '', 'rabbits are bunnies'))
   args['configurationOptionsList'] = opts
   sys.argv.append('-c')
   sys.argv.append('--rabbit=bunny')
   conf = CM.newConfiguration(**args)
   assert('chickensoup' in conf)
   assert('rabbit' in conf)
   assert('bunny' == conf.rabbit)
Exemple #27
0
 def setUp(self):
     global me
     # config gets messed up by some tests. Use this one by preference
     self.config = configurationManager.newConfiguration(
         configurationModule=testConfig,
         applicationName='Testing Postgresql Utils')
     for i in self.config:
         try:
             self.config[i] = self.config.get(i) % (replDict)
         except:
             pass
     self.connection = psycopg2.connect(me.dsn)
     self.testDB = TestDB()
Exemple #28
0
    def setUp(self):
        global me
        self.config = cfgManager.newConfiguration(
            configurationModule=testConfig,
            applicationName='Testing ftpscraper')

        myDir = os.path.split(__file__)[0]
        if not myDir:
            myDir = '.'
        replDict = {'testDir': '%s' % myDir}
        for i in self.config:
            try:
                self.config[i] = self.config.get(i) % (replDict)
            except:
                pass
        self.logger = TestingLogger(me.fileLogger)

        self.testConfig = cfgManager.Config([
            ('t', 'testPath', True, './TEST-BUILDS', ''),
            ('f', 'testFileName', True, 'lastrun.pickle', '')
        ])
        self.testConfig["persistentDataPathname"] = os.path.join(
            self.testConfig.testPath, self.testConfig.testFileName)
Exemple #29
0
  def setUp(self):
    global me
    self.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing builds')

    myDir = os.path.split(__file__)[0]
    if not myDir: myDir = '.'
    replDict = {'testDir':'%s'%myDir}
    for i in self.config:
      try:
        self.config[i] = self.config.get(i)%(replDict)
      except:
        pass
    self.logger = TestingLogger(me.fileLogger)

    self.testConfig = configurationManager.Config([('t','testPath', True, './TEST-BUILDS', ''),
                                                   ('f','testFileName', True, 'lastrun.pickle', ''),
                                                  ])
    self.testConfig["persistentDataPathname"] = os.path.join(self.testConfig.testPath, self.testConfig.testFileName)
def createMe():
  global me
  if not me:
    me = Me()
  me.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName = "Testing TopCrashers")
  myDir = os.path.split(__file__)[0]
  if not myDir: myDir = '.'
  replDict = {'testDir':'%s'%myDir}
  for i in me.config:
    try:
      me.config[i] = me.config.get(i)%(replDict)
    except:
      pass
  me.logFilePathname = me.config.logFilePathname
  logfileDir = os.path.split(me.config.logFilePathname)[0]
  try:
    os.makedirs(logfileDir)
  except OSError,x:
    if errno.EEXIST != x.errno: raise
Exemple #31
0
    def requested_processor_id(requested_id):
        """This method makes sure that the configuration parameter
        'processorID' is in the proper form.  If it is an integer, it is cast
        into the integer type.  If it is a string, it is ensured to be one
        of the acceptable values.

        Parameters:
            requested_id - the value is passed in from the 'processorId'
                           configuration parameter.
        Returns:
            an integer or the the strings 'host' or 'auto'"""
        try:
            return int(requested_id)
        except ValueError:
            if requested_id in ('auto', 'host'):
                return requested_id
            else:
                raise scm.OptionError("'%s' is not a valid value" %
                                      requested_id)
  def testNewConfiguration(self):
    '''
    Trick: To call f(**kwargs) with a dictionary d as the single arg, chant f(**d)
    '''
    opts = []
    args = {}
    args['automaticHelp'] = False

    # Test for empty
    conf = CM.newConfiguration(**args)
    assert(not conf.allowableOptionDictionary)

    # Test for autoHelp
    args['automaticHelp'] = True
    conf = CM.newConfiguration(**args)
    assert(2 == len(conf.allowableOptionDictionary))

    # Test for another legal option
    opts.append(('c','chickensoup',False,False,'Help for the ailing'))
    args['automaticHelp'] = True
    args['configurationOptionsList'] = opts
    conf = CM.newConfiguration(**args)
    assert(4 == len(conf.allowableOptionDictionary))

    # Test a config module
    conf = CM.newConfiguration(automaticHelp=False,configurationModule=optionfile)
    cd = conf.allowableOptionDictionary
    assert(5 == len(cd))
    assert ['T', 'testSingleCharacter', True, None] == cd.get('T')[:-1],'but got %s' % (str(cd.get('T')[:-1]))
    assert 'testSingleCharacter imported from' in cd.get('T')[-1],'but got %s' % (str(cd.get('T')[-1]))
    assert 'optionfile' in cd.get('T')[-1],'but got %s' % (str(cd.get('T')[-1]))
    #assert ['T', 'testSingleCharacter', True, None, 'testSingleCharacter imported from optionfile'] == cd.get('T'),'but got %s' % (str(cd.get('T')))
    assert [None, 'testDefault', True, 'default'] == cd.get('testDefault')[:-1], "but got %s" %(str(cd.get('testDefault')[:-1]))
    assert 'testDefault imported from' in cd.get('testDefault')[-1], "but got %s" %(cd.get('testDefault')[-1])
    assert 'optionfile' in cd.get('testDefault')[-1],  "but got %s" %(cd.get('testDefault')[-1])
    #assert([None, 'testDefault', True, 'default', 'testDefault imported from optionfile'] == cd.get('testDefault'))
    assert([None, 'testDoc', True, None, 'test doc'] == cd.get('testDoc'))
    assert [None, 'testNil', True, None] == cd.get('testNil')[:-1], "but got %s" %(str(cd.get('testNil')[:-1]))
    assert 'testNil imported from' in cd.get('testNil')[-1], "but got %s" %(cd.get('testNil')[-1])
    assert 'optionfile' in cd.get('testNil')[-1], "but got %s" %(cd.get('testNil')[-1])
    #assert([None, 'testNil', True, None, 'testNil imported from optionfile'] == cd.get('testNil'))

    # Test failure with good option, bad file
    try:
      copt = [('c',  'config', True, './badone', "the badconfig file")]
      CM.newConfiguration(configurationOptionsList=copt,optionNameForConfigFile = 'config', configurationFileRequired = True)
      assert(False)
    except CM.ConfigFileMissingError, e:
      assert(True)
  def setUp(self):
    configContext = configurationManager.newConfiguration(configurationModule=config, applicationName="Top Crash By URL Summary")
    logger = logging.getLogger("tcbyurl_summary")
    logger.setLevel(logging.DEBUG)
    stderrLog = logging.StreamHandler()
    stderrLog.setLevel(configContext.stderrErrorLoggingLevel)
    stderrLogFormatter = logging.Formatter(configContext.stderrLineFormatString)
    stderrLog.setFormatter(stderrLogFormatter)
    logger.addHandler(stderrLog)
    rotatingFileLog = logging.handlers.RotatingFileHandler(configContext.logFilePathname, "a", configContext.logFileMaximumSize, configContext.logFileMaximumBackupHistory)
    rotatingFileLog.setLevel(logging.DEBUG)
    rotatingFileLogFormatter = logging.Formatter(configContext.logFileLineFormatString)
    rotatingFileLog.setFormatter(rotatingFileLogFormatter)
    logger.addHandler(rotatingFileLog)
    logger.info("current configuration\n%s", str(configContext))


    self.configContext = configContext
    self.logger = logger
    self.rotatingFileLog = rotatingFileLog
def createMe():
    global me
    if not me:
        me = Me()
    me.config = configurationManager.newConfiguration(
        configurationModule=testConfig, applicationName="Testing TopCrashers")
    myDir = os.path.split(__file__)[0]
    if not myDir: myDir = '.'
    replDict = {'testDir': '%s' % myDir}
    for i in me.config:
        try:
            me.config[i] = me.config.get(i) % (replDict)
        except:
            pass
    me.logFilePathname = me.config.logFilePathname
    logfileDir = os.path.split(me.config.logFilePathname)[0]
    try:
        os.makedirs(logfileDir)
    except OSError, x:
        if errno.EEXIST != x.errno: raise
Exemple #35
0
def main():
  addedConfigOptions = [
    ('h','help',False,None,'print this list',None),
    ('P','product-os-count',True,12,'how many product/os pairs to use [1-56]',lambda x: rangeChecker(1,56,x) ),
    ('S','signature-count',True,83,'how many signatures to use [1-121]',lambda x: rangeChecker(1,121,x)),
    ('R','repeat-count',True,2,'how many times to loop [smallish N]',None),
    ('m','mtbf-fill',False,True,'Fill the time_before_failure table',lambda x: True),
    ('u','url-fill',False,True,'Fill the top_crashes_by_url table',lambda x: True),
    ('s','sig-fill',False,True,'Fill the top_crashes_by_signature table',lambda x: True),
    ('a','all-fill',False,True,'Fill all three matrialized view tables',[('mtbf-fill',None),('url-fill',None),('sig-fill',None)]),
    ('D','drop-all',False,True,'Drop all the database tables. Do no other work',lambda x: True),
    (None,'logFileErrorLoggingLevel',True,logging.WARNING,'logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)',None),
    ]
  config = configurationManager.newConfiguration(configurationModule = commonConfig, applicationName='fillSchema.py',configurationOptionsList = addedConfigOptions)

  myDir = os.path.split(__file__)[0]
  logDir = os.path.join(myDir,'logs')
  logFile = os.path.join(logDir,'fillDB.log')
  try:
    os.makedirs(logDir)
  except OSError:
    pass
  logger = logging.getLogger("fillDB")
  fileLog = logging.FileHandler(logFile)
  fileLog.setLevel(int(config.logFileErrorLoggingLevel))
  fileLogFormatter = logging.Formatter('%(asctime)s %(levelname)s - %(message)s')
  fileLog.setFormatter(fileLogFormatter)

  # now add the handler for all the logs of interest
  logger.addHandler(fileLog)
  logger.setLevel(int(config.logFileErrorLoggingLevel))
  socorro_cia.logger.addHandler(fileLog)
  socorro_cia.logger.setLevel(int(config.logFileErrorLoggingLevel))
  topcrasher.logger.addHandler(fileLog)
  topcrasher.logger.setLevel(int(config.logFileErrorLoggingLevel))
  topcrashbyurl.logger.addHandler(fileLog)
  topcrashbyurl.logger.setLevel(int(config.logFileErrorLoggingLevel))

  logger.info("Config is\n%s",str(config))

  createData(config,logger)
  def setUp(self):
    global me
    self.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing MTBF')
    
    myDir = os.path.split(__file__)[0]
    if not myDir: myDir = '.'
    replDict = {'testDir':'%s'%myDir}
    for i in self.config:
      try:
        self.config[i] = self.config.get(i)%(replDict)
      except:
        pass
    self.logger = TestingLogger(me.fileLogger)
    self.connection = psycopg2.connect(me.dsn)
    cursor = self.connection.cursor()
    self.testDB = TestDB()
    self.testDB.removeDB(self.config,self.logger)

    self.testDB.createDB(self.config,self.logger)
    self.prods = ['zorro','vogel','lizz',]
    self.oss = ['OSX','LOX','WOX',]
    self.productDimData = [] # filled in by fillMtbfTables
Exemple #37
0
def setup_module():
  global me
  tutil.nosePrintModule(__file__)
  if me:
    return
  me = Me()
  me.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing TCByUrl')
  tcbu.logger.setLevel(logging.DEBUG)
  myDir = os.path.split(__file__)[0]
  if not myDir: myDir = '.'
  replDict = {'testDir':'%s'%myDir}
  for i in me.config:
    try:
      me.config[i] = me.config.get(i)%(replDict)
    except:
      pass
  me.logFilePathname = me.config.logFilePathname
  logFileDir = os.path.split(me.logFilePathname)[0]
  try:
    os.makedirs(logFileDir)
  except OSError,x:
    if errno.EEXIST == x.errno: pass
    else: raise
Exemple #38
0
def setup_module():
    global me
    if me:
        return
    me = Me()
    me.config = configurationManager.newConfiguration(
        configurationModule=testConfig, applicationName='Testing dbtestutil')
    tutil.nosePrintModule(__file__)
    myDir = os.path.split(__file__)[0]
    if not myDir: myDir = '.'
    replDict = {'testDir': '%s' % myDir}
    for i in me.config:
        try:
            me.config[i] = me.config.get(i) % (replDict)
        except:
            pass

    logger.setLevel(logging.DEBUG)
    logFilePathname = me.config.logFilePathname
    logfileDir = os.path.split(me.config.logFilePathname)[0]
    try:
        os.makedirs(logfileDir)
    except OSError, x:
        if errno.EEXIST != x.errno: raise
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# Deliberately use base-class's Option class just to see if things break (they better not)
import socorro.lib.ConfigurationManager as cm

testOption0 = cm.Option()
testOption0.doc = 'option 0'
testOption0.default = 0

testOption1 = cm.Option()
testOption1.doc = 'option one'
testOption1.default = 'one'

logFilePathname = cm.Option()
logFilePathname.default = '/some/bogus/location'
Exemple #40
0
import logging
import logging.handlers
import sys

try:
  import config.fixbrokendumpsconfig as configModule
except ImportError:
    import fixbrokendumpsconfig as configModule

import socorro.lib.ConfigurationManager as configurationManager
import socorro.cron.fixBrokenDumps as fixBrokenDumps
import socorro.lib.util as sutil

try:
  config = configurationManager.newConfiguration(configurationModule=configModule, applicationName="Fix Broken Dumps")
except configurationManager.NotAnOptionError, x:
  print >>sys.stderr, x
  print >>sys.stderr, "for usage, try --help"
  sys.exit(1)

logger = logging.getLogger("fix_broken_dumps")
logger.setLevel(logging.DEBUG)

sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)

try:
  last_date_processed = fixBrokenDumps.fix(config, logger, config.brokenFirefoxLinuxQuery, config.brokenFirefoxLinuxFixer)
  last_date_processed = fixBrokenDumps.fix(config, logger, config.brokenFennecQuery, config.brokenFennecFixer)
Exemple #41
0
import sys
import logging
import logging.handlers

try:
  import config.duplicatesconfig as configModule
except ImportError:
  import duplicatesconfig as configModule

import socorro.lib.ConfigurationManager as configurationManager
import socorro.cron.duplicates as duplicates
import socorro.lib.util as sutil

try:
  config = configurationManager.newConfiguration(configurationModule=configModule, applicationName="Duplicate Detector 0.1")
except configurationManager.NotAnOptionError, x:
  print >>sys.stderr, x
  print >>sys.stderr, "for usage, try --help"
  sys.exit()

logger = logging.getLogger("duplicates")
logger.setLevel(logging.DEBUG)

sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)

try:
  duplicates.find_duplicates(config)
finally:
  logger.info("done.")
Exemple #42
0
import sys
import logging
import logging.handlers

try:
    import config.dailyurlconfig as configModule
except ImportError:
    import dailyurlconfig as configModule

import socorro.lib.ConfigurationManager as configurationManager
import socorro.cron.dailyUrl as url
import socorro.lib.util as sutil

try:
    config = configurationManager.newConfiguration(
        configurationModule=configModule, applicationName="Daily URL Dump 0.1")
except configurationManager.NotAnOptionError, x:
    print >> sys.stderr, x
    print >> sys.stderr, "for usage, try --help"
    sys.exit()

logger = logging.getLogger("dailyUrlDump")
logger.setLevel(logging.DEBUG)

sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)

try:
    url.dailyUrlDump(config)
finally:
    logger.info("done.")
Exemple #43
0
#bigStorageRoot.doc = 'the root of the file system where big test set is found'
#bigStorageRoot.default = '%(testDir)s/testdata'

#dumpDirPrefix = cm.Option()
#dumpDirPrefix.doc = 'dump directory names begin with this prefix'
#dumpDirPrefix.default = 'tst_'

#jsonFileSuffix = cm.Option()
#jsonFileSuffix.doc = 'the suffix used to identify a json file'
#jsonFileSuffix.default = '.json'

#dumpFileSuffix = cm.Option()
#dumpFileSuffix.doc = 'the suffix used to identify a dump file'
#dumpFileSuffix.default = '.dump'

processorCheckInTime = cm.Option()
processorCheckInTime.doc = 'the time after which a processor is considered dead (HH:MM:SS)'
processorCheckInTime.default = "00:05:00"
processorCheckInTime.fromStringConverter = lambda x: str(
    cm.timeDeltaConverter(x))

standardLoopDelay = cm.Option()
standardLoopDelay.doc = 'the time between scans for jobs (HHH:MM:SS)'
standardLoopDelay.default = '00:05:00'
standardLoopDelay.fromStringConverter = cm.timeDeltaConverter

cleanupJobsLoopDelay = cm.Option()
cleanupJobsLoopDelay.doc = 'the time between runs of the job clean up routines (HHH:MM:SS)'
cleanupJobsLoopDelay.default = '00:05:00'
cleanupJobsLoopDelay.fromStringConverter = cm.timeDeltaConverter
Exemple #44
0
import sys
import logging
import logging.handlers

try:
  import config.bugzillaconfig as configModule
except ImportError:
  import bugzillaconfig as configModule

import socorro.lib.ConfigurationManager as configurationManager
import socorro.cron.bugzilla as bug
import socorro.lib.util as sutil

try:
  config = configurationManager.newConfiguration(configurationModule=configModule, applicationName="Bugzilla Associations 0.1")
except configurationManager.NotAnOptionError, x:
  print >>sys.stderr, x
  print >>sys.stderr, "for usage, try --help"
  sys.exit()

logger = logging.getLogger("bugzilla")
logger.setLevel(logging.DEBUG)

sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)

try:
  bug.record_associations(config)
finally:
  logger.info("done.")
Exemple #45
0
from socorro.unittest.config.commonconfig \
    import databaseHost as database_hostname
try:
  from socorro.unittest.config.commonconfig \
    import databasePort as database_port
except:
  database_port = 5432
from socorro.unittest.config.commonconfig \
    import oldDatabaseName as database_name
from socorro.unittest.config.commonconfig \
    import databaseUserName as database_username
from socorro.unittest.config.commonconfig \
    import databasePassword as database_password

logFilePathname = cm.Option()
logFilePathname.doc = 'full pathname for the log file'
logFilePathname.default = '%(testDir)s/logs/lib_test.log'

logFileMaximumSize = cm.Option()
logFileMaximumSize.doc = 'maximum size in bytes of the log file'
logFileMaximumSize.default = 1000000

logFileMaximumBackupHistory = cm.Option()
logFileMaximumBackupHistory.doc = 'maximum number of log files to keep'
logFileMaximumBackupHistory.default = 50

logFileLineFormatString = cm.Option()
logFileLineFormatString.doc = 'python logging system format for log file entries'
logFileLineFormatString.default = '%(asctime)s %(levelname)s - %(message)s'
Exemple #46
0
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import socorro.lib.ConfigurationManager as cm

testNil = cm.Option()

testSingleCharacter = cm.Option()
testSingleCharacter.singleCharacter = 'T'

testDefault = cm.Option()
testDefault.default = 'default'

testDoc = cm.Option()
testDoc.doc = 'test doc'

Exemple #47
0
import logging
import logging.handlers
import sys
import datetime as dt

try:
  import config.createpartitionsconfig as configModule
except ImportError:
    import createpartitionsconfig as configModule

import socorro.lib.ConfigurationManager as configurationManager
import socorro.database.schema as schema
import socorro.lib.util as sutil

try:
  config = configurationManager.newConfiguration(configurationModule=configModule, applicationName="startNextPartition")
except configurationManager.NotAnOptionError, x:
  print >>sys.stderr, x
  print >>sys.stderr, "for usage, try --help"
  sys.exit(1)

logger = logging.getLogger("nextPartition")
logger.setLevel(logging.DEBUG)

sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)

try:
  config["endDate"] = config.startDate + dt.timedelta(config.weeksIntoFuture * 7)
  print
  schema.createPartitions(config, logger)
Exemple #48
0
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import web
import socorro.webapi.class_partial as cpart
import socorro.lib.ConfigurationManager as cm
import socorro.collector.wsgicollector as wscol
import socorro.lib.util as sutil
#import socorro.webapi.hello as hello

import config.collectorconfig as collectorConfig

#-------------------------------------------------------------------------------
config = \
    cm.newConfiguration(configurationModule=collectorConfig,
                                        applicationName="Socorro Collector 3.0")

#-------------------------------------------------------------------------------
import logging
import logging.handlers

logger = logging.getLogger("collector")
logger.setLevel(logging.DEBUG)

syslog = logging.handlers.SysLogHandler(
  address=(config.syslogHost, config.syslogPort),
  facility=config.syslogFacilityString,
)
syslog.setLevel(config.syslogErrorLoggingLevel)
syslogFormatter = logging.Formatter(config.syslogLineFormatString)
syslog.setFormatter(syslogFormatter)
import logging
import logging.handlers
import sys
import time

try:
    import config.topCrashesByUrlConfig as configModule
except ImportError:
    import topCrashesByUrlConfig as configModule

import socorro.lib.ConfigurationManager as configurationManager
import socorro.cron.topCrashesByUrl as tcbyurl
import socorro.lib.util as sutil

config = configurationManager.newConfiguration(
    configurationModule=configModule,
    applicationName="Top Crash By URL Summary")

logger = logging.getLogger("topCrashesByUrl")
logger.setLevel(logging.DEBUG)

sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)

try:
    before = time.time()
    tu = tcbyurl.TopCrashesByUrl(config)
    tu.processDateInterval()
    logger.info("Successfully ran in %d seconds" % (time.time() - before))
finally:
    logger.info("done.")
Exemple #50
0
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import sys
import logging
import logging.handlers

import config.crashmoverconfig as cmconf

import socorro.lib.ConfigurationManager as configurationManager
import socorro.storage.storageMover as smover
import socorro.lib.util as sutil

try:
    config = configurationManager.newConfiguration(
        configurationModule=cmconf, applicationName="New Crash Mover")
except configurationManager.NotAnOptionError, x:
    print >> sys.stderr, x
    print >> sys.stderr, "for usage, try --help"
    sys.exit()

logger = logging.getLogger("newCrashMover")
logger.setLevel(logging.DEBUG)

sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)

config.logger = logger

try:
    smover.move(config)
import sys
import logging
import logging.handlers
from datetime import date, timedelta

try:
    import config.dailyMatviewsConfig as configModule
except ImportError:
    import dailyMatviewsConfig as configModule

import socorro.lib.ConfigurationManager as configurationManager
import socorro.cron.dailyMatviews as dailyMatviews
import socorro.lib.util as sutil

try:
    config = configurationManager.newConfiguration(
        configurationModule=configModule, applicationName="dailyMatviews 0.1")
except configurationManager.NotAnOptionError, x:
    print >> sys.stderr, x
    print >> sys.stderr, "for usage, try --help"
    sys.exit()

logger = logging.getLogger("dailyMatviews")
logger.setLevel(logging.DEBUG)

sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)

exitCode = 255

try:
    targetDate = date.today() - timedelta(1)
Exemple #52
0
import sys
import logging
import logging.handlers

try:
    import config.updateadus as configModule
except ImportError:
    import updateadus as configModule

import socorro.lib.ConfigurationManager as configurationManager
import socorro.cron.updateADUs as updateADUs
import socorro.lib.util as sutil

try:
    config = configurationManager.newConfiguration(
        configurationModule=configModule, applicationName="Update ADUs 0.1")
except configurationManager.NotAnOptionError, x:
    print >> sys.stderr, x
    print >> sys.stderr, "for usage, try --help"
    sys.exit()

logger = logging.getLogger("updateADUs")
logger.setLevel(logging.DEBUG)

sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)

try:
    updateADUs.update_adus(config)
finally:
    logger.info("done.")
Exemple #53
0
# XXX Set to be deprecated in favor of socorro/external/postgresql/models.py

import socorro.database.database as db
import psycopg2
import psycopg2.extensions
import logging
import threading

from socorro.unittest.testlib.loggerForTest import TestingLogger
from createDBforTest import *
import socorro.lib.util as util
from socorro.unittest.testbase import TestCase
import socorro.lib.ConfigurationManager as cm
import dbTestconfig as testConfig
config = cm.newConfiguration(configurationModule=testConfig,
                             applicationName='Testing Psycopghelper')
"""
Assume that psycopg2 works, then all we need to do is assure ourselves
that our simplistic wrap around a returned array is correct
"""


class TestMultiCursor(psycopg2.extensions.cursor):
    def __init__(self, numCols=4, numRows=2, **kwargs):
        self.result = []
        for i in range(numRows):
            aRow = []
            for j in range(numCols):
                aRow.append('Row %d, Column %d' % (i, j))
            self.result.append(aRow)
        self.next = self.__next()
Exemple #54
0
import web
import datetime as dt

import config.webapiconfig as configModule
from config import revisionsconfig

import socorro.lib.ConfigurationManager as configurationManager
import socorro.lib.datetimeutil as dtutil
import socorro.lib.productVersionCache as pvc
import socorro.webapi.webapiService as webapi
import socorro.lib.util as sutil

import logging
import logging.handlers

config = configurationManager.newConfiguration(
    configurationModule=configModule, applicationName="Socorro Webapi")

# Adding revisions of Socorro and Breakpad for the server status service.
revisions = configurationManager.newConfiguration(
    configurationModule=revisionsconfig, applicationName="Socorro Revisions")
config.update(revisions)

logger = logging.getLogger("webapi")
logger.setLevel(logging.DEBUG)

syslog = logging.handlers.SysLogHandler(facility=config.syslogFacilityString)
syslog.setLevel(config.syslogErrorLoggingLevel)
syslogFormatter = logging.Formatter(config.syslogLineFormatString)
syslog.setFormatter(syslogFormatter)
logger.addHandler(syslog)
Exemple #55
0
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

## Test config file for database utilities
import socorro.lib.ConfigurationManager as cm
import datetime

from socorro.unittest.config.commonconfig import databaseHost
from socorro.unittest.config.commonconfig import databaseName
from socorro.unittest.config.commonconfig import databaseUserName
from socorro.unittest.config.commonconfig import databasePassword

logFilePathname = cm.Option()
logFilePathname.doc = 'full pathname for the log file'
logFilePathname.default = '%(testDir)s/logs/db_test.log'

Exemple #56
0
 def setUp(self):
     """Create a configuration context."""
     self.config = ConfigurationManager.newConfiguration(
         configurationModule=commonconfig, applicationName="RabbitMQ Tests")
Exemple #57
0
import unittest
import socorro.database.database as db
import psycopg2
import psycopg2.extensions
import logging
import threading

from socorro.unittest.testlib.loggerForTest import TestingLogger
import socorro.unittest.testlib.util as tutil
from createDBforTest import *
import socorro.lib.util as util

import socorro.lib.ConfigurationManager as cm
import dbTestconfig as testConfig
config = cm.newConfiguration(configurationModule = testConfig, applicationName='Testing Psycopghelper')
def setup_module():
  tutil.nosePrintModule(__file__)

"""
Assume that psycopg2 works, then all we need to do is assure ourselves
that our simplistic wrap around a returned array is correct
"""

class TestMultiCursor(psycopg2.extensions.cursor):
  def __init__(self,numCols = 4, numRows=2, **kwargs):
    self.result = []
    for i in range(numRows):
      aRow = []
      for j in range(numCols):
        aRow.append('Row %d, Column %d' %(i,j))
Exemple #58
0
import sys
import logging
import logging.handlers

import config.submitterconfig as subconf

import socorro.lib.ConfigurationManager as configurationManager
import socorro.collector.submitter as sub
import socorro.lib.util as sutil

import poster

poster.streaminghttp.register_openers()

try:
    config = configurationManager.newConfiguration(
        configurationModule=subconf, applicationName="submitter 2.0")
except configurationManager.NotAnOptionError, x:
    print >> sys.stderr, x
    print >> sys.stderr, "for usage, try --help"
    sys.exit()

logger = logging.getLogger("submitter")
logger.setLevel(logging.DEBUG)

sutil.setupLoggingHandlers(logger, config)
sutil.echoConfig(logger, config)

config.logger = logger

if config.numberOfSubmissions == 'forever':
    config.iteratorFunc = sub.createInfiniteFileSystemIterator