def testCrashStorageSystemForHBase___init__(): d = util.DotDict() j = util.DotDict() d.hbaseHost = 'fred' d.hbasePort = 'ethel' d.hbaseTimeout = 9000 j.root = d.hbaseFallbackFS = '.' d.throttleConditions = [] j.maxDirectoryEntries = d.hbaseFallbackDumpDirCount = 1000000 j.jsonSuffix = d.jsonFileSuffix = '.json' j.dumpSuffix = d.dumpFileSuffix = '.dump' j.dumpGID = d.hbaseFallbackdumpGID = 666 j.dumpPermissions = d.hbaseFallbackDumpPermissions = 660 j.dirPermissions = d.hbaseFallbackDirPermissions = 770 j.logger = d.logger = util.SilentFakeLogger() fakeHbaseConnection = exp.DummyObjectWithExpectations( 'fakeHbaseConnection') fakeHbaseModule = exp.DummyObjectWithExpectations('fakeHbaseModule') fakeHbaseModule.expect('HBaseConnectionForCrashReports', (d.hbaseHost, d.hbasePort, d.hbaseTimeout), {"logger": d.logger}, fakeHbaseConnection, None) fakeHbaseConnection.expect('hbaseThriftExceptions', None, None, (), None) fakeHbaseModule.expect('NoConnectionException', None, None, hbc.NoConnectionException, None) fakeJsonDumpStore = exp.DummyObjectWithExpectations('fakeJsonDumpStore') fakeJsonDumpModule = exp.DummyObjectWithExpectations('fakeJsonDumpModule') fakeJsonDumpModule.expect('JsonDumpStorage', (), j, fakeJsonDumpStore, None) css = cstore.CrashStorageSystemForHBase(d, configPrefix='', hbaseClient=fakeHbaseModule, jsonDumpStorage=fakeJsonDumpModule) assert css.hbaseConnection == fakeHbaseConnection
def __init__(self, host, port, timeout, thrift=Thrift, tsocket=TSocket, ttrans=TTransport, protocol=TBinaryProtocol, ttp=ttypes, client=Client, column=ColumnDescriptor, mutation=Mutation, logger=utl.SilentFakeLogger()): super(HBaseConnectionForCrashReports, self).__init__( host, port, timeout, thrift, tsocket, ttrans, protocol, ttp, client, column, mutation, logger )
def testDoingWorkWithTwoWorkers(): logger = sutil.SilentFakeLogger() config = sutil.DotDict({'logger': logger, 'numberOfThreads': 2}) myList = [] def insertIntoList(anItem): myList.append(anItem[0]) return siwf.OK iwf = siwf.IteratorWorkerFramework(config, name='Wilma', taskFunc=insertIntoList) try: iwf.start() time.sleep(2.0) assert len(iwf.workerPool.threadList ) == 2, "expected 2 threads, but found %d" % len( iwf.workerPool.threadList) assert len( myList ) == 10, 'expected to do 10 inserts, but %d were done instead' % len( myList) assert sorted(myList) == range( 10), 'expected %s, but got %s' % (range(10), sorted(myList)) iwf.stop() except Exception: # we got threads to join iwf.workerPool.waitForCompletion() raise
def testCrashStorageForDualHbaseCrashStorageSystem01(): d = util.DotDict() j = util.DotDict() d.hbaseHost = 'fred' d.secondaryHbaseHost = 'barney' d.hbasePort = 'ethel' d.secondaryHbasePort = 'betty' d.hbaseTimeout = 3000 d.secondaryHbaseTimeout = 10000 j.root = d.hbaseFallbackFS = '.' d.throttleConditions = [] j.maxDirectoryEntries = d.hbaseFallbackDumpDirCount = 1000000 j.jsonSuffix = d.jsonFileSuffix = '.json' j.dumpSuffix = d.dumpFileSuffix = '.dump' j.dumpGID = d.hbaseFallbackdumpGID = 666 j.dumpPermissions = d.hbaseFallbackDumpPermissions = 660 j.dirPermissions = d.hbaseFallbackDirPermissions = 770 j.logger = d.logger = util.SilentFakeLogger() fakeHbaseConnection1 = exp.DummyObjectWithExpectations( 'fakeHbaseConnection1') fakeHbaseConnection2 = exp.DummyObjectWithExpectations( 'fakeHbaseConnection2') fakeHbaseConnection1.expect('hbaseThriftExceptions', None, None, (), None) fakeHbaseConnection1.expect('get_json', ('fakeOoid1', ), {'number_of_retries': 2}, 'fake_json1') import socorro.storage.hbaseClient as hbc fakeHbaseConnection1.expect('get_json', ('fakeOoid2', ), {'number_of_retries': 2}, None, hbc.OoidNotFoundException()) fakeHbaseConnection2.expect('hbaseThriftExceptions', None, None, (), None) fakeHbaseConnection2.expect('get_json', ('fakeOoid2', ), {'number_of_retries': 2}, 'fake_json2') fakeHbaseModule = exp.DummyObjectWithExpectations('fakeHbaseModule') fakeHbaseModule.expect('HBaseConnectionForCrashReports', (d.hbaseHost, d.hbasePort, d.hbaseTimeout), {"logger": d.logger}, fakeHbaseConnection1, None) fakeHbaseModule.expect('NoConnectionException', None, None, hbc.NoConnectionException, None) fakeHbaseModule.expect( 'HBaseConnectionForCrashReports', (d.secondaryHbaseHost, d.secondaryHbasePort, d.secondaryHbaseTimeout), {"logger": d.logger}, fakeHbaseConnection2, None) fakeHbaseModule.expect('NoConnectionException', None, None, hbc.NoConnectionException, None) fakeJsonDumpStore = exp.DummyObjectWithExpectations('fakeJsonDumpStore') fakeJsonDumpModule = exp.DummyObjectWithExpectations('fakeJsonDumpModule') fakeJsonDumpModule.expect('JsonDumpStorage', (), j, fakeJsonDumpStore, None) fakeJsonDumpModule.expect('JsonDumpStorage', (), j, fakeJsonDumpStore, None) css = cstore.DualHbaseCrashStorageSystem( d, hbaseClient=fakeHbaseModule, jsonDumpStorage=fakeJsonDumpModule) assert css.hbaseConnection == fakeHbaseConnection1 assert css.fallbackHBase.hbaseConnection == fakeHbaseConnection2 result = css.get_meta('fakeOoid1') assert result == 'fake_json1' result = css.get_meta('fakeOoid2') assert result == 'fake_json2'
def testMovement(): """testMovement (this will take 15-20 seconds)""" config = sutil.DotDict({ 'logger': sutil.SilentFakeLogger(), 'numberOfThreads': 1, }) fakeHbaseStorage = getHbaseStorage1(config) smover.move(config, sourceCrashStorageClass=fakeSource, destCrashStorageClass=fakeHbaseStorage)
def twoPeriodTopCrasherComparison( databaseConnection, context, closestEntryFunction=latestEntryBeforeOrEqualTo, listOfTopCrashersFunction=getListOfTopCrashersBySignature): try: context['logger'].debug('entered twoPeriodTopCrasherComparison') except KeyError: context['logger'] = util.SilentFakeLogger() assertions = ['to_date', 'duration', 'product', 'version'] for param in assertions: assert param in context, ("%s is missing from the configuration" % param) context['numberOfComparisonPoints'] = 2 if not context['limit']: context['limit'] = 100 #context['logger'].debug('about to latestEntryBeforeOrEqualTo') context['to_date'] = closestEntryFunction(databaseConnection, context['to_date'], context['product'], context['version']) context['logger'].debug('New to_date: %s' % context['to_date']) context['startDate'] = context.to_date - (context.duration * context.numberOfComparisonPoints) #context['logger'].debug('after %s' % context) listOfTopCrashers = listOfListsWithChangeInRank( rangeOfQueriesGenerator(databaseConnection, context, listOfTopCrashersFunction))[0] #context['logger'].debug('listOfTopCrashers %s' % listOfTopCrashers) totalNumberOfCrashes = totalPercentOfTotal = 0 for x in listOfTopCrashers: if 'total_crashes' in x: totalNumberOfCrashes = x['total_crashes'] del x['total_crashes'] totalPercentOfTotal += x.get('percentOfTotal', 0) result = { 'crashes': listOfTopCrashers, 'start_date': datetimeutil.date_to_string(context.to_date - context.duration), 'end_date': datetimeutil.date_to_string(context.to_date), 'totalNumberOfCrashes': totalNumberOfCrashes, 'totalPercentage': totalPercentOfTotal, } #logger.debug("about to return %s", result) return result
def testCrashStorageSystemForHBase_save_1(): """straight save into hbase with no trouble""" currentTimestamp = 'now' expectedDumpResult = '1234567890/n' jdict = util.DotDict({ 'ProductName': 'FireFloozy', 'Version': '3.6', 'legacy_processing': 1 }) d = util.DotDict() j = util.DotDict() d.hbaseHost = 'fred' d.hbasePort = 'ethel' d.hbaseTimeout = 9000 j.root = d.hbaseFallbackFS = '.' d.throttleConditions = [] j.maxDirectoryEntries = d.hbaseFallbackDumpDirCount = 1000000 j.jsonSuffix = d.jsonFileSuffix = '.json' j.dumpSuffix = d.dumpFileSuffix = '.dump' j.dumpGID = d.hbaseFallbackdumpGID = 666 j.dumpPermissions = d.hbaseFallbackDumpPermissions = 660 j.dirPermissions = d.hbaseFallbackDirPermissions = 770 d.logger = util.SilentFakeLogger() fakeHbaseConnection = exp.DummyObjectWithExpectations( 'fakeHbaseConnection') fakeHbaseConnection.expect('hbaseThriftExceptions', None, None, (), None) fakeHbaseConnection.expect('put_json_dump', ('uuid', jdict, expectedDumpResult), {"number_of_retries": 2}, None, None) fakeHbaseModule = exp.DummyObjectWithExpectations('fakeHbaseModule') fakeHbaseModule.expect('HBaseConnectionForCrashReports', (d.hbaseHost, d.hbasePort, d.hbaseTimeout), {"logger": d.logger}, fakeHbaseConnection, None) fakeHbaseModule.expect('NoConnectionException', None, None, hbc.NoConnectionException, None) fakeJsonDumpStore = exp.DummyObjectWithExpectations('fakeJsonDumpStore') fakeJsonDumpModule = exp.DummyObjectWithExpectations('fakeJsonDumpModule') fakeJsonDumpModule.expect('JsonDumpStorage', (), j, fakeJsonDumpStore, None) css = cstore.CrashStorageSystemForHBase(d, configPrefix='', hbaseClient=fakeHbaseModule, jsonDumpStorage=fakeJsonDumpModule) expectedResult = cstore.CrashStorageSystem.OK result = css.save_raw('uuid', jdict, expectedDumpResult, currentTimestamp) assert result == expectedResult, 'expected %s but got %s' % ( expectedResult, result)
def testConstuctor1(): logger = sutil.SilentFakeLogger() config = sutil.DotDict({'logger': logger, 'numberOfThreads': 1}) iwf = siwf.IteratorWorkerFramework( config, name='Wilma', ) try: assert iwf.config == config assert iwf.name == 'Wilma' assert iwf.logger == logger assert iwf.taskFunc == siwf.defaultTaskFunc assert iwf.quit == False finally: # we got threads to join iwf.workerPool.waitForCompletion()
def setUp(self): self.testDir = os.path.join('.','TEST-DUMPSTORAGE')+'/' fakeLogger = socorro_util.SilentFakeLogger() self.initKwargs = { 0:{'logger': fakeLogger,}, 1:{'logger': fakeLogger,'fileSuffix':'DSgz',}, 2:{'logger': fakeLogger,'fileSuffix':'.DSgz',}, 3:{'logger': fakeLogger,'gzipCompression':'3',}, 4:{'logger': fakeLogger,'storageDepth':'3',}, 5:{'logger': fakeLogger,'rootName':'someRoot', 'dateName':'someDate', 'minutesPerSlot':'12'} } try: shutil.rmtree(self.testDir) except OSError: pass # ok if there is no such test directory os.mkdir(self.testDir)
def testCrashStorageSystemForHBase_save_4(): """hbase fails with a NoConnectionException but there is no filesystem fallback - expecting retry return""" currentTimestamp = 'now' expectedDumpResult = '1234567890/n' jdict = {'a': 2, 'b': 'hello'} d = util.DotDict() d.hbaseHost = 'fred' d.hbasePort = 'ethel' d.hbaseTimeout = 9000 d.hbaseFallbackFS = '' d.throttleConditions = [] d.hbaseFallbackDumpDirCount = 1000000 d.jsonFileSuffix = '.json' d.dumpFileSuffix = '.dump' d.hbaseFallbackDumpGID = 666 d.hbaseFallbackDumpPermissions = 660 d.hbaseFallbackDirPermissions = 770 d.logger = util.SilentFakeLogger() fakeHbaseConnection = exp.DummyObjectWithExpectations( 'fakeHbaseConnection') fakeHbaseConnection.expect('hbaseThriftExceptions', None, None, (), None) fakeHbaseConnection.expect('put_json_dump', ('uuid', jdict, expectedDumpResult), {"number_of_retries": 2}, None, hbc.NoConnectionException(Exception())) fakeHbaseModule = exp.DummyObjectWithExpectations('fakeHbaseModule') fakeHbaseModule.expect('HBaseConnectionForCrashReports', (d.hbaseHost, d.hbasePort, d.hbaseTimeout), {"logger": d.logger}, fakeHbaseConnection, None) fakeHbaseModule.expect('NoConnectionException', None, None, hbc.NoConnectionException, None) fakeJsonDumpModule = exp.DummyObjectWithExpectations('fakeJsonDumpModule') cstore.logger = loggerForTest.TestingLogger() css = cstore.CrashStorageSystemForHBase(d, '', fakeHbaseModule, fakeJsonDumpModule) expectedResult = cstore.CrashStorageSystem.RETRY print css.exceptionsEligibleForRetry result = css.save_raw('uuid', jdict, expectedDumpResult, currentTimestamp) assert result == expectedResult, 'expected %s but got %s' % ( expectedResult, result)
def __init__(self, host, port, timeout, thrift=Thrift, tsocket=TSocket, ttrans=TTransport, protocol=TBinaryProtocol, ttp=ttypes, client=Client, column=ColumnDescriptor, mutation=Mutation, logger=utl.SilentFakeLogger()): self.host = host self.port = port self.timeout = timeout self.thriftModule = thrift self.tsocketModule = tsocket self.transportModule = ttrans self.protocolModule = protocol self.ttypesModule = ttp self.clientClass = client self.columnClass = column self.mutationClass = mutation self.logger = logger self.hbaseThriftExceptions = ( self.ttypesModule.IOError, #self.ttypesModule.IllegalArgument, #self.ttypesModule.AlreadyExists, self.thriftModule.TException, #HBaseClientException, socket.timeout, socket.error, FatalException) self.operational_exceptions = self.hbaseThriftExceptions self.conditional_exceptions = () try: self.make_connection(timeout=self.timeout) except NoConnectionException: utl.reportExceptionAndContinue(logger=self.logger)
def testStart1(): logger = sutil.SilentFakeLogger() config = sutil.DotDict({'logger': logger, 'numberOfThreads': 1}) iwf = siwf.IteratorWorkerFramework( config, name='Wilma', ) try: iwf.start() time.sleep(2.0) assert iwf.queuingThread.isAlive(), "the queing thread is not running" assert len( iwf.workerPool.threadList) == 1, "where's the worker thread?" assert iwf.workerPool.threadList[0].isAlive( ), "the worker thread is stillborn" iwf.stop() assert iwf.queuingThread.isAlive( ) == False, "the queuing thread did not stop" except Exception: # we got threads to join iwf.workerPool.waitForCompletion()
def testLegacyThrottler(): config = util.DotDict() config.throttleConditions = [('alpha', re.compile('ALPHA'), 100), ('beta', 'BETA', 100), ('gamma', lambda x: x == 'GAMMA', 100), ('delta', True, 100), (None, True, 0)] config.minimalVersionForUnderstandingRefusal = { 'product1': '3.5', 'product2': '4.0' } config.neverDiscard = False config.logger = util.SilentFakeLogger() thr = cstore.LegacyThrottler(config) expected = 5 actual = len(thr.processedThrottleConditions) assert expected == actual, "expected thr.preprocessThrottleConditions to have length %d, but got %d instead" % ( expected, actual) json1 = util.DotDict({ 'ProductName': 'product1', 'Version': '3.0', 'alpha': 'ALPHA', }) expected = False actual = thr.understandsRefusal(json1) assert expected == actual, "understand refusal expected %d, but got %d instead" % ( expected, actual) json1 = util.DotDict({ 'ProductName': 'product1', 'Version': '3.6', 'alpha': 'ALPHA', }) expected = True actual = thr.understandsRefusal(json1) assert expected == actual, "understand refusal expected %d, but got %d instead" % ( expected, actual) expected = cstore.LegacyThrottler.ACCEPT actual = thr.throttle(json1) assert expected == actual, "regexp throttle expected %d, but got %d instead" % ( expected, actual) json1 = util.DotDict({ 'ProductName': 'product1', 'Version': '3.4', 'alpha': 'not correct', }) expected = cstore.LegacyThrottler.DEFER actual = thr.throttle(json1) assert expected == actual, "regexp throttle expected %d, but got %d instead" % ( expected, actual) json1 = util.DotDict({ 'ProductName': 'product1', 'Version': '3.6', 'alpha': 'not correct', }) expected = cstore.LegacyThrottler.DISCARD actual = thr.throttle(json1) assert expected == actual, "regexp throttle expected %d, but got %d instead" % ( expected, actual) json1 = util.DotDict({ 'ProductName': 'product1', 'Version': '3.6', 'beta': 'BETA', }) expected = cstore.LegacyThrottler.ACCEPT actual = thr.throttle(json1) assert expected == actual, "string equality throttle expected %d, but got %d instead" % ( expected, actual) json1 = util.DotDict({ 'ProductName': 'product1', 'Version': '3.6', 'beta': 'not BETA', }) expected = cstore.LegacyThrottler.DISCARD actual = thr.throttle(json1) assert expected == actual, "string equality throttle expected %d, but got %d instead" % ( expected, actual) json1 = util.DotDict({ 'ProductName': 'product1', 'Version': '3.6', 'gamma': 'GAMMA', }) expected = cstore.LegacyThrottler.ACCEPT actual = thr.throttle(json1) assert expected == actual, "string equality throttle expected %d, but got %d instead" % ( expected, actual) json1 = util.DotDict({ 'ProductName': 'product1', 'Version': '3.6', 'gamma': 'not GAMMA', }) expected = cstore.LegacyThrottler.DISCARD actual = thr.throttle(json1) assert expected == actual, "string equality throttle expected %d, but got %d instead" % ( expected, actual) json1 = util.DotDict({ 'ProductName': 'product1', 'Version': '3.6', 'delta': "value doesn't matter", }) expected = cstore.LegacyThrottler.ACCEPT actual = thr.throttle(json1) assert expected == actual, "string equality throttle expected %d, but got %d instead" % ( expected, actual) # phase 2 tests config = util.DotDict() config.throttleConditions = [ ('*', lambda x: 'alpha' in x, None), ('*', lambda x: x['beta'] == 'BETA', 100), ] config.minimalVersionForUnderstandingRefusal = { 'product1': '3.5', 'product2': '4.0' } config.neverDiscard = True config.logger = mock.Mock() thr = cstore.LegacyThrottler(config) expected = 2 actual = len(thr.processedThrottleConditions) assert expected == actual, \ "expected thr.preprocessThrottleConditions to have length %d, but got " \ "%d instead" % (expected, actual) raw_crash = util.DotDict({ 'ProductName': 'product1', 'Version': '3.6', 'beta': 'ugh', 'alpha': "value doesn't matter", }) expected = cstore.LegacyThrottler.IGNORE actual = thr.throttle(raw_crash) assert expected == actual, \ "IGNORE expected %d, but got %d instead" % \ (expected, actual) raw_crash = util.DotDict({ 'ProductName': 'product1', 'Version': '3.6', 'beta': 'ugh', 'delta': "value doesn't matter", }) expected = cstore.LegacyThrottler.DEFER actual = thr.throttle(raw_crash) assert expected == actual, \ "DEFER expected %d, but got %d instead" % \ (expected, actual) raw_crash = util.DotDict({ 'ProductName': 'product1', 'Version': '3.6', 'beta': 'BETA', 'alpha': "value doesn't matter", }) expected = cstore.LegacyThrottler.IGNORE actual = thr.throttle(raw_crash) assert expected == actual, \ "IGNORE expected %d, but got %d instead" % \ (expected, actual) raw_crash = util.DotDict({ 'ProductName': 'product1', 'Version': '3.6', 'beta': 'BETA', 'delta': "value doesn't matter", }) expected = cstore.LegacyThrottler.ACCEPT actual = thr.throttle(raw_crash) assert expected == actual, \ "ACCEPT expected %d, but got %d instead" % \ (expected, actual)
def testCrashStorageSystemForHBase_save_2(): """hbase fails, must save to fallback""" currentTimestamp = 'now' expectedDumpResult = '1234567890/n' jdict = util.DotDict({ 'ProductName': 'FireFloozy', 'Version': '3.6', 'legacy_processing': 1 }) d = util.DotDict() j = util.DotDict() d.hbaseHost = 'fred' d.hbasePort = 'ethel' d.hbaseTimeout = 9000 j.root = d.hbaseFallbackFS = '.' d.throttleConditions = [] j.maxDirectoryEntries = d.hbaseFallbackDumpDirCount = 1000000 j.jsonSuffix = d.jsonFileSuffix = '.json' j.dumpSuffix = d.dumpFileSuffix = '.dump' j.dumpGID = d.hbaseFallbackDumpGID = 666 j.dumpPermissions = d.hbaseFallbackDumpPermissions = 660 j.dirPermissions = d.hbaseFallbackDirPermissions = 770 j.logger = d.logger = util.SilentFakeLogger() fakeHbaseConnection = exp.DummyObjectWithExpectations( 'fakeHbaseConnection') fakeHbaseConnection.expect('hbaseThriftExceptions', None, None, (), None) fakeHbaseConnection.expect('put_json_dump', ('uuid', jdict, expectedDumpResult), {"number_of_retries": 2}, None, Exception()) fakeHbaseModule = exp.DummyObjectWithExpectations('fakeHbaseModule') fakeHbaseModule.expect('HBaseConnectionForCrashReports', (d.hbaseHost, d.hbasePort, d.hbaseTimeout), {"logger": d.logger}, fakeHbaseConnection, None) fakeHbaseModule.expect('NoConnectionException', None, None, hbc.NoConnectionException, None) class FakeFile(object): def write(self, x): pass def close(self): pass fakeJsonFile = FakeFile() fakeDumpFile = exp.DummyObjectWithExpectations('fakeDumpFile') fakeDumpFile.expect('write', (expectedDumpResult, ), {}) fakeDumpFile.expect('close', (), {}) fakeJsonDumpStore = exp.DummyObjectWithExpectations('fakeJsonDumpStore') fakeJsonDumpStore.expect('newEntry', ('uuid', os.uname()[1], currentTimestamp), {}, (fakeJsonFile, fakeDumpFile)) fakeJsonDumpModule = exp.DummyObjectWithExpectations('fakeJsonDumpModule') fakeJsonDumpModule.expect('JsonDumpStorage', (), j, fakeJsonDumpStore, None) cstore.logger = loggerForTest.TestingLogger() css = cstore.CollectorCrashStorageSystemForHBase(d, fakeHbaseModule, fakeJsonDumpModule) expectedResult = cstore.CrashStorageSystem.OK result = css.save_raw('uuid', jdict, expectedDumpResult, currentTimestamp) assert result == expectedResult, 'expected %s but got %s' % ( expectedResult, result)
def testCrashStorageSystem_save(): css = cstore.CrashStorageSystem( util.DotDict({'logger': util.SilentFakeLogger()})) result = css.save_raw('fred', 'ethel', 'lucy') assert result == cstore.CrashStorageSystem.NO_ACTION