def test_get_dumps(): dumpBlob = 'abcdefghijklmnopqrstuvwxyz01234567890!@#$%^&*()_-=+' dumpBlob2 = 'abcdefghijklmnopqr$$uvwxyz01234567890!@#$%^&*()_-=+' dumpBlob3 = 'abcdefghijklmnopqrstuvw%%z01234567890!@#$%^&*()_-=+' dummyColumn = exp.DummyObjectWithExpectations() dummyColumn.expect('value', None, None, dumpBlob) dummyColumn2 = exp.DummyObjectWithExpectations() dummyColumn2.expect('value', None, None, dumpBlob2) dummyColumn3 = exp.DummyObjectWithExpectations() dummyColumn3.expect('value', None, None, dumpBlob3) dummyClientRowObject = exp.DummyObjectWithExpectations() dummyClientRowObject.expect('columns', None, None, {'raw_data:dump':dummyColumn, 'raw_data:flash1':dummyColumn2, 'raw_data:flash2':dummyColumn3, }) getRowWithColumnsReturnValue = [dummyClientRowObject] hbcfcr = HBaseConnectionForCrashReportsWithPresetExpectations() conn = hbcfcr.conn dummy_clientObject = hbcfcr.dummy_clientObject dummy_clientObject.expect('getRowWithColumns', ('crash_reports', 'a100102abcdefghijklmnopqrstuvwxyz100102', ['raw_data']), {}, getRowWithColumnsReturnValue) result = conn.get_dumps('abcdefghijklmnopqrstuvwxyz100102') assert result['upload_file_minidump'] == dumpBlob, 'expected %s, but got %s' % (dumpBlob, str(result['upload_file_minidump'])) assert result['flash1'] == dumpBlob2, 'expected %s, but got %s' % (dumpBlob2, str(result['flash1'])) assert result['flash2'] == dumpBlob3, 'expected %s, but got %s' % (dumpBlob3, str(result['flash2']))
def testCrashStorageSystemForHBase___init__(): d = util.DotDict() j = util.DotDict() d.hbaseHost = 'fred' d.hbasePort = 'ethel' d.hbaseTimeout = 9000 j.root = d.hbaseFallbackFS = '.' d.throttleConditions = [] j.maxDirectoryEntries = d.hbaseFallbackDumpDirCount = 1000000 j.jsonSuffix = d.jsonFileSuffix = '.json' j.dumpSuffix = d.dumpFileSuffix = '.dump' j.dumpGID = d.hbaseFallbackdumpGID = 666 j.dumpPermissions = d.hbaseFallbackDumpPermissions = 660 j.dirPermissions = d.hbaseFallbackDirPermissions = 770 j.logger = d.logger = util.SilentFakeLogger() fakeHbaseConnection = exp.DummyObjectWithExpectations( 'fakeHbaseConnection') fakeHbaseModule = exp.DummyObjectWithExpectations('fakeHbaseModule') fakeHbaseModule.expect('HBaseConnectionForCrashReports', (d.hbaseHost, d.hbasePort, d.hbaseTimeout), {"logger": d.logger}, fakeHbaseConnection, None) fakeHbaseConnection.expect('hbaseThriftExceptions', None, None, (), None) fakeHbaseModule.expect('NoConnectionException', None, None, hbc.NoConnectionException, None) fakeJsonDumpStore = exp.DummyObjectWithExpectations('fakeJsonDumpStore') fakeJsonDumpModule = exp.DummyObjectWithExpectations('fakeJsonDumpModule') fakeJsonDumpModule.expect('JsonDumpStorage', (), j, fakeJsonDumpStore, None) css = cstore.CrashStorageSystemForHBase(d, configPrefix='', hbaseClient=fakeHbaseModule, jsonDumpStorage=fakeJsonDumpModule) assert css.hbaseConnection == fakeHbaseConnection
def test_get_json_2(): """that ooid doesn't exist""" jsonDataAsString = '{"a": 1, "b": "hello"}' expectedJson = js.loads(jsonDataAsString) dummyColumn = exp.DummyObjectWithExpectations() dummyColumn.expect('value', None, None, jsonDataAsString) dummyClientRowObject = exp.DummyObjectWithExpectations() dummyClientRowObject.expect('columns', None, None, {'meta_data:json': dummyColumn}) dummyClientRowObject.expect('row', None, None, 'a100102abcdefghijklmnopqrstuvwxyz100102') hbcfcr = HBaseConnectionForCrashReportsWithPresetExpectations() conn = hbcfcr.conn dummy_clientObject = hbcfcr.dummy_clientObject dummy_clientObject.expect( 'getRowWithColumns', ('crash_reports', 'a100102abcdefghijklmnopqrstuvwxyz100102', ['meta_data:json']), {}, []) try: result = conn.get_json('abcdefghijklmnopqrstuvwxyz100102') except Exception, x: expected_exception_as_string = 'OOID not found: abcdefghijklmnopqrstuvwxyz100102 - a100102abcdefghijklmnopqrstuvwxyz100102' actual_exception_as_string = str(x) assert expected_exception_as_string == actual_exception_as_string, 'expected %s, but got %s' % ( expected_exception_as_string, actual_exception_as_string)
def testCrashStorageForDualHbaseCrashStorageSystem01(): d = util.DotDict() j = util.DotDict() d.hbaseHost = 'fred' d.secondaryHbaseHost = 'barney' d.hbasePort = 'ethel' d.secondaryHbasePort = 'betty' d.hbaseTimeout = 3000 d.secondaryHbaseTimeout = 10000 j.root = d.hbaseFallbackFS = '.' d.throttleConditions = [] j.maxDirectoryEntries = d.hbaseFallbackDumpDirCount = 1000000 j.jsonSuffix = d.jsonFileSuffix = '.json' j.dumpSuffix = d.dumpFileSuffix = '.dump' j.dumpGID = d.hbaseFallbackdumpGID = 666 j.dumpPermissions = d.hbaseFallbackDumpPermissions = 660 j.dirPermissions = d.hbaseFallbackDirPermissions = 770 j.logger = d.logger = util.SilentFakeLogger() fakeHbaseConnection1 = exp.DummyObjectWithExpectations( 'fakeHbaseConnection1') fakeHbaseConnection2 = exp.DummyObjectWithExpectations( 'fakeHbaseConnection2') fakeHbaseConnection1.expect('hbaseThriftExceptions', None, None, (), None) fakeHbaseConnection1.expect('get_json', ('fakeOoid1', ), {'number_of_retries': 2}, 'fake_json1') import socorro.storage.hbaseClient as hbc fakeHbaseConnection1.expect('get_json', ('fakeOoid2', ), {'number_of_retries': 2}, None, hbc.OoidNotFoundException()) fakeHbaseConnection2.expect('hbaseThriftExceptions', None, None, (), None) fakeHbaseConnection2.expect('get_json', ('fakeOoid2', ), {'number_of_retries': 2}, 'fake_json2') fakeHbaseModule = exp.DummyObjectWithExpectations('fakeHbaseModule') fakeHbaseModule.expect('HBaseConnectionForCrashReports', (d.hbaseHost, d.hbasePort, d.hbaseTimeout), {"logger": d.logger}, fakeHbaseConnection1, None) fakeHbaseModule.expect('NoConnectionException', None, None, hbc.NoConnectionException, None) fakeHbaseModule.expect( 'HBaseConnectionForCrashReports', (d.secondaryHbaseHost, d.secondaryHbasePort, d.secondaryHbaseTimeout), {"logger": d.logger}, fakeHbaseConnection2, None) fakeHbaseModule.expect('NoConnectionException', None, None, hbc.NoConnectionException, None) fakeJsonDumpStore = exp.DummyObjectWithExpectations('fakeJsonDumpStore') fakeJsonDumpModule = exp.DummyObjectWithExpectations('fakeJsonDumpModule') fakeJsonDumpModule.expect('JsonDumpStorage', (), j, fakeJsonDumpStore, None) fakeJsonDumpModule.expect('JsonDumpStorage', (), j, fakeJsonDumpStore, None) css = cstore.DualHbaseCrashStorageSystem( d, hbaseClient=fakeHbaseModule, jsonDumpStorage=fakeJsonDumpModule) assert css.hbaseConnection == fakeHbaseConnection1 assert css.fallbackHBase.hbaseConnection == fakeHbaseConnection2 result = css.get_meta('fakeOoid1') assert result == 'fake_json1' result = css.get_meta('fakeOoid2') assert result == 'fake_json2'
def testCrashStorageSystemForHBase_save_1(): """straight save into hbase with no trouble""" currentTimestamp = 'now' expectedDumpResult = '1234567890/n' jdict = util.DotDict({ 'ProductName': 'FireFloozy', 'Version': '3.6', 'legacy_processing': 1 }) d = util.DotDict() j = util.DotDict() d.hbaseHost = 'fred' d.hbasePort = 'ethel' d.hbaseTimeout = 9000 j.root = d.hbaseFallbackFS = '.' d.throttleConditions = [] j.maxDirectoryEntries = d.hbaseFallbackDumpDirCount = 1000000 j.jsonSuffix = d.jsonFileSuffix = '.json' j.dumpSuffix = d.dumpFileSuffix = '.dump' j.dumpGID = d.hbaseFallbackdumpGID = 666 j.dumpPermissions = d.hbaseFallbackDumpPermissions = 660 j.dirPermissions = d.hbaseFallbackDirPermissions = 770 d.logger = util.SilentFakeLogger() fakeHbaseConnection = exp.DummyObjectWithExpectations( 'fakeHbaseConnection') fakeHbaseConnection.expect('hbaseThriftExceptions', None, None, (), None) fakeHbaseConnection.expect('put_json_dump', ('uuid', jdict, expectedDumpResult), {"number_of_retries": 2}, None, None) fakeHbaseModule = exp.DummyObjectWithExpectations('fakeHbaseModule') fakeHbaseModule.expect('HBaseConnectionForCrashReports', (d.hbaseHost, d.hbasePort, d.hbaseTimeout), {"logger": d.logger}, fakeHbaseConnection, None) fakeHbaseModule.expect('NoConnectionException', None, None, hbc.NoConnectionException, None) fakeJsonDumpStore = exp.DummyObjectWithExpectations('fakeJsonDumpStore') fakeJsonDumpModule = exp.DummyObjectWithExpectations('fakeJsonDumpModule') fakeJsonDumpModule.expect('JsonDumpStorage', (), j, fakeJsonDumpStore, None) css = cstore.CrashStorageSystemForHBase(d, configPrefix='', hbaseClient=fakeHbaseModule, jsonDumpStorage=fakeJsonDumpModule) expectedResult = cstore.CrashStorageSystem.OK result = css.save_raw('uuid', jdict, expectedDumpResult, currentTimestamp) assert result == expectedResult, 'expected %s but got %s' % ( expectedResult, result)
def test_get_dump(): dumpBlob = 'abcdefghijklmnopqrstuvwxyz01234567890!@#$%^&*()_-=+' dummyColumn = exp.DummyObjectWithExpectations() dummyColumn.expect('value', None, None, dumpBlob) dummyClientRowObject = exp.DummyObjectWithExpectations() dummyClientRowObject.expect('columns', None, None, {'raw_data:dump':dummyColumn}) getRowWithColumnsReturnValue = [dummyClientRowObject] hbcfcr = HBaseConnectionForCrashReportsWithPresetExpectations() conn = hbcfcr.conn dummy_clientObject = hbcfcr.dummy_clientObject dummy_clientObject.expect('getRowWithColumns', ('crash_reports', 'a100102abcdefghijklmnopqrstuvwxyz100102', ['raw_data:dump']), {}, getRowWithColumnsReturnValue) result = conn.get_dump('abcdefghijklmnopqrstuvwxyz100102') assert result == dumpBlob, 'expected %s, but got %s' % (dumpBlob, str(result))
def test_get_json_meta_as_string(): jsonDataAsString = '{"a": 1, "b": "hello"}' dummyColumn = exp.DummyObjectWithExpectations() dummyColumn.expect('value', None, None, jsonDataAsString) dummyClientRowObject = exp.DummyObjectWithExpectations() dummyClientRowObject.expect('columns', None, None, {'meta_data:json':dummyColumn}) dummyClientRowObject.expect('row', None, None, 'a100102abcdefghijklmnopqrstuvwxyz100102') getRowWithColumnsReturnValue = [dummyClientRowObject] hbcfcr = HBaseConnectionForCrashReportsWithPresetExpectations() conn = hbcfcr.conn dummy_clientObject = hbcfcr.dummy_clientObject dummy_clientObject.expect('getRowWithColumns', ('crash_reports', 'a100102abcdefghijklmnopqrstuvwxyz100102', ['meta_data:json']), {}, getRowWithColumnsReturnValue) result = conn.get_json_meta_as_string('abcdefghijklmnopqrstuvwxyz100102') assert result == jsonDataAsString, 'expected %s, but got %s' % (jsonDataAsString, str(result))
def testSaveCampaign(): context = getDummyContext() product = 'Foobar' versions = '5' signature = 'JohnHancock' subject = 'email subject' body = 'email body' start_date = utc_now() end_date = start_date + timedelta(hours=1) author = '*****@*****.**' email_count = 0 parameters = (product, versions, signature, subject, body, start_date, end_date, email_count, author) sql = """INSERT INTO email_campaigns (product, versions, signature, subject, body, start_date, end_date, email_count, author) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id""" dummyCursor = expect.DummyObjectWithExpectations() dummyCursor.expect('mogrify', (sql, list(parameters)), {}, None) dummyCursor.expect('execute', (sql, list(parameters)), {}, None) dummyCursor.expect('fetchone', (), {}, ['123']) campaign = ecc.EmailCampaignCreate(context) campaignId = campaign.save_campaign(dummyCursor, product, versions, signature, subject, body, start_date, end_date, author) assert campaignId == '123'
def test_make_row_nice_3(): """indirect test by invoking a base class method that exercises HBaseConnectionForCrashReports._make_row_nice""" listOfRows = [] expectedListOfRows = [] for x in range(3): dummy_client_row_object = exp.DummyObjectWithExpectations( 'dummy_client_row_object') d = { 'a': ValueObject(x), 'b': ValueObject(x * 10.0), 'c': ValueObject('C' * x) } dummy_client_row_object.expect('columns', None, None, d) ooid = '%s100102' % (str(x) * 26) expectedOoid = '%s100102' % (str(x) * 26) dummy_client_row_object.expect('row', None, None, ooid) expectedDict = { 'a': x, 'b': x * 10.0, 'c': 'C' * x, '_rowkey': expectedOoid } listOfRows.append(dummy_client_row_object) expectedListOfRows.append(expectedDict) hbcfcr = HBaseConnectionForCrashReportsWithPresetExpectations() conn = hbcfcr.conn result = conn._make_rows_nice(listOfRows) for a, b in zip(expectedListOfRows, result): assert a == b, 'expected %s, but got %s' % (str(a), str(b))
def testCrashStorageSystemForHBase_save_4(): """hbase fails with a NoConnectionException but there is no filesystem fallback - expecting retry return""" currentTimestamp = 'now' expectedDumpResult = '1234567890/n' jdict = {'a': 2, 'b': 'hello'} d = util.DotDict() d.hbaseHost = 'fred' d.hbasePort = 'ethel' d.hbaseTimeout = 9000 d.hbaseFallbackFS = '' d.throttleConditions = [] d.hbaseFallbackDumpDirCount = 1000000 d.jsonFileSuffix = '.json' d.dumpFileSuffix = '.dump' d.hbaseFallbackDumpGID = 666 d.hbaseFallbackDumpPermissions = 660 d.hbaseFallbackDirPermissions = 770 d.logger = util.SilentFakeLogger() fakeHbaseConnection = exp.DummyObjectWithExpectations( 'fakeHbaseConnection') fakeHbaseConnection.expect('hbaseThriftExceptions', None, None, (), None) fakeHbaseConnection.expect('put_json_dump', ('uuid', jdict, expectedDumpResult), {"number_of_retries": 2}, None, hbc.NoConnectionException(Exception())) fakeHbaseModule = exp.DummyObjectWithExpectations('fakeHbaseModule') fakeHbaseModule.expect('HBaseConnectionForCrashReports', (d.hbaseHost, d.hbasePort, d.hbaseTimeout), {"logger": d.logger}, fakeHbaseConnection, None) fakeHbaseModule.expect('NoConnectionException', None, None, hbc.NoConnectionException, None) fakeJsonDumpModule = exp.DummyObjectWithExpectations('fakeJsonDumpModule') cstore.logger = loggerForTest.TestingLogger() css = cstore.CrashStorageSystemForHBase(d, '', fakeHbaseModule, fakeJsonDumpModule) expectedResult = cstore.CrashStorageSystem.RETRY print css.exceptionsEligibleForRetry result = css.save_raw('uuid', jdict, expectedDumpResult, currentTimestamp) assert result == expectedResult, 'expected %s but got %s' % ( expectedResult, result)
def test_make_row_nice_1(): conn = HBaseConnectionWithPresetExpectations().conn dummy_client_row_object = exp.DummyObjectWithExpectations('dummy_client_row_object') d = {'a':ValueObject(1), 'b':ValueObject(2.1), 'c':ValueObject('C')} dummy_client_row_object.expect('columns', None, None, d) expectedDict = {'a':1, 'b':2.1, 'c':'C'} result = conn._make_row_nice(dummy_client_row_object) assert result == expectedDict, "expected %s but got %s" % (str(expectedDict), str(result))
def getHbaseStorage1(config): h = exp.DummyObjectWithExpectations() h.expect('__call__', (config, ), {}, h) # return self h.expect('save_raw', ('1', 'one', 'eins'), {}, cstore.CrashStorageSystem.OK) h.expect('save_raw', ('2', 'two', 'zwei'), {}, cstore.CrashStorageSystem.RETRY) h.expect('save_raw', ('2', 'two', 'zwei'), {}, cstore.CrashStorageSystem.OK) return h
def test_getLinks(self): self.config.products = ('PRODUCT1', 'PRODUCT2') self.config.base_url = 'http://www.example.com/' fake_response_url = "%s%s" % (self.config.base_url, self.config.products[0]) fake_response_contents = """ blahblahblahblahblah <a href="product1-v1.en-US.p1.txt">product1-v1.en-US.p1.txt</a> <a href="product1-v1.en-US.p1.zip">product1-v1.en-US.p1.zip</a> <a href="product2-v2.en-US.p2.txt">product2-v2.en-US.p2.txt</a> <a href="product2-v2.en-US.p2.zip">product2-v2.en-US.p2.zip</a> blahblahblahblahblah """ fakeResponse = exp.DummyObjectWithExpectations() fakeResponse.code = 200 fakeResponse.expect('read', (), {}, fake_response_contents) fakeResponse.expect('close', (), {}) fakeUrllib2 = exp.DummyObjectWithExpectations() fakeUrllib2.expect('urlopen', (fake_response_url, ), {}, fakeResponse) actual = ftpscraper.getLinks('http://www.example.com/PRODUCT1', startswith='product1', urllib=fakeUrllib2) expected = ['product1-v1.en-US.p1.txt', 'product1-v1.en-US.p1.zip'] assert actual == expected, "expected %s, got %s" % (expected, actual) fakeResponse = exp.DummyObjectWithExpectations() fakeResponse.code = 200 fakeResponse.expect('read', (), {}, fake_response_contents) fakeResponse.expect('close', (), {}) fakeUrllib2 = exp.DummyObjectWithExpectations() fakeUrllib2.expect('urlopen', (fake_response_url, ), {}, fakeResponse) expected = ['product1-v1.en-US.p1.zip', 'product2-v2.en-US.p2.zip'] actual = ftpscraper.getLinks('http://www.example.com/PRODUCT1', endswith='.zip', urllib=fakeUrllib2) assert actual == expected, "expected %s, got %s" % (expected, actual)
def test_parseInfoFile(self): self.config.products = ('PRODUCT1', 'PRODUCT2') self.config.base_url = 'http://www.example.com/' fake_response_url = "%s%s" % (self.config.base_url, self.config.products[0]) fake_response_contents = """ 20111011042016 http://hg.mozilla.org/releases/mozilla-aurora/rev/327f5fdae663 """ fakeResponse = exp.DummyObjectWithExpectations() fakeResponse.code = 200 fakeResponse.expect('read', (), {}, fake_response_contents) fakeResponse.expect('close', (), {}) fakeUrllib2 = exp.DummyObjectWithExpectations() fakeUrllib2.expect('urlopen', (fake_response_url, ), {}, fakeResponse) rev = 'http://hg.mozilla.org/releases/mozilla-aurora/rev/327f5fdae663' expected = {'buildID': '20111011042016', 'rev': rev} actual = ftpscraper.parseInfoFile('http://www.example.com/PRODUCT1', nightly=True, urllib=fakeUrllib2) assert actual == expected, "expected %s, got %s" % (expected, actual) fake_response_contents = """ buildID=20110705195857 """ fakeResponse = exp.DummyObjectWithExpectations() fakeResponse.code = 200 fakeResponse.expect('read', (), {}, fake_response_contents) fakeResponse.expect('close', (), {}) fakeUrllib2 = exp.DummyObjectWithExpectations() fakeUrllib2.expect('urlopen', (fake_response_url, ), {}, fakeResponse) expected = {'buildID': '20110705195857'} actual = ftpscraper.parseInfoFile('http://www.example.com/PRODUCT1', nightly=False, urllib=fakeUrllib2) assert actual == expected, "expected %s, got %s" % (expected, actual)
def test_make_row_nice_2(): conn = HBaseConnectionWithPresetExpectations().conn dummy_client_row_object = exp.DummyObjectWithExpectations('dummy_client_row_object') d = {'a':ValueObject(1), 'b':ValueObject(2.1), 'c':'C'} dummy_client_row_object.expect('columns', None, None, d) expectedDict = {'a':1, 'b':2.1, 'c':'C'} try: result = conn._make_row_nice(dummy_client_row_object) except Exception, x: expected_exception_string = "'str' object has no attribute 'value'" actual_exception_string = str(x)
def setup_mocked_register(register_class): conf = sutil.DotDict() conf.processorCheckInTime = dt.timedelta(0, 300) conf.processorCheckInFrequency = dt.timedelta(0, 300) conf.processorId = 17 fake_logger = exp.DummyObjectWithExpectations() conf.logger = fake_logger threshold = now_func() + conf.processorCheckInTime os_module = exp.DummyObjectWithExpectations() sdb_module = exp.DummyObjectWithExpectations() db_conn = exp.DummyObjectWithExpectations() db_cur = exp.DummyObjectWithExpectations() db_pool = exp.DummyObjectWithExpectations() fake_logger.expect('info', ('connecting to database', ), {}) db_pool.expect('connectionCursorPair', (), {}, (db_conn, db_cur)) os_module.expect('uname', (), {}, ['a', 'b', 'c']) os_module.expect('getpid', (), {}, 1111) sdb_module.expect('singleValueSql', (db_cur, register_class.NOW_SQL, (conf.processorCheckInTime, )), {}, threshold) fake_logger.expect('info', ("registering with 'processors' table", ), {}) db_conn.expect('commit', (), {}) return register_class(conf, db_pool, now_func, os_module, sdb_module)
def test_assume_any_identity_1(): conf = sutil.DotDict() fake_logger = exp.DummyObjectWithExpectations() conf.logger = fake_logger conf.processorCheckInTime = dt.timedelta(0, 300) threshold = now_func() + conf.processorCheckInTime os_module = exp.DummyObjectWithExpectations() sdb_module = exp.DummyObjectWithExpectations() db_conn = exp.DummyObjectWithExpectations() db_cur = exp.DummyObjectWithExpectations() db_pool = exp.DummyObjectWithExpectations() hostname = 'fred' class MyRegister(reg.ProcessorRegistrationAgent): def __init__(self, config, db_conn_source, now_func, os, sdb): super(MyRegister, self).__init__(config, db_conn_source, now_func, os, sdb) def take_over_dead_processor(self, cursor, proc_id): expected_assert(db_cur, cursor) expected_assert(proc_id, 17) def registration(self): pass fake_logger.expect('debug', ('looking for any dead processor', ), {}) sql = ("select id from processors" " where lastseendatetime < %s limit 1") sdb_module.expect('singleValueSql', (db_cur, sql, (threshold, )), {}, 17) fake_logger.expect('info', ('will step in for processor %d', 17), {}) r = MyRegister(conf, db_pool, now_func, os_module, sdb_module) id = r.assume_any_identity(db_cur, threshold, hostname, 17) expected_assert(17, id)
def testUpdateCampaign(): context = getDummyContext() campaign_id = 123 email_count = 321 sql = """UPDATE email_campaigns SET email_count = %s WHERE id = %s""" parameters = (email_count, campaign_id) dummyCursor = expect.DummyObjectWithExpectations() dummyCursor.expect('execute', (sql, parameters), {}, None) sender = es.EmailSender(context)
def test_make_rows_nice_1(): listOfRows = [] expectedListOfRows = [] for x in range(3): dummy_client_row_object = exp.DummyObjectWithExpectations('dummy_client_row_object') d = {'a':ValueObject(x), 'b':ValueObject(x * 10.0), 'c':ValueObject('C'*x)} dummy_client_row_object.expect('columns', None, None, d) expectedDict = {'a':x, 'b':x * 10.0, 'c':'C'*x} listOfRows.append(dummy_client_row_object) expectedListOfRows.append(expectedDict) conn = HBaseConnectionWithPresetExpectations().conn result = conn._make_rows_nice(listOfRows) for a, b in zip(result, expectedListOfRows): assert a == b, 'expected %s, but got %s' % (str(a), str(b))
def test_checkin(): def now_func(): return dt.datetime(2011, 1, 1, 0, 6, 0, tzinfo=UTC) conf = sutil.DotDict() conf.processorCheckInTime = dt.timedelta(0, 300) conf.processorCheckInFrequency = dt.timedelta(0, 300) conf.processorId = 17 fake_logger = exp.DummyObjectWithExpectations() conf.logger = fake_logger threshold = now_func() + conf.processorCheckInTime os_module = exp.DummyObjectWithExpectations() sdb_module = exp.DummyObjectWithExpectations() db_conn = exp.DummyObjectWithExpectations() db_cur = exp.DummyObjectWithExpectations() db_pool = exp.DummyObjectWithExpectations() fake_logger.expect('info', ('connecting to database', ), {}) db_pool.expect('connectionCursorPair', (), {}, (db_conn, db_cur)) os_module.expect('uname', (), {}, ['a', 'b', 'c']) os_module.expect('getpid', (), {}, 1111) sdb_module.expect('singleValueSql', (db_cur, reg.ProcessorRegistrationAgent.NOW_SQL, (conf.processorCheckInTime, )), {}, threshold) fake_logger.expect('info', ("registering with 'processors' table", ), {}) db_conn.expect('commit', (), {}) fake_logger.expect('debug', ("updating 'processor' table registration", ), {}) db_pool.expect('connectionCursorPair', (), {}, (db_conn, db_cur)) db_cur.expect('execute', ("update processors set lastseendatetime = %s " "where id = %s", (now_func(), 17)), {}) db_conn.expect('commit', (), {}) r = MockedRegister(conf, db_pool, now_func, os_module, sdb_module) r.checkin() expected_assert(now_func(), r.last_checkin_ts) r.last_checkin_ts = dt.datetime(2011, 1, 1, 0, 6, 0, tzinfo=UTC) r.checkin() r.last_checkin_ts = dt.datetime(2011, 1, 1, 0, 5, 0, tzinfo=UTC) r.checkin() r.last_checkin_ts = dt.datetime(2011, 1, 1, 0, 4, 0, tzinfo=UTC) r.checkin() r.last_checkin_ts = dt.datetime(2011, 1, 1, 0, 3, 0, tzinfo=UTC) r.checkin() r.last_checkin_ts = dt.datetime(2011, 1, 1, 0, 2, 0, tzinfo=UTC) r.checkin() r.last_checkin_ts = dt.datetime(2011, 1, 1, 0, 1, 0, tzinfo=UTC) r.checkin() fake_logger.expect('debug', ("updating 'processor' table registration", ), {}) db_pool.expect('connectionCursorPair', (), {}, (db_conn, db_cur)) db_cur.expect('execute', ("update processors set lastseendatetime = %s " "where id = %s", (now_func(), 17)), {}) db_conn.expect('commit', (), {}) r.last_checkin_ts = dt.datetime(2011, 1, 1, 0, 0, 0, tzinfo=UTC) r.checkin() expected_assert(now_func(), r.last_checkin_ts)
def testEnsureContacts(): context = getDummyContext() parameters = [('*****@*****.**', 'd64298ce-6217-4a97-917b-7c18d3f67e18', 'abcdefg', '2011-09-01 00:00')] sql = """INSERT INTO email_contacts (email, subscribe_token) VALUES (%s, %s) RETURNING id""" sql = """INSERT INTO email_contacts (email, subscribe_token, ooid, crash_date) VALUES (%s, %s, %s, %s) RETURNING id""" dummyCursor = expect.DummyObjectWithExpectations() dummyCursor.expect('executemany', (sql, parameters), {}, None) # with dbID already set campaign = ecc.EmailCampaignCreate(context) email_rows = [('1234', '*****@*****.**', '2011-09-01 00:00', 'abcdefg', 'hijklmn')] full_email_rows = campaign.ensure_contacts(dummyCursor, email_rows) assert full_email_rows == [{'token': 'hijklmn', 'crash_date': '2011-09-01 00:00', 'id': '1234', 'ooid': 'abcdefg', 'email': '*****@*****.**'}]
def testDetermineEmails(): context = getDummyContext() product = 'Foobar' versions = '5' signature = 'JohnHancock' start_date = utc_now() end_date = start_date + timedelta(hours=1) parameters = { 'product': product, 'versions': versions, 'signature': signature, } version_clause = '' if len(versions) > 0: version_clause = " version IN %(versions)s AND " sql = """ SELECT DISTINCT contacts.id, reports.email, reports.client_crash_date AS crash_date, reports.uuid AS ooid, contacts.subscribe_token FROM reports LEFT JOIN email_contacts AS contacts ON reports.email = contacts.email WHERE TIMESTAMP WITH TIME ZONE '%s' <= reports.date_processed AND TIMESTAMP WITH TIME ZONE '%s' > reports.date_processed AND reports.product = %%(product)s AND %s reports.signature = %%(signature)s AND LENGTH(reports.email) > 4 AND contacts.subscribe_status IS NOT FALSE AND contacts.email NOT IN ( SELECT contacted.email FROM email_campaigns AS prev_campaigns JOIN email_campaigns_contacts ON email_campaigns_contacts.email_campaigns_id = prev_campaigns.id JOIN email_contacts AS contacted ON email_campaigns_contacts.email_contacts_id = contacted.id WHERE prev_campaigns.product = %%(product)s AND prev_campaigns.signature = %%(signature)s ) """ % (start_date, end_date, version_clause) dummyCursor = expect.DummyObjectWithExpectations() dummyCursor.expect('mogrify', (sql, parameters), {}, None) dummyCursor.expect('execute', (sql, parameters), {}, None) dummyCursor.expect('fetchall', (), {}, []) campaign = ecc.EmailCampaignCreate(context) email_rows = campaign.determine_emails(dummyCursor, product, versions, signature, start_date, end_date)
def test_get_full_row(): listOfRows = [] expectedListOfRows = [] for x in range(3): dummy_client_row_object = exp.DummyObjectWithExpectations('dummy_client_row_object') d = {'a':ValueObject(x), 'b':ValueObject(x * 10.0), 'c':ValueObject('C'*x)} dummy_client_row_object.expect('columns', None, None, d) expectedDict = {'a':x, 'b':x * 10.0, 'c':'C'*x} listOfRows.append(dummy_client_row_object) expectedListOfRows.append(expectedDict) uhbc = HBaseConnectionWithPresetExpectations() conn = uhbc.conn dummy_clientObject = uhbc.dummy_clientObject dummy_clientObject.expect('getRow', ('fred', '22'), {}, listOfRows) result = conn.get_full_row('fred', '22') for a, b in zip(result, expectedListOfRows): assert a == b, 'expected %s, but got %s' % (str(a), str(b))
def testSaveCampaignContacts(): context = getDummyContext() campaign_id = 123 contacted_emails = ['*****@*****.**', '*****@*****.**'] sql = """ INSERT INTO email_campaigns_contacts (email_campaigns_id, email_contacts_id) SELECT %(campaign_id)s, email_contacts.id FROM email_contacts WHERE email IN %(emails)s """ parameters = {'campaign_id': campaign_id, 'emails': tuple(contacted_emails)} dummyCursor = expect.DummyObjectWithExpectations() dummyCursor.expect('execute', (sql, parameters), {}, None) sender = es.EmailSender(context)
def testSendAllEmails(): context = getDummyContext() testContacts = ['*****@*****.**', '*****@*****.**'] crash_date = string_to_datetime('2011-09-01') contacts = [ (0, testContacts[0], 'abc', 'ooid1', crash_date), (0, testContacts[1], 'abc', 'ooid2', crash_date) ] subject = 'email subject' body = 'email body' dummySmtp = expect.DummyObjectWithExpectations() # no variables noVarBody = 'Content-Type: text/plain; charset="utf-8"\nMIME-Version: 1.0\nContent-Transfer-Encoding: base64\nFrom: [email protected]\nSubject: email subject\nTo: %s\n\nZW1haWwgYm9keQ==\n' dummySmtp.expect('sendmail', (context.fromEmailAddress, [testContacts[0]], noVarBody % testContacts[0]), {}, None) dummySmtp.expect('sendmail', (context.fromEmailAddress, [testContacts[1]], noVarBody % testContacts[1]), {}, None) sender = es.EmailSender(context) contacted_emails = sender.send_all_emails(contacts, subject, body, dummySmtp) assert contacted_emails == {0: 'sent'} # FIXME # # unsubscribe variable # unsubVarBody1 = 'Content-Type: text/plain; charset="utf-8"\nMIME-Version: 1.0\nContent-Transfer-Encoding: base64\nFrom: [email protected]\nSubject: email subject\nTo: %s\n\nZW1haWwgYm9keSBodHRwOi8vZXhhbXBsZS5jb20vdW5zdWJzY3JpYmUvYWJj\n' # unsubVarBody2 = 'Content-Type: text/plain; charset="utf-8"\nMIME-Version: 1.0\nContent-Transfer-Encoding: base64\nFrom: [email protected]\nSubject: email subject\nTo: %s\n\nZW1haWwgYm9keSBodHRwOi8vZXhhbXBsZS5jb20vdW5zdWJzY3JpYmUvZGVm\n' # dummySmtp.expect('sendmail', (context.fromEmailAddress, [testContacts[0]], unsubVarBody1 % testContacts[0]), {}, None) # dummySmtp.expect('sendmail', (context.fromEmailAddress, [testContacts[1]], unsubVarBody2 % testContacts[1]), {}, None) # # body = 'email body *|UNSUBSCRIBE_URL|*' # contacted_emails = sender.send_all_emails(contacts, subject, body, dummySmtp) # print contacted_emails #assert contacted_emails == [testContacts[0], testContacts[1]] # email_address variable emailVarBody1 = 'Content-Type: text/plain; charset="utf-8"\nMIME-Version: 1.0\nContent-Transfer-Encoding: base64\nFrom: [email protected]\nSubject: email subject\nTo: %s\n\nZW1haWwgYm9keSAxQGV4YW1wbGUuY29t\n' emailVarBody2 = 'Content-Type: text/plain; charset="utf-8"\nMIME-Version: 1.0\nContent-Transfer-Encoding: base64\nFrom: [email protected]\nSubject: email subject\nTo: %s\n\nZW1haWwgYm9keSAyQGV4YW1wbGUuY29t\n' dummySmtp.expect('sendmail', (context.fromEmailAddress, [testContacts[0]], emailVarBody1 % testContacts[0]), {}, None) dummySmtp.expect('sendmail', (context.fromEmailAddress, [testContacts[1]], emailVarBody2 % testContacts[1]), {}, None) body = 'email body *|EMAIL_ADDRESS|*' contacted_emails = sender.send_all_emails(contacts, subject, body, dummySmtp) assert contacted_emails == {0: 'sent'}
def testCrashStorageSystem_makeJsonDictFromForm(): d = util.DotDict() d.dumpField = 'd' fakeValue = util.DotDict() fakeValue.value = 2 f = util.DotDict() f.a = '1' f.b = fakeValue f.c = '3' f.d = '4' f.e = '5' expectedTime = '12:00:01' fakeTimeModule = exp.DummyObjectWithExpectations('fakeTimeModule') fakeTimeModule.expect('time', (), {}, expectedTime, None) css = cstore.CrashStorageSystem(d) resultJson = css.makeJsonDictFromForm(f, fakeTimeModule) assert resultJson.a == '1' assert resultJson.b == 2 assert resultJson.c == '3' assert resultJson.e == '5'
def test_assume_identity_by_host_3(): conf = sutil.DotDict() fake_logger = exp.DummyObjectWithExpectations() conf.logger = fake_logger conf.processorCheckInTime = dt.timedelta(0, 300) threshold = now_func() + conf.processorCheckInTime os_module = exp.DummyObjectWithExpectations() sdb_module = exp.DummyObjectWithExpectations() db_conn = exp.DummyObjectWithExpectations() db_cur = exp.DummyObjectWithExpectations() db_pool = exp.DummyObjectWithExpectations() hostname = 'fred' class MyRegister(reg.ProcessorRegistrationAgent): def __init__(self, config, db_conn_source, now_func, os, sdb): super(MyRegister, self).__init__(config, db_conn_source, now_func, os, sdb) def take_over_dead_processor(self, cursor, proc_id): expected_assert(db_cur, cursor) expected_assert(proc_id, 17) def assume_new_identity(self, cursor, thresh, host, proc_id): expected_assert(db_cur, cursor) expected_assert(threshold, thresh) expected_assert(hostname, host) expected_assert(proc_id, 17) return proc_id def registration(self): pass fake_logger.expect('debug', ('looking for a dead processor for host %s', 'fred'), {}) sql = ("select id from processors" " where lastseendatetime < %s" " and name like %s limit 1") sdb_module.expect('singleValueSql', (db_cur, sql, (threshold, hostname + '%')), {}, None, sdb.SQLDidNotReturnSingleValue) fake_logger.expect('debug', ("no dead processor found for host, %s", hostname), {}) sql2 = "select id from processors where name like 'fred%'" sdb_module.expect('singleValueSql', (db_cur, sql2), {}, None, sdb.SQLDidNotReturnSingleValue) r = MyRegister(conf, db_pool, now_func, os_module, sdb_module) id = r.assume_identity_by_host(db_cur, threshold, hostname, 17) expected_assert(17, id)
def test_assume_specific_identity_2(): conf = sutil.DotDict() fake_logger = exp.DummyObjectWithExpectations() conf.logger = fake_logger conf.processorCheckInTime = dt.timedelta(0, 300) threshold = now_func() + conf.processorCheckInTime os_module = exp.DummyObjectWithExpectations() sdb_module = exp.DummyObjectWithExpectations() db_conn = exp.DummyObjectWithExpectations() db_cur = exp.DummyObjectWithExpectations() db_pool = exp.DummyObjectWithExpectations() hostname = 'fred' class MyRegister(reg.ProcessorRegistrationAgent): def __init__(self, config, db_conn_source, now_func, os, sdb): super(MyRegister, self).__init__(config, db_conn_source, now_func, os, sdb) def take_over_dead_processor(self, cursor, proc_id): expected_assert(db_cur, cursor) expected_assert(proc_id, 17) def registration(self): pass fake_logger.expect('debug', ('looking for a specific dead processor', ), {}) sql = ("select id from processors " "where lastSeenDateTime < %s " "and id = %s") sdb_module.expect('singleValueSql', (db_cur, sql, (threshold, 17)), {}, None, sdb.SQLDidNotReturnSingleValue) #fake_logger.expect('info', #('stepping in for processor %d', 17), {}) r = MyRegister(conf, db_pool, now_func, os_module, sdb_module) try: id = r.assume_specific_identity(db_cur, threshold, hostname, 17) except reg.RegistrationError: assert True except Exception, x: assert False, "%s exception was not expected" % str(x)
def test_assume_new_identity(): conf = sutil.DotDict() fake_logger = exp.DummyObjectWithExpectations() conf.logger = fake_logger conf.processorCheckInTime = dt.timedelta(0, 300) threshold = now_func() + conf.processorCheckInTime os_module = exp.DummyObjectWithExpectations() sdb_module = exp.DummyObjectWithExpectations() db_conn = exp.DummyObjectWithExpectations() db_cur = exp.DummyObjectWithExpectations() db_pool = exp.DummyObjectWithExpectations() hostname = 'fred' class MyRegister(reg.ProcessorRegistrationAgent): def __init__(self, config, db_conn_source, now_func, os, sdb): super(MyRegister, self).__init__(config, db_conn_source, now_func, os, sdb) def registration(self): self.processor_name = 'fred' fake_logger.expect('debug', ('becoming a new processor', ), {}) sql = ("insert into processors" " (id," " name," " startdatetime," " lastseendatetime) " "values" " (default," " %s," " now()," " now()) " "returning id") sdb_module.expect('singleValueSql', (db_cur, sql, ('fred', )), {}, 17) r = MyRegister(conf, db_pool, now_func, os_module, sdb_module) id = r.assume_new_identity(db_cur, threshold, hostname, 17) expected_assert(17, id)
def test_take_over_dead_processor(): conf = sutil.DotDict() fake_logger = exp.DummyObjectWithExpectations() conf.logger = fake_logger conf.processorCheckInTime = dt.timedelta(0, 300) threshold = now_func() + conf.processorCheckInTime os_module = exp.DummyObjectWithExpectations() sdb_module = exp.DummyObjectWithExpectations() db_conn = exp.DummyObjectWithExpectations() db_cur = exp.DummyObjectWithExpectations() db_pool = exp.DummyObjectWithExpectations() hostname = 'fred' class MyRegister(reg.ProcessorRegistrationAgent): def __init__(self, config, db_conn_source, now_func, os, sdb): super(MyRegister, self).__init__(config, db_conn_source, now_func, os, sdb) def registration(self): self.processor_name = 'fred' fake_logger.expect('debug', ('taking over a dead processor', ), {}) sql = ("update processors set name = %s, " "startdatetime = now(), lastseendatetime = now()" " where id = %s") db_cur.expect('execute', (sql, ('fred', 17)), {}, 17) sql2 = ("update jobs set" " starteddatetime = NULL," " completeddatetime = NULL," " success = NULL " "where" " owner = %s") db_cur.expect('execute', (sql2, (17, )), {}, 17) r = MyRegister(conf, db_pool, now_func, os_module, sdb_module) id = r.take_over_dead_processor(db_cur, 17) assert id is None, "expected None but got %s" % id