def test_get_dumps():
dumpBlob = 'abcdefghijklmnopqrstuvwxyz01234567890!@#$%^&*()_-=+'
dumpBlob2 = 'abcdefghijklmnopqr$$uvwxyz01234567890!@#$%^&*()_-=+'
dumpBlob3 = 'abcdefghijklmnopqrstuvw%%z01234567890!@#$%^&*()_-=+'
dummyColumn = exp.DummyObjectWithExpectations()
dummyColumn.expect('value', None, None, dumpBlob)
dummyColumn2 = exp.DummyObjectWithExpectations()
dummyColumn2.expect('value', None, None, dumpBlob2)
dummyColumn3 = exp.DummyObjectWithExpectations()
dummyColumn3.expect('value', None, None, dumpBlob3)
dummyClientRowObject = exp.DummyObjectWithExpectations()
dummyClientRowObject.expect('columns', None, None, {'raw_data:dump':dummyColumn,
'raw_data:flash1':dummyColumn2,
'raw_data:flash2':dummyColumn3,
})
getRowWithColumnsReturnValue = [dummyClientRowObject]
hbcfcr = HBaseConnectionForCrashReportsWithPresetExpectations()
conn = hbcfcr.conn
dummy_clientObject = hbcfcr.dummy_clientObject
dummy_clientObject.expect('getRowWithColumns',
('crash_reports', 'a100102abcdefghijklmnopqrstuvwxyz100102', ['raw_data']),
{}, getRowWithColumnsReturnValue)
result = conn.get_dumps('abcdefghijklmnopqrstuvwxyz100102')
assert result['upload_file_minidump'] == dumpBlob, 'expected %s, but got %s' % (dumpBlob, str(result['upload_file_minidump']))
assert result['flash1'] == dumpBlob2, 'expected %s, but got %s' % (dumpBlob2, str(result['flash1']))
assert result['flash2'] == dumpBlob3, 'expected %s, but got %s' % (dumpBlob3, str(result['flash2']))
def testCrashStorageSystemForHBase___init__():
d = util.DotDict()
j = util.DotDict()
d.hbaseHost = 'fred'
d.hbasePort = 'ethel'
d.hbaseTimeout = 9000
j.root = d.hbaseFallbackFS = '.'
d.throttleConditions = []
j.maxDirectoryEntries = d.hbaseFallbackDumpDirCount = 1000000
j.jsonSuffix = d.jsonFileSuffix = '.json'
j.dumpSuffix = d.dumpFileSuffix = '.dump'
j.dumpGID = d.hbaseFallbackdumpGID = 666
j.dumpPermissions = d.hbaseFallbackDumpPermissions = 660
j.dirPermissions = d.hbaseFallbackDirPermissions = 770
j.logger = d.logger = util.SilentFakeLogger()
fakeHbaseConnection = exp.DummyObjectWithExpectations(
'fakeHbaseConnection')
fakeHbaseModule = exp.DummyObjectWithExpectations('fakeHbaseModule')
fakeHbaseModule.expect('HBaseConnectionForCrashReports',
(d.hbaseHost, d.hbasePort, d.hbaseTimeout),
{"logger": d.logger}, fakeHbaseConnection, None)
fakeHbaseConnection.expect('hbaseThriftExceptions', None, None, (), None)
fakeHbaseModule.expect('NoConnectionException', None, None,
hbc.NoConnectionException, None)
fakeJsonDumpStore = exp.DummyObjectWithExpectations('fakeJsonDumpStore')
fakeJsonDumpModule = exp.DummyObjectWithExpectations('fakeJsonDumpModule')
fakeJsonDumpModule.expect('JsonDumpStorage', (), j, fakeJsonDumpStore,
None)
css = cstore.CrashStorageSystemForHBase(d,
configPrefix='',
hbaseClient=fakeHbaseModule,
jsonDumpStorage=fakeJsonDumpModule)
assert css.hbaseConnection == fakeHbaseConnection
def test_get_json_2():
"""that ooid doesn't exist"""
jsonDataAsString = '{"a": 1, "b": "hello"}'
expectedJson = js.loads(jsonDataAsString)
dummyColumn = exp.DummyObjectWithExpectations()
dummyColumn.expect('value', None, None, jsonDataAsString)
dummyClientRowObject = exp.DummyObjectWithExpectations()
dummyClientRowObject.expect('columns', None, None,
{'meta_data:json': dummyColumn})
dummyClientRowObject.expect('row', None, None,
'a100102abcdefghijklmnopqrstuvwxyz100102')
hbcfcr = HBaseConnectionForCrashReportsWithPresetExpectations()
conn = hbcfcr.conn
dummy_clientObject = hbcfcr.dummy_clientObject
dummy_clientObject.expect(
'getRowWithColumns',
('crash_reports', 'a100102abcdefghijklmnopqrstuvwxyz100102',
['meta_data:json']), {}, [])
try:
result = conn.get_json('abcdefghijklmnopqrstuvwxyz100102')
except Exception, x:
expected_exception_as_string = 'OOID not found: abcdefghijklmnopqrstuvwxyz100102 - a100102abcdefghijklmnopqrstuvwxyz100102'
actual_exception_as_string = str(x)
assert expected_exception_as_string == actual_exception_as_string, 'expected %s, but got %s' % (
expected_exception_as_string, actual_exception_as_string)
def testCrashStorageForDualHbaseCrashStorageSystem01():
d = util.DotDict()
j = util.DotDict()
d.hbaseHost = 'fred'
d.secondaryHbaseHost = 'barney'
d.hbasePort = 'ethel'
d.secondaryHbasePort = 'betty'
d.hbaseTimeout = 3000
d.secondaryHbaseTimeout = 10000
j.root = d.hbaseFallbackFS = '.'
d.throttleConditions = []
j.maxDirectoryEntries = d.hbaseFallbackDumpDirCount = 1000000
j.jsonSuffix = d.jsonFileSuffix = '.json'
j.dumpSuffix = d.dumpFileSuffix = '.dump'
j.dumpGID = d.hbaseFallbackdumpGID = 666
j.dumpPermissions = d.hbaseFallbackDumpPermissions = 660
j.dirPermissions = d.hbaseFallbackDirPermissions = 770
j.logger = d.logger = util.SilentFakeLogger()
fakeHbaseConnection1 = exp.DummyObjectWithExpectations(
'fakeHbaseConnection1')
fakeHbaseConnection2 = exp.DummyObjectWithExpectations(
'fakeHbaseConnection2')
fakeHbaseConnection1.expect('hbaseThriftExceptions', None, None, (), None)
fakeHbaseConnection1.expect('get_json', ('fakeOoid1', ),
{'number_of_retries': 2}, 'fake_json1')
import socorro.storage.hbaseClient as hbc
fakeHbaseConnection1.expect('get_json', ('fakeOoid2', ),
{'number_of_retries': 2}, None,
hbc.OoidNotFoundException())
fakeHbaseConnection2.expect('hbaseThriftExceptions', None, None, (), None)
fakeHbaseConnection2.expect('get_json', ('fakeOoid2', ),
{'number_of_retries': 2}, 'fake_json2')
fakeHbaseModule = exp.DummyObjectWithExpectations('fakeHbaseModule')
fakeHbaseModule.expect('HBaseConnectionForCrashReports',
(d.hbaseHost, d.hbasePort, d.hbaseTimeout),
{"logger": d.logger}, fakeHbaseConnection1, None)
fakeHbaseModule.expect('NoConnectionException', None, None,
hbc.NoConnectionException, None)
fakeHbaseModule.expect(
'HBaseConnectionForCrashReports',
(d.secondaryHbaseHost, d.secondaryHbasePort, d.secondaryHbaseTimeout),
{"logger": d.logger}, fakeHbaseConnection2, None)
fakeHbaseModule.expect('NoConnectionException', None, None,
hbc.NoConnectionException, None)
fakeJsonDumpStore = exp.DummyObjectWithExpectations('fakeJsonDumpStore')
fakeJsonDumpModule = exp.DummyObjectWithExpectations('fakeJsonDumpModule')
fakeJsonDumpModule.expect('JsonDumpStorage', (), j, fakeJsonDumpStore,
None)
fakeJsonDumpModule.expect('JsonDumpStorage', (), j, fakeJsonDumpStore,
None)
css = cstore.DualHbaseCrashStorageSystem(
d, hbaseClient=fakeHbaseModule, jsonDumpStorage=fakeJsonDumpModule)
assert css.hbaseConnection == fakeHbaseConnection1
assert css.fallbackHBase.hbaseConnection == fakeHbaseConnection2
result = css.get_meta('fakeOoid1')
assert result == 'fake_json1'
result = css.get_meta('fakeOoid2')
assert result == 'fake_json2'
def testCrashStorageSystemForHBase_save_1():
"""straight save into hbase with no trouble"""
currentTimestamp = 'now'
expectedDumpResult = '1234567890/n'
jdict = util.DotDict({
'ProductName': 'FireFloozy',
'Version': '3.6',
'legacy_processing': 1
})
d = util.DotDict()
j = util.DotDict()
d.hbaseHost = 'fred'
d.hbasePort = 'ethel'
d.hbaseTimeout = 9000
j.root = d.hbaseFallbackFS = '.'
d.throttleConditions = []
j.maxDirectoryEntries = d.hbaseFallbackDumpDirCount = 1000000
j.jsonSuffix = d.jsonFileSuffix = '.json'
j.dumpSuffix = d.dumpFileSuffix = '.dump'
j.dumpGID = d.hbaseFallbackdumpGID = 666
j.dumpPermissions = d.hbaseFallbackDumpPermissions = 660
j.dirPermissions = d.hbaseFallbackDirPermissions = 770
d.logger = util.SilentFakeLogger()
fakeHbaseConnection = exp.DummyObjectWithExpectations(
'fakeHbaseConnection')
fakeHbaseConnection.expect('hbaseThriftExceptions', None, None, (), None)
fakeHbaseConnection.expect('put_json_dump',
('uuid', jdict, expectedDumpResult),
{"number_of_retries": 2}, None, None)
fakeHbaseModule = exp.DummyObjectWithExpectations('fakeHbaseModule')
fakeHbaseModule.expect('HBaseConnectionForCrashReports',
(d.hbaseHost, d.hbasePort, d.hbaseTimeout),
{"logger": d.logger}, fakeHbaseConnection, None)
fakeHbaseModule.expect('NoConnectionException', None, None,
hbc.NoConnectionException, None)
fakeJsonDumpStore = exp.DummyObjectWithExpectations('fakeJsonDumpStore')
fakeJsonDumpModule = exp.DummyObjectWithExpectations('fakeJsonDumpModule')
fakeJsonDumpModule.expect('JsonDumpStorage', (), j, fakeJsonDumpStore,
None)
css = cstore.CrashStorageSystemForHBase(d,
configPrefix='',
hbaseClient=fakeHbaseModule,
jsonDumpStorage=fakeJsonDumpModule)
expectedResult = cstore.CrashStorageSystem.OK
result = css.save_raw('uuid', jdict, expectedDumpResult, currentTimestamp)
assert result == expectedResult, 'expected %s but got %s' % (
expectedResult, result)
def test_get_dump():
dumpBlob = 'abcdefghijklmnopqrstuvwxyz01234567890!@#$%^&*()_-=+'
dummyColumn = exp.DummyObjectWithExpectations()
dummyColumn.expect('value', None, None, dumpBlob)
dummyClientRowObject = exp.DummyObjectWithExpectations()
dummyClientRowObject.expect('columns', None, None, {'raw_data:dump':dummyColumn})
getRowWithColumnsReturnValue = [dummyClientRowObject]
hbcfcr = HBaseConnectionForCrashReportsWithPresetExpectations()
conn = hbcfcr.conn
dummy_clientObject = hbcfcr.dummy_clientObject
dummy_clientObject.expect('getRowWithColumns',
('crash_reports', 'a100102abcdefghijklmnopqrstuvwxyz100102', ['raw_data:dump']),
{}, getRowWithColumnsReturnValue)
result = conn.get_dump('abcdefghijklmnopqrstuvwxyz100102')
assert result == dumpBlob, 'expected %s, but got %s' % (dumpBlob, str(result))
def test_get_json_meta_as_string():
jsonDataAsString = '{"a": 1, "b": "hello"}'
dummyColumn = exp.DummyObjectWithExpectations()
dummyColumn.expect('value', None, None, jsonDataAsString)
dummyClientRowObject = exp.DummyObjectWithExpectations()
dummyClientRowObject.expect('columns', None, None, {'meta_data:json':dummyColumn})
dummyClientRowObject.expect('row', None, None, 'a100102abcdefghijklmnopqrstuvwxyz100102')
getRowWithColumnsReturnValue = [dummyClientRowObject]
hbcfcr = HBaseConnectionForCrashReportsWithPresetExpectations()
conn = hbcfcr.conn
dummy_clientObject = hbcfcr.dummy_clientObject
dummy_clientObject.expect('getRowWithColumns',
('crash_reports', 'a100102abcdefghijklmnopqrstuvwxyz100102', ['meta_data:json']),
{}, getRowWithColumnsReturnValue)
result = conn.get_json_meta_as_string('abcdefghijklmnopqrstuvwxyz100102')
assert result == jsonDataAsString, 'expected %s, but got %s' % (jsonDataAsString, str(result))
def testSaveCampaign():
context = getDummyContext()
product = 'Foobar'
versions = '5'
signature = 'JohnHancock'
subject = 'email subject'
body = 'email body'
start_date = utc_now()
end_date = start_date + timedelta(hours=1)
author = '*****@*****.**'
email_count = 0
parameters = (product, versions, signature, subject, body, start_date, end_date, email_count, author)
sql = """INSERT INTO email_campaigns (product, versions, signature, subject, body, start_date, end_date, email_count, author)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id"""
dummyCursor = expect.DummyObjectWithExpectations()
dummyCursor.expect('mogrify', (sql, list(parameters)), {}, None)
dummyCursor.expect('execute', (sql, list(parameters)), {}, None)
dummyCursor.expect('fetchone', (), {}, ['123'])
campaign = ecc.EmailCampaignCreate(context)
campaignId = campaign.save_campaign(dummyCursor, product, versions, signature, subject, body, start_date, end_date, author)
assert campaignId == '123'
def test_make_row_nice_3():
"""indirect test by invoking a base class method that exercises HBaseConnectionForCrashReports._make_row_nice"""
listOfRows = []
expectedListOfRows = []
for x in range(3):
dummy_client_row_object = exp.DummyObjectWithExpectations(
'dummy_client_row_object')
d = {
'a': ValueObject(x),
'b': ValueObject(x * 10.0),
'c': ValueObject('C' * x)
}
dummy_client_row_object.expect('columns', None, None, d)
ooid = '%s100102' % (str(x) * 26)
expectedOoid = '%s100102' % (str(x) * 26)
dummy_client_row_object.expect('row', None, None, ooid)
expectedDict = {
'a': x,
'b': x * 10.0,
'c': 'C' * x,
'_rowkey': expectedOoid
}
listOfRows.append(dummy_client_row_object)
expectedListOfRows.append(expectedDict)
hbcfcr = HBaseConnectionForCrashReportsWithPresetExpectations()
conn = hbcfcr.conn
result = conn._make_rows_nice(listOfRows)
for a, b in zip(expectedListOfRows, result):
assert a == b, 'expected %s, but got %s' % (str(a), str(b))
def testCrashStorageSystemForHBase_save_4():
"""hbase fails with a NoConnectionException but there is no filesystem fallback - expecting retry return"""
currentTimestamp = 'now'
expectedDumpResult = '1234567890/n'
jdict = {'a': 2, 'b': 'hello'}
d = util.DotDict()
d.hbaseHost = 'fred'
d.hbasePort = 'ethel'
d.hbaseTimeout = 9000
d.hbaseFallbackFS = ''
d.throttleConditions = []
d.hbaseFallbackDumpDirCount = 1000000
d.jsonFileSuffix = '.json'
d.dumpFileSuffix = '.dump'
d.hbaseFallbackDumpGID = 666
d.hbaseFallbackDumpPermissions = 660
d.hbaseFallbackDirPermissions = 770
d.logger = util.SilentFakeLogger()
fakeHbaseConnection = exp.DummyObjectWithExpectations(
'fakeHbaseConnection')
fakeHbaseConnection.expect('hbaseThriftExceptions', None, None, (), None)
fakeHbaseConnection.expect('put_json_dump',
('uuid', jdict, expectedDumpResult),
{"number_of_retries": 2}, None,
hbc.NoConnectionException(Exception()))
fakeHbaseModule = exp.DummyObjectWithExpectations('fakeHbaseModule')
fakeHbaseModule.expect('HBaseConnectionForCrashReports',
(d.hbaseHost, d.hbasePort, d.hbaseTimeout),
{"logger": d.logger}, fakeHbaseConnection, None)
fakeHbaseModule.expect('NoConnectionException', None, None,
hbc.NoConnectionException, None)
fakeJsonDumpModule = exp.DummyObjectWithExpectations('fakeJsonDumpModule')
cstore.logger = loggerForTest.TestingLogger()
css = cstore.CrashStorageSystemForHBase(d, '', fakeHbaseModule,
fakeJsonDumpModule)
expectedResult = cstore.CrashStorageSystem.RETRY
print css.exceptionsEligibleForRetry
result = css.save_raw('uuid', jdict, expectedDumpResult, currentTimestamp)
assert result == expectedResult, 'expected %s but got %s' % (
expectedResult, result)
def test_make_row_nice_1():
conn = HBaseConnectionWithPresetExpectations().conn
dummy_client_row_object = exp.DummyObjectWithExpectations('dummy_client_row_object')
d = {'a':ValueObject(1), 'b':ValueObject(2.1), 'c':ValueObject('C')}
dummy_client_row_object.expect('columns', None, None, d)
expectedDict = {'a':1, 'b':2.1, 'c':'C'}
result = conn._make_row_nice(dummy_client_row_object)
assert result == expectedDict, "expected %s but got %s" % (str(expectedDict), str(result))
def getHbaseStorage1(config):
h = exp.DummyObjectWithExpectations()
h.expect('__call__', (config, ), {}, h) # return self
h.expect('save_raw', ('1', 'one', 'eins'), {},
cstore.CrashStorageSystem.OK)
h.expect('save_raw', ('2', 'two', 'zwei'), {},
cstore.CrashStorageSystem.RETRY)
h.expect('save_raw', ('2', 'two', 'zwei'), {},
cstore.CrashStorageSystem.OK)
return h
def test_getLinks(self):
self.config.products = ('PRODUCT1', 'PRODUCT2')
self.config.base_url = 'http://www.example.com/'
fake_response_url = "%s%s" % (self.config.base_url,
self.config.products[0])
fake_response_contents = """
blahblahblahblahblah
<a href="product1-v1.en-US.p1.txt">product1-v1.en-US.p1.txt</a>
<a href="product1-v1.en-US.p1.zip">product1-v1.en-US.p1.zip</a>
<a href="product2-v2.en-US.p2.txt">product2-v2.en-US.p2.txt</a>
<a href="product2-v2.en-US.p2.zip">product2-v2.en-US.p2.zip</a>
blahblahblahblahblah
"""
fakeResponse = exp.DummyObjectWithExpectations()
fakeResponse.code = 200
fakeResponse.expect('read', (), {}, fake_response_contents)
fakeResponse.expect('close', (), {})
fakeUrllib2 = exp.DummyObjectWithExpectations()
fakeUrllib2.expect('urlopen', (fake_response_url, ), {}, fakeResponse)
actual = ftpscraper.getLinks('http://www.example.com/PRODUCT1',
startswith='product1',
urllib=fakeUrllib2)
expected = ['product1-v1.en-US.p1.txt', 'product1-v1.en-US.p1.zip']
assert actual == expected, "expected %s, got %s" % (expected, actual)
fakeResponse = exp.DummyObjectWithExpectations()
fakeResponse.code = 200
fakeResponse.expect('read', (), {}, fake_response_contents)
fakeResponse.expect('close', (), {})
fakeUrllib2 = exp.DummyObjectWithExpectations()
fakeUrllib2.expect('urlopen', (fake_response_url, ), {}, fakeResponse)
expected = ['product1-v1.en-US.p1.zip', 'product2-v2.en-US.p2.zip']
actual = ftpscraper.getLinks('http://www.example.com/PRODUCT1',
endswith='.zip',
urllib=fakeUrllib2)
assert actual == expected, "expected %s, got %s" % (expected, actual)
def test_parseInfoFile(self):
self.config.products = ('PRODUCT1', 'PRODUCT2')
self.config.base_url = 'http://www.example.com/'
fake_response_url = "%s%s" % (self.config.base_url,
self.config.products[0])
fake_response_contents = """
20111011042016
http://hg.mozilla.org/releases/mozilla-aurora/rev/327f5fdae663
"""
fakeResponse = exp.DummyObjectWithExpectations()
fakeResponse.code = 200
fakeResponse.expect('read', (), {}, fake_response_contents)
fakeResponse.expect('close', (), {})
fakeUrllib2 = exp.DummyObjectWithExpectations()
fakeUrllib2.expect('urlopen', (fake_response_url, ), {}, fakeResponse)
rev = 'http://hg.mozilla.org/releases/mozilla-aurora/rev/327f5fdae663'
expected = {'buildID': '20111011042016', 'rev': rev}
actual = ftpscraper.parseInfoFile('http://www.example.com/PRODUCT1',
nightly=True,
urllib=fakeUrllib2)
assert actual == expected, "expected %s, got %s" % (expected, actual)
fake_response_contents = """
buildID=20110705195857
"""
fakeResponse = exp.DummyObjectWithExpectations()
fakeResponse.code = 200
fakeResponse.expect('read', (), {}, fake_response_contents)
fakeResponse.expect('close', (), {})
fakeUrllib2 = exp.DummyObjectWithExpectations()
fakeUrllib2.expect('urlopen', (fake_response_url, ), {}, fakeResponse)
expected = {'buildID': '20110705195857'}
actual = ftpscraper.parseInfoFile('http://www.example.com/PRODUCT1',
nightly=False,
urllib=fakeUrllib2)
assert actual == expected, "expected %s, got %s" % (expected, actual)
def test_make_row_nice_2():
conn = HBaseConnectionWithPresetExpectations().conn
dummy_client_row_object = exp.DummyObjectWithExpectations('dummy_client_row_object')
d = {'a':ValueObject(1), 'b':ValueObject(2.1), 'c':'C'}
dummy_client_row_object.expect('columns', None, None, d)
expectedDict = {'a':1, 'b':2.1, 'c':'C'}
try:
result = conn._make_row_nice(dummy_client_row_object)
except Exception, x:
expected_exception_string = "'str' object has no attribute 'value'"
actual_exception_string = str(x)
def setup_mocked_register(register_class):
conf = sutil.DotDict()
conf.processorCheckInTime = dt.timedelta(0, 300)
conf.processorCheckInFrequency = dt.timedelta(0, 300)
conf.processorId = 17
fake_logger = exp.DummyObjectWithExpectations()
conf.logger = fake_logger
threshold = now_func() + conf.processorCheckInTime
os_module = exp.DummyObjectWithExpectations()
sdb_module = exp.DummyObjectWithExpectations()
db_conn = exp.DummyObjectWithExpectations()
db_cur = exp.DummyObjectWithExpectations()
db_pool = exp.DummyObjectWithExpectations()
fake_logger.expect('info', ('connecting to database', ), {})
db_pool.expect('connectionCursorPair', (), {}, (db_conn, db_cur))
os_module.expect('uname', (), {}, ['a', 'b', 'c'])
os_module.expect('getpid', (), {}, 1111)
sdb_module.expect('singleValueSql', (db_cur, register_class.NOW_SQL,
(conf.processorCheckInTime, )), {},
threshold)
fake_logger.expect('info', ("registering with 'processors' table", ), {})
db_conn.expect('commit', (), {})
return register_class(conf, db_pool, now_func, os_module, sdb_module)
def test_assume_any_identity_1():
conf = sutil.DotDict()
fake_logger = exp.DummyObjectWithExpectations()
conf.logger = fake_logger
conf.processorCheckInTime = dt.timedelta(0, 300)
threshold = now_func() + conf.processorCheckInTime
os_module = exp.DummyObjectWithExpectations()
sdb_module = exp.DummyObjectWithExpectations()
db_conn = exp.DummyObjectWithExpectations()
db_cur = exp.DummyObjectWithExpectations()
db_pool = exp.DummyObjectWithExpectations()
hostname = 'fred'
class MyRegister(reg.ProcessorRegistrationAgent):
def __init__(self, config, db_conn_source, now_func, os, sdb):
super(MyRegister, self).__init__(config, db_conn_source, now_func,
os, sdb)
def take_over_dead_processor(self, cursor, proc_id):
expected_assert(db_cur, cursor)
expected_assert(proc_id, 17)
def registration(self):
pass
fake_logger.expect('debug', ('looking for any dead processor', ), {})
sql = ("select id from processors" " where lastseendatetime < %s limit 1")
sdb_module.expect('singleValueSql', (db_cur, sql, (threshold, )), {}, 17)
fake_logger.expect('info', ('will step in for processor %d', 17), {})
r = MyRegister(conf, db_pool, now_func, os_module, sdb_module)
id = r.assume_any_identity(db_cur, threshold, hostname, 17)
expected_assert(17, id)
def testUpdateCampaign():
context = getDummyContext()
campaign_id = 123
email_count = 321
sql = """UPDATE email_campaigns SET email_count = %s WHERE id = %s"""
parameters = (email_count, campaign_id)
dummyCursor = expect.DummyObjectWithExpectations()
dummyCursor.expect('execute', (sql, parameters), {}, None)
sender = es.EmailSender(context)
def test_make_rows_nice_1():
listOfRows = []
expectedListOfRows = []
for x in range(3):
dummy_client_row_object = exp.DummyObjectWithExpectations('dummy_client_row_object')
d = {'a':ValueObject(x), 'b':ValueObject(x * 10.0), 'c':ValueObject('C'*x)}
dummy_client_row_object.expect('columns', None, None, d)
expectedDict = {'a':x, 'b':x * 10.0, 'c':'C'*x}
listOfRows.append(dummy_client_row_object)
expectedListOfRows.append(expectedDict)
conn = HBaseConnectionWithPresetExpectations().conn
result = conn._make_rows_nice(listOfRows)
for a, b in zip(result, expectedListOfRows):
assert a == b, 'expected %s, but got %s' % (str(a), str(b))
def test_checkin():
def now_func():
return dt.datetime(2011, 1, 1, 0, 6, 0, tzinfo=UTC)
conf = sutil.DotDict()
conf.processorCheckInTime = dt.timedelta(0, 300)
conf.processorCheckInFrequency = dt.timedelta(0, 300)
conf.processorId = 17
fake_logger = exp.DummyObjectWithExpectations()
conf.logger = fake_logger
threshold = now_func() + conf.processorCheckInTime
os_module = exp.DummyObjectWithExpectations()
sdb_module = exp.DummyObjectWithExpectations()
db_conn = exp.DummyObjectWithExpectations()
db_cur = exp.DummyObjectWithExpectations()
db_pool = exp.DummyObjectWithExpectations()
fake_logger.expect('info', ('connecting to database', ), {})
db_pool.expect('connectionCursorPair', (), {}, (db_conn, db_cur))
os_module.expect('uname', (), {}, ['a', 'b', 'c'])
os_module.expect('getpid', (), {}, 1111)
sdb_module.expect('singleValueSql',
(db_cur, reg.ProcessorRegistrationAgent.NOW_SQL,
(conf.processorCheckInTime, )), {}, threshold)
fake_logger.expect('info', ("registering with 'processors' table", ), {})
db_conn.expect('commit', (), {})
fake_logger.expect('debug', ("updating 'processor' table registration", ),
{})
db_pool.expect('connectionCursorPair', (), {}, (db_conn, db_cur))
db_cur.expect('execute', ("update processors set lastseendatetime = %s "
"where id = %s", (now_func(), 17)), {})
db_conn.expect('commit', (), {})
r = MockedRegister(conf, db_pool, now_func, os_module, sdb_module)
r.checkin()
expected_assert(now_func(), r.last_checkin_ts)
r.last_checkin_ts = dt.datetime(2011, 1, 1, 0, 6, 0, tzinfo=UTC)
r.checkin()
r.last_checkin_ts = dt.datetime(2011, 1, 1, 0, 5, 0, tzinfo=UTC)
r.checkin()
r.last_checkin_ts = dt.datetime(2011, 1, 1, 0, 4, 0, tzinfo=UTC)
r.checkin()
r.last_checkin_ts = dt.datetime(2011, 1, 1, 0, 3, 0, tzinfo=UTC)
r.checkin()
r.last_checkin_ts = dt.datetime(2011, 1, 1, 0, 2, 0, tzinfo=UTC)
r.checkin()
r.last_checkin_ts = dt.datetime(2011, 1, 1, 0, 1, 0, tzinfo=UTC)
r.checkin()
fake_logger.expect('debug', ("updating 'processor' table registration", ),
{})
db_pool.expect('connectionCursorPair', (), {}, (db_conn, db_cur))
db_cur.expect('execute', ("update processors set lastseendatetime = %s "
"where id = %s", (now_func(), 17)), {})
db_conn.expect('commit', (), {})
r.last_checkin_ts = dt.datetime(2011, 1, 1, 0, 0, 0, tzinfo=UTC)
r.checkin()
expected_assert(now_func(), r.last_checkin_ts)
def testEnsureContacts():
context = getDummyContext()
parameters = [('*****@*****.**', 'd64298ce-6217-4a97-917b-7c18d3f67e18', 'abcdefg', '2011-09-01 00:00')]
sql = """INSERT INTO email_contacts (email, subscribe_token) VALUES (%s, %s) RETURNING id"""
sql = """INSERT INTO email_contacts (email, subscribe_token, ooid, crash_date) VALUES (%s, %s, %s, %s) RETURNING id"""
dummyCursor = expect.DummyObjectWithExpectations()
dummyCursor.expect('executemany', (sql, parameters), {}, None)
# with dbID already set
campaign = ecc.EmailCampaignCreate(context)
email_rows = [('1234', '*****@*****.**', '2011-09-01 00:00', 'abcdefg', 'hijklmn')]
full_email_rows = campaign.ensure_contacts(dummyCursor, email_rows)
assert full_email_rows == [{'token': 'hijklmn', 'crash_date': '2011-09-01 00:00', 'id': '1234', 'ooid': 'abcdefg', 'email': '*****@*****.**'}]
def testDetermineEmails():
context = getDummyContext()
product = 'Foobar'
versions = '5'
signature = 'JohnHancock'
start_date = utc_now()
end_date = start_date + timedelta(hours=1)
parameters = {
'product': product,
'versions': versions,
'signature': signature,
}
version_clause = ''
if len(versions) > 0:
version_clause = " version IN %(versions)s AND "
sql = """
SELECT DISTINCT contacts.id, reports.email, reports.client_crash_date AS crash_date, reports.uuid AS ooid, contacts.subscribe_token
FROM reports
LEFT JOIN email_contacts AS contacts ON reports.email = contacts.email
WHERE TIMESTAMP WITH TIME ZONE '%s' <= reports.date_processed AND
TIMESTAMP WITH TIME ZONE '%s' > reports.date_processed AND
reports.product = %%(product)s AND
%s
reports.signature = %%(signature)s AND
LENGTH(reports.email) > 4 AND
contacts.subscribe_status IS NOT FALSE
AND contacts.email NOT IN (
SELECT contacted.email
FROM email_campaigns AS prev_campaigns
JOIN email_campaigns_contacts ON email_campaigns_contacts.email_campaigns_id = prev_campaigns.id
JOIN email_contacts AS contacted ON email_campaigns_contacts.email_contacts_id = contacted.id
WHERE prev_campaigns.product = %%(product)s
AND prev_campaigns.signature = %%(signature)s
) """ % (start_date, end_date, version_clause)
dummyCursor = expect.DummyObjectWithExpectations()
dummyCursor.expect('mogrify', (sql, parameters), {}, None)
dummyCursor.expect('execute', (sql, parameters), {}, None)
dummyCursor.expect('fetchall', (), {}, [])
campaign = ecc.EmailCampaignCreate(context)
email_rows = campaign.determine_emails(dummyCursor, product, versions, signature, start_date, end_date)
def test_get_full_row():
listOfRows = []
expectedListOfRows = []
for x in range(3):
dummy_client_row_object = exp.DummyObjectWithExpectations('dummy_client_row_object')
d = {'a':ValueObject(x), 'b':ValueObject(x * 10.0), 'c':ValueObject('C'*x)}
dummy_client_row_object.expect('columns', None, None, d)
expectedDict = {'a':x, 'b':x * 10.0, 'c':'C'*x}
listOfRows.append(dummy_client_row_object)
expectedListOfRows.append(expectedDict)
uhbc = HBaseConnectionWithPresetExpectations()
conn = uhbc.conn
dummy_clientObject = uhbc.dummy_clientObject
dummy_clientObject.expect('getRow', ('fred', '22'), {}, listOfRows)
result = conn.get_full_row('fred', '22')
for a, b in zip(result, expectedListOfRows):
assert a == b, 'expected %s, but got %s' % (str(a), str(b))
def testSaveCampaignContacts():
context = getDummyContext()
campaign_id = 123
contacted_emails = ['*****@*****.**', '*****@*****.**']
sql = """
INSERT INTO email_campaigns_contacts (email_campaigns_id, email_contacts_id)
SELECT %(campaign_id)s, email_contacts.id
FROM email_contacts
WHERE email IN %(emails)s
"""
parameters = {'campaign_id': campaign_id, 'emails': tuple(contacted_emails)}
dummyCursor = expect.DummyObjectWithExpectations()
dummyCursor.expect('execute', (sql, parameters), {}, None)
sender = es.EmailSender(context)
def testSendAllEmails():
context = getDummyContext()
testContacts = ['*****@*****.**', '*****@*****.**']
crash_date = string_to_datetime('2011-09-01')
contacts = [
(0, testContacts[0], 'abc', 'ooid1', crash_date),
(0, testContacts[1], 'abc', 'ooid2', crash_date)
]
subject = 'email subject'
body = 'email body'
dummySmtp = expect.DummyObjectWithExpectations()
# no variables
noVarBody = 'Content-Type: text/plain; charset="utf-8"\nMIME-Version: 1.0\nContent-Transfer-Encoding: base64\nFrom: [email protected]\nSubject: email subject\nTo: %s\n\nZW1haWwgYm9keQ==\n'
dummySmtp.expect('sendmail', (context.fromEmailAddress, [testContacts[0]], noVarBody % testContacts[0]), {}, None)
dummySmtp.expect('sendmail', (context.fromEmailAddress, [testContacts[1]], noVarBody % testContacts[1]), {}, None)
sender = es.EmailSender(context)
contacted_emails = sender.send_all_emails(contacts, subject, body, dummySmtp)
assert contacted_emails == {0: 'sent'}
# FIXME
# # unsubscribe variable
# unsubVarBody1 = 'Content-Type: text/plain; charset="utf-8"\nMIME-Version: 1.0\nContent-Transfer-Encoding: base64\nFrom: [email protected]\nSubject: email subject\nTo: %s\n\nZW1haWwgYm9keSBodHRwOi8vZXhhbXBsZS5jb20vdW5zdWJzY3JpYmUvYWJj\n'
# unsubVarBody2 = 'Content-Type: text/plain; charset="utf-8"\nMIME-Version: 1.0\nContent-Transfer-Encoding: base64\nFrom: [email protected]\nSubject: email subject\nTo: %s\n\nZW1haWwgYm9keSBodHRwOi8vZXhhbXBsZS5jb20vdW5zdWJzY3JpYmUvZGVm\n'
# dummySmtp.expect('sendmail', (context.fromEmailAddress, [testContacts[0]], unsubVarBody1 % testContacts[0]), {}, None)
# dummySmtp.expect('sendmail', (context.fromEmailAddress, [testContacts[1]], unsubVarBody2 % testContacts[1]), {}, None)
#
# body = 'email body *|UNSUBSCRIBE_URL|*'
# contacted_emails = sender.send_all_emails(contacts, subject, body, dummySmtp)
# print contacted_emails
#assert contacted_emails == [testContacts[0], testContacts[1]]
# email_address variable
emailVarBody1 = 'Content-Type: text/plain; charset="utf-8"\nMIME-Version: 1.0\nContent-Transfer-Encoding: base64\nFrom: [email protected]\nSubject: email subject\nTo: %s\n\nZW1haWwgYm9keSAxQGV4YW1wbGUuY29t\n'
emailVarBody2 = 'Content-Type: text/plain; charset="utf-8"\nMIME-Version: 1.0\nContent-Transfer-Encoding: base64\nFrom: [email protected]\nSubject: email subject\nTo: %s\n\nZW1haWwgYm9keSAyQGV4YW1wbGUuY29t\n'
dummySmtp.expect('sendmail', (context.fromEmailAddress, [testContacts[0]], emailVarBody1 % testContacts[0]), {}, None)
dummySmtp.expect('sendmail', (context.fromEmailAddress, [testContacts[1]], emailVarBody2 % testContacts[1]), {}, None)
body = 'email body *|EMAIL_ADDRESS|*'
contacted_emails = sender.send_all_emails(contacts, subject, body, dummySmtp)
assert contacted_emails == {0: 'sent'}
def testCrashStorageSystem_makeJsonDictFromForm():
d = util.DotDict()
d.dumpField = 'd'
fakeValue = util.DotDict()
fakeValue.value = 2
f = util.DotDict()
f.a = '1'
f.b = fakeValue
f.c = '3'
f.d = '4'
f.e = '5'
expectedTime = '12:00:01'
fakeTimeModule = exp.DummyObjectWithExpectations('fakeTimeModule')
fakeTimeModule.expect('time', (), {}, expectedTime, None)
css = cstore.CrashStorageSystem(d)
resultJson = css.makeJsonDictFromForm(f, fakeTimeModule)
assert resultJson.a == '1'
assert resultJson.b == 2
assert resultJson.c == '3'
assert resultJson.e == '5'
def test_assume_identity_by_host_3():
conf = sutil.DotDict()
fake_logger = exp.DummyObjectWithExpectations()
conf.logger = fake_logger
conf.processorCheckInTime = dt.timedelta(0, 300)
threshold = now_func() + conf.processorCheckInTime
os_module = exp.DummyObjectWithExpectations()
sdb_module = exp.DummyObjectWithExpectations()
db_conn = exp.DummyObjectWithExpectations()
db_cur = exp.DummyObjectWithExpectations()
db_pool = exp.DummyObjectWithExpectations()
hostname = 'fred'
class MyRegister(reg.ProcessorRegistrationAgent):
def __init__(self, config, db_conn_source, now_func, os, sdb):
super(MyRegister, self).__init__(config, db_conn_source, now_func,
os, sdb)
def take_over_dead_processor(self, cursor, proc_id):
expected_assert(db_cur, cursor)
expected_assert(proc_id, 17)
def assume_new_identity(self, cursor, thresh, host, proc_id):
expected_assert(db_cur, cursor)
expected_assert(threshold, thresh)
expected_assert(hostname, host)
expected_assert(proc_id, 17)
return proc_id
def registration(self):
pass
fake_logger.expect('debug',
('looking for a dead processor for host %s', 'fred'),
{})
sql = ("select id from processors"
" where lastseendatetime < %s"
" and name like %s limit 1")
sdb_module.expect('singleValueSql',
(db_cur, sql, (threshold, hostname + '%')), {}, None,
sdb.SQLDidNotReturnSingleValue)
fake_logger.expect('debug',
("no dead processor found for host, %s", hostname), {})
sql2 = "select id from processors where name like 'fred%'"
sdb_module.expect('singleValueSql', (db_cur, sql2), {}, None,
sdb.SQLDidNotReturnSingleValue)
r = MyRegister(conf, db_pool, now_func, os_module, sdb_module)
id = r.assume_identity_by_host(db_cur, threshold, hostname, 17)
expected_assert(17, id)
def test_assume_specific_identity_2():
conf = sutil.DotDict()
fake_logger = exp.DummyObjectWithExpectations()
conf.logger = fake_logger
conf.processorCheckInTime = dt.timedelta(0, 300)
threshold = now_func() + conf.processorCheckInTime
os_module = exp.DummyObjectWithExpectations()
sdb_module = exp.DummyObjectWithExpectations()
db_conn = exp.DummyObjectWithExpectations()
db_cur = exp.DummyObjectWithExpectations()
db_pool = exp.DummyObjectWithExpectations()
hostname = 'fred'
class MyRegister(reg.ProcessorRegistrationAgent):
def __init__(self, config, db_conn_source, now_func, os, sdb):
super(MyRegister, self).__init__(config, db_conn_source, now_func,
os, sdb)
def take_over_dead_processor(self, cursor, proc_id):
expected_assert(db_cur, cursor)
expected_assert(proc_id, 17)
def registration(self):
pass
fake_logger.expect('debug', ('looking for a specific dead processor', ),
{})
sql = ("select id from processors "
"where lastSeenDateTime < %s "
"and id = %s")
sdb_module.expect('singleValueSql', (db_cur, sql, (threshold, 17)), {},
None, sdb.SQLDidNotReturnSingleValue)
#fake_logger.expect('info',
#('stepping in for processor %d', 17), {})
r = MyRegister(conf, db_pool, now_func, os_module, sdb_module)
try:
id = r.assume_specific_identity(db_cur, threshold, hostname, 17)
except reg.RegistrationError:
assert True
except Exception, x:
assert False, "%s exception was not expected" % str(x)
def test_assume_new_identity():
conf = sutil.DotDict()
fake_logger = exp.DummyObjectWithExpectations()
conf.logger = fake_logger
conf.processorCheckInTime = dt.timedelta(0, 300)
threshold = now_func() + conf.processorCheckInTime
os_module = exp.DummyObjectWithExpectations()
sdb_module = exp.DummyObjectWithExpectations()
db_conn = exp.DummyObjectWithExpectations()
db_cur = exp.DummyObjectWithExpectations()
db_pool = exp.DummyObjectWithExpectations()
hostname = 'fred'
class MyRegister(reg.ProcessorRegistrationAgent):
def __init__(self, config, db_conn_source, now_func, os, sdb):
super(MyRegister, self).__init__(config, db_conn_source, now_func,
os, sdb)
def registration(self):
self.processor_name = 'fred'
fake_logger.expect('debug', ('becoming a new processor', ), {})
sql = ("insert into processors"
" (id,"
" name,"
" startdatetime,"
" lastseendatetime) "
"values"
" (default,"
" %s,"
" now(),"
" now()) "
"returning id")
sdb_module.expect('singleValueSql', (db_cur, sql, ('fred', )), {}, 17)
r = MyRegister(conf, db_pool, now_func, os_module, sdb_module)
id = r.assume_new_identity(db_cur, threshold, hostname, 17)
expected_assert(17, id)
def test_take_over_dead_processor():
conf = sutil.DotDict()
fake_logger = exp.DummyObjectWithExpectations()
conf.logger = fake_logger
conf.processorCheckInTime = dt.timedelta(0, 300)
threshold = now_func() + conf.processorCheckInTime
os_module = exp.DummyObjectWithExpectations()
sdb_module = exp.DummyObjectWithExpectations()
db_conn = exp.DummyObjectWithExpectations()
db_cur = exp.DummyObjectWithExpectations()
db_pool = exp.DummyObjectWithExpectations()
hostname = 'fred'
class MyRegister(reg.ProcessorRegistrationAgent):
def __init__(self, config, db_conn_source, now_func, os, sdb):
super(MyRegister, self).__init__(config, db_conn_source, now_func,
os, sdb)
def registration(self):
self.processor_name = 'fred'
fake_logger.expect('debug', ('taking over a dead processor', ), {})
sql = ("update processors set name = %s, "
"startdatetime = now(), lastseendatetime = now()"
" where id = %s")
db_cur.expect('execute', (sql, ('fred', 17)), {}, 17)
sql2 = ("update jobs set"
" starteddatetime = NULL,"
" completeddatetime = NULL,"
" success = NULL "
"where"
" owner = %s")
db_cur.expect('execute', (sql2, (17, )), {}, 17)
r = MyRegister(conf, db_pool, now_func, os_module, sdb_module)
id = r.take_over_dead_processor(db_cur, 17)
assert id is None, "expected None but got %s" % id
