def _testUpload(self, inputs, sym_paths=None): """Helper for testing uploading of specific paths""" if sym_paths is None: sym_paths = inputs self.upload_mock.return_value = 0 with parallel_unittest.ParallelMock(): ret = upload_symbols.UploadSymbols(sym_paths=inputs, sleep=0, retry=False) self.assertEquals(ret, 0) self.assertEquals(self.upload_mock.call_count, len(sym_paths)) # Since upload order is arbitrary, we have to do a manual scan for each # path ourselves against the uploaded file list. found_syms = [ x[0][1].symbol_item.sym_file for x in self.upload_mock.call_args_list ] for found_sym in found_syms: for path in sym_paths: if found_sym.endswith(path): break else: raise AssertionError('Could not locate %s in %r' % (path, found_syms))
def testUploadSymbolsEmpty(self): """Upload dir is empty.""" result = upload_symbols.UploadSymbols([self.data], 'fake_url', 'product') self.assertEquals(result, 0) self.assertEqual(self.urlopen_mock.call_count, 0)
def testUploadSymbolFailureSimple(self): """Verify that when UploadSymbol fails, the error count is passed up""" def UploadSymbol(*_args, **kwargs): kwargs['num_errors'].value = 4 upload_symbols.UploadSymbol = mock.Mock(side_effect=UploadSymbol) with parallel_unittest.ParallelMock(): ret = upload_symbols.UploadSymbols('', breakpad_dir=self.tempdir, sleep=0) self.assertEquals(ret, 4)
def testSlowDedupeSystem(self): """Verify a slow-to-join process doesn't break things when dedupe is off""" # The sleep value here is inherently a little racy, but seems to be good # enough to trigger the bug on a semi-regular basis on developer systems. self.PatchObject(upload_symbols, 'SymbolDeduplicatorNotify', side_effect=lambda *args: time.sleep(1)) # Test passing means the code didn't throw an exception. upload_symbols.UploadSymbols(sym_paths=[self.tempdir])
def testUploadCount(self): """Verify we can limit the number of uploaded symbols""" m = upload_symbols.UploadSymbol = mock.Mock(return_value=0) for c in xrange(3): m.reset_mock() with parallel_unittest.ParallelMock(): ret = upload_symbols.UploadSymbols('', breakpad_dir=self.tempdir, sleep=0, upload_count=c) self.assertEquals(ret, 0) self.assertEqual(m.call_count, c)
def testSuccess(self): """The server returns success for all uploads""" class Handler(SymbolServerRequestHandler): """Always return 200""" RESP_CODE = 200 self.SpawnServer(Handler) ret = upload_symbols.UploadSymbols(sym_paths=[self.sym_file] * 10, upload_url=self.server_url, product_name='upload_symbols_test') self.assertEqual(ret, 0)
def testError(self): """The server returns errors for all uploads""" class Handler(SymbolServerRequestHandler): """All connections error""" RESP_CODE = 500 RESP_MSG = 'Internal Server Error' self.SpawnServer(Handler) ret = upload_symbols.UploadSymbols(sym_paths=[self.sym_file] * 10, upload_url=self.server_url, api_key='testkey') self.assertEqual(ret, 10)
def testUploadSymbolsLimited(self): """Upload a few files.""" self.createSymbolFile('slim.sym', self.SLIM_CONTENT) self.createSymbolFile(os.path.join('nested', 'inner.sym')) self.createSymbolFile('fat.sym', self.FAT_CONTENT) result = upload_symbols.UploadSymbols( [self.data], 'fake_url', 'product', upload_limit=2) self.assertEquals(result, 0) self.assertEqual(self.urlopen_mock.call_count, 2) self.assertFalse(os.path.exists(self.failure_file))
def _testUploadURL(self, official, expected_url): """Helper for checking the url used""" m = upload_symbols.UploadSymbol = mock.Mock(return_value=0) with parallel_unittest.ParallelMock(): ret = upload_symbols.UploadSymbols('', official=official, breakpad_dir=self.tempdir, sleep=0) self.assertEqual(ret, 0) self.assertEqual(m.call_count, 3) for call_args in m.call_args_list: sym_file, url = call_args[0] self.assertEqual(url, expected_url) self.assertTrue(sym_file.endswith('.sym'))
def testError(self): """The server returns errors for all uploads""" class Handler(SymbolServerRequestHandler): """Always return 500""" RESP_CODE = 500 RESP_MSG = 'Internal Server Error' self.SpawnServer(Handler) ret = upload_symbols.UploadSymbols(sym_paths=[self.sym_file] * 10, upload_url=self.server_url, product_name='upload_symbols_test') self.assertEqual(ret, 10)
def testUploadSymbols(self): """Upload a few files.""" self.createSymbolFile('slim.sym', self.SLIM_CONTENT) self.createSymbolFile(os.path.join('nested', 'inner.sym')) self.createSymbolFile('fat.sym', self.FAT_CONTENT) result = upload_symbols.UploadSymbols( [self.data], 'fake_url', 'product', failed_list=self.failure_file, strip_cfi=len(self.SLIM_CONTENT)+1) self.assertEquals(result, 0) self.assertEqual(self.urlopen_mock.call_count, 3) self.assertEquals(osutils.ReadFile(self.failure_file), '')
def testSuccess(self): """The server returns success for all uploads""" class Handler(SymbolServerRequestHandler): """Always return 200""" RESP_CODE = 200 self.SpawnServer(Handler) ret = upload_symbols.UploadSymbols('', server=self.server, sleep=0, sym_paths=[self.sym_file] * 10, retry=False) self.assertEqual(ret, 0)
def testError(self): """The server returns errors for all uploads""" class Handler(SymbolServerRequestHandler): """Always return 500""" RESP_CODE = 500 RESP_MSG = 'Internal Server Error' self.SpawnServer(Handler) ret = upload_symbols.UploadSymbols('', server=self.server, sleep=0, sym_paths=[self.sym_file] * 10, retry=False) self.assertEqual(ret, 4)
def testUploadSymbolsLimited(self): """Upload a few files.""" self.createSymbolFile('slim.sym', self.SLIM_CONTENT) self.createSymbolFile(os.path.join('nested', 'inner.sym')) self.createSymbolFile('fat.sym', self.FAT_CONTENT) result = upload_symbols.UploadSymbols([self.data], 'fake_url', upload_limit=2, api_key='testkey') self.assertEqual(result, 0) self.assertEqual(self.request_mock.call_count, 7) self.assertNotExists(self.failure_file)
def testHungServer(self): """The server chokes, but we recover""" class Handler(SymbolServerRequestHandler): """All connections choke forever""" def do_POST(self): while True: time.sleep(1000) self.SpawnServer(Handler) with mock.patch.object(upload_symbols, 'GetUploadTimeout') as m: m.return_value = 0.01 ret = upload_symbols.UploadSymbols( sym_paths=[self.sym_file] * 10, upload_url=self.server_url, product_name='upload_symbols_test') self.assertEqual(ret, 10)
def _testUploadURL(self, official, expected_url): """Helper for checking the url used""" self.upload_mock.return_value = 0 with parallel_unittest.ParallelMock(): ret = upload_symbols.UploadSymbols('', official=official, retry=False, breakpad_dir=self.tempdir, sleep=0) self.assertEqual(ret, 0) self.assertEqual(self.upload_mock.call_count, 3) for call_args in self.upload_mock.call_args_list: url, sym_element = call_args[0] self.assertEqual(url, expected_url) self.assertTrue( sym_element.symbol_item.sym_file.endswith('.sym'))
def testHungServer(self): """The server chokes, but we recover""" class Handler(SymbolServerRequestHandler): """All connections choke forever""" def do_POST(self): while True: time.sleep(1000) self.SpawnServer(Handler) with mock.patch.object(upload_symbols, 'GetUploadTimeout') as m: m.return_value = 0.1 ret = upload_symbols.UploadSymbols('', server=self.server, sleep=0, sym_paths=[self.sym_file] * 10, retry=False) self.assertEqual(ret, 4)
def testSuccess(self): """The server returns success for all uploads""" class Handler(SymbolServerRequestHandler): """Always return 200""" RESP_CODE = 200 self.PatchObject(upload_symbols, 'ExecRequest', return_value={ 'uploadUrl': 'testurl', 'uploadKey': 'testSuccess' }) self.SpawnServer(Handler) ret = upload_symbols.UploadSymbols(sym_paths=[self.sym_file] * 10, upload_url=self.server_url, api_key='testSuccess') self.assertEqual(ret, 0)
def testDedupeNotifyFailure(self): """Test that a dedupe server failure midway doesn't wedge things""" api_mock = mock.MagicMock() def _Contains(items): """Do not dedupe anything""" return items api_mock.contains.side_effect = _Contains # Use a list so the closure below can modify the value. item_count = [0] # Pick a number big enough to trigger a hang normally, but not so # big it adds a lot of overhead. item_limit = 50 def _Push(*_args): """Die in the middle of the push list""" item_count[0] += 1 if item_count[0] > (item_limit / 10): raise ValueError('time to die') api_mock.push.side_effect = _Push self.PatchObject(isolateserver, 'get_storage_api', return_value=api_mock) def _Uploader(*args, **kwargs): """Pass the uploaded symbol to the deduper""" sym_item = args[1] passed_queue = kwargs['passed_queue'] passed_queue.put(sym_item) self.upload_mock.side_effect = _Uploader self.upload_mock.return_value = 0 with parallel_unittest.ParallelMock(): ret = upload_symbols.UploadSymbols( '', sym_paths=[self.tempdir] * item_limit, sleep=0, dedupe_namespace='inva!id name$pace') self.assertEqual(ret, 0)
def testFailedFileList(self): """Verify the failed file list is populated with the right content""" def UploadSymbol(*args, **kwargs): kwargs['failed_queue'].put(args[1].symbol_item.sym_file) kwargs['num_errors'].value = 4 self.upload_mock.side_effect = UploadSymbol with parallel_unittest.ParallelMock(): failed_list = os.path.join(self.tempdir, 'list') ret = upload_symbols.UploadSymbols('', breakpad_dir=self.tempdir, sleep=0, retry=False, failed_list=failed_list) self.assertEquals(ret, 4) # Need to sort the output as parallel/fs discovery can be unordered. got_list = sorted(osutils.ReadFile(failed_list).splitlines()) self.assertEquals(self.sym_paths, got_list)
def testHungServer(self): """The server chokes, but we recover""" class Handler(SymbolServerRequestHandler): """All connections choke forever""" self.PatchObject(upload_symbols, 'ExecRequest', return_value={'pairs': []}) def do_POST(self): while True: time.sleep(1000) self.SpawnServer(Handler) with mock.patch.object(upload_symbols, 'GetUploadTimeout') as m: m.return_value = 0.01 ret = upload_symbols.UploadSymbols(sym_paths=[self.sym_file] * 10, upload_url=self.server_url, timeout=m.return_value, api_key='testkey') self.assertEqual(ret, 10)
def testUploadSymbolsFailures(self): """Upload a few files.""" self.createSymbolFile('pass.sym') fail = self.createSymbolFile('fail.sym') def failSome(_url, symbol, _product): if symbol.file_name == fail.file_name: raise urllib2.URLError('network failure') # Mock out UploadSymbolFile so it's easy to see which file to fail for. upload_mock = self.PatchObject(upload_symbols, 'UploadSymbolFile', side_effect=failSome) result = upload_symbols.UploadSymbols( [self.data], 'fake_url', 'product', failed_list=self.failure_file) self.assertEquals(result, 1) self.assertEqual(upload_mock.call_count, 8) self.assertEquals(osutils.ReadFile(self.failure_file), 'fail.sym\n')
def testUploadSymbolsFailures(self): """Upload a few files.""" self.createSymbolFile('pass.sym') fail = self.createSymbolFile('fail.sym') def failSome(_url, symbol, _api_key): if symbol.file_name == fail.file_name: raise IOError('network failure') # Mock out UploadSymbolFile so it's easy to see which file to fail for. upload_mock = self.PatchObject(upload_symbols, 'UploadSymbolFile', side_effect=failSome) # Mock __name__ for logging. upload_mock.__name__ = 'UploadSymbolFileMock' result = upload_symbols.UploadSymbols([self.data], 'fake_url', failed_list=self.failure_file, api_key='testkey') self.assertEqual(result, 1) self.assertEqual(upload_mock.call_count, 7) self.assertEqual(osutils.ReadFile(self.failure_file), 'fail.sym\n')