def test_get_object_data_errors(self): p = log_processor.LogProcessor(self.proxy_config, DumbLogger()) p._internal_proxy = DumbInternalProxy(code=500) result = p.get_object_data('a', 'c', 'o') self.assertRaises(log_processor.BadFileDownload, list, result) p._internal_proxy = DumbInternalProxy(bad_compressed=True) result = p.get_object_data('a', 'c', 'o.gz', True) self.assertRaises(log_processor.BadFileDownload, list, result) p._internal_proxy = DumbInternalProxy(timeout=True) result = p.get_object_data('a', 'c', 'o') self.assertRaises(log_processor.BadFileDownload, list, result)
def test_process_one_access_file_error(self): access_proxy_config = self.proxy_config.copy() access_proxy_config.update({ 'log-processor-access': { 'source_filename_format': '%Y%m%d%H*', 'class_path': 'swift.stats.access_processor.AccessLogProcessor' } }) p = log_processor.LogProcessor(access_proxy_config, DumbLogger()) p._internal_proxy = DumbInternalProxy(code=500) self.assertRaises(log_processor.BadFileDownload, p.process_one_file, 'access', 'a', 'c', 'o')
def test_stats_keylist_mapping_format(self): proxy_config = self.proxy_config.copy() proxy_config.update({ 'log-processor-stats': { 'class_path': 'swift.stats.stats_processor.StatsLogProcessor' } }) p = log_processor.LogProcessor(proxy_config, DumbLogger()) mapping = p.generate_keylist_mapping() for k, v in mapping.items(): # these only work for Py2.7+ #self.assertIsInstance(k, str) self.assertTrue(isinstance(k, str), type(k))
def test_lazy_load_internal_proxy(self): # stub out internal_proxy's upload_app internal_proxy.BaseApplication = FakeUploadApp dummy_proxy_config = """[app:proxy-server] use = egg:swift#proxy """ with tmpfile(dummy_proxy_config) as proxy_config_file: conf = { 'log-processor': { 'proxy_server_conf': proxy_config_file, } } p = log_processor.LogProcessor(conf, DumbLogger()) self.assert_(isinstance(p._internal_proxy, None.__class__)) self.assert_( isinstance(p.internal_proxy, log_processor.InternalProxy)) self.assertEquals(p.internal_proxy, p._internal_proxy) # reset FakeUploadApp reload(internal_proxy)
def test_generate_keylist_mapping_with_dummy_plugins(self): class Plugin1(object): def keylist_mapping(self): return {'a': 'b', 'c': 'd', 'e': ['f', 'g']} class Plugin2(object): def keylist_mapping(self): return {'a': '1', 'e': '2', 'h': '3'} p = log_processor.LogProcessor(self.proxy_config, DumbLogger()) p.plugins['plugin1'] = {'instance': Plugin1()} p.plugins['plugin2'] = {'instance': Plugin2()} result = p.generate_keylist_mapping() expected = { 'a': set(['b', '1']), 'c': 'd', 'e': set(['2', 'f', 'g']), 'h': '3' } self.assertEquals(result, expected)
def test_access_log_line_parser(self): access_proxy_config = self.proxy_config.copy() access_proxy_config.update({ 'log-processor-access': { 'source_filename_format': '%Y%m%d%H*', 'class_path': 'swift.stats.access_processor.AccessLogProcessor' } }) p = log_processor.LogProcessor(access_proxy_config, DumbLogger()) result = p.plugins['access']['instance'].log_line_parser( self.access_test_line) self.assertEquals( result, { 'code': 200, 'processing_time': '0.0262', 'auth_token': 'tk4e350daf-9338-4cc6-aabb-090e49babfbd', 'month': '07', 'second': '30', 'year': '2010', 'query': 'format=json&foo', 'tz': '+0000', 'http_version': 'HTTP/1.0', 'object_name': 'bar', 'etag': '-', 'foo': 1, 'method': 'GET', 'trans_id': 'txfa431231-7f07-42fd-8fc7-7da9d8cc1f90', 'client_ip': '1.2.3.4', 'format': 1, 'bytes_out': 95, 'container_name': 'foo', 'day': '09', 'minute': '14', 'account': 'acct', 'hour': '04', 'referrer': '-', 'request': '/v1/acct/foo/bar', 'user_agent': 'curl', 'bytes_in': 6, 'lb_ip': '4.5.6.7' })
def test_get_stat_totals(self): stats_proxy_config = self.proxy_config.copy() stats_proxy_config.update({ 'log-processor-stats': { 'class_path': 'swift.stats.stats_processor.StatsLogProcessor' } }) p = log_processor.LogProcessor(stats_proxy_config, DumbLogger()) p._internal_proxy = DumbInternalProxy() def get_object_data(*a, **kw): return [self.stats_test_line] p.get_object_data = get_object_data result = p.process_one_file('stats', 'a', 'c', 'y/m/d/h/o') expected = { ('account', 'y', 'm', 'd', 'h'): { 'replica_count': 1, 'object_count': 2, 'container_count': 1, 'bytes_used': 3 } } self.assertEquals(result, expected)
def test_generate_keylist_mapping(self): p = log_processor.LogProcessor(self.proxy_config, DumbLogger()) result = p.generate_keylist_mapping() expected = {} self.assertEquals(result, expected)