def test_lazy_load_internal_proxy(self): # stub out internal_proxy's upload_app internal_proxy.Application = FakeUploadApp dummy_proxy_config = """[app:proxy-server] use = egg:swift#proxy """ with tmpfile(dummy_proxy_config) as proxy_config_file: conf = { 'log-processor': { 'proxy_server_conf': proxy_config_file, 'swift_account': 'foo' } } p = log_processor.LogProcessor(conf, DumbLogger()) self.assert_(isinstance(p._internal_proxy, None.__class__)) self.assert_( isinstance(p.internal_proxy, log_processor.InternalProxy)) self.assertEquals(p.internal_proxy, p._internal_proxy) # test with empty config variable conf = { 'log-processor': { 'proxy_server_conf': '', 'swift_account': 'foo' } } q = log_processor.LogProcessor(conf, DumbLogger()) self.assert_(isinstance(q._internal_proxy, None.__class__)) self.assert_(isinstance(q.internal_proxy, log_processor.InternalProxy)) self.assertEquals(q.internal_proxy, q._internal_proxy) # reset FakeUploadApp reload(internal_proxy)
def test_get_container_listing(self): p = log_processor.LogProcessor(self.proxy_config, DumbLogger()) p._internal_proxy = DumbInternalProxy() result = p.get_container_listing('a', 'foo') expected = ['2010/03/14/13/obj1'] self.assertEquals(result, expected) result = p.get_container_listing('a', 'foo', listing_filter=expected) expected = [] self.assertEquals(result, expected) result = p.get_container_listing('a', 'foo', start_date='2010031412', end_date='2010031414') expected = ['2010/03/14/13/obj1'] self.assertEquals(result, expected) result = p.get_container_listing('a', 'foo', start_date='2010031414') expected = [] self.assertEquals(result, expected) result = p.get_container_listing('a', 'foo', start_date='2010031410', end_date='2010031412') expected = [] self.assertEquals(result, expected) result = p.get_container_listing('a', 'foo', start_date='2010031412', end_date='2010031413') expected = ['2010/03/14/13/obj1'] self.assertEquals(result, expected)
def test_process_one_access_file(self): access_proxy_config = self.proxy_config.copy() access_proxy_config.update({ 'log-processor-access': { 'source_filename_format': '%Y%m%d%H*', 'class_path': 'slogging.access_processor.AccessLogProcessor' } }) p = log_processor.LogProcessor(access_proxy_config, DumbLogger()) def get_object_data(*a, **kw): return [self.access_test_line] p.get_object_data = get_object_data result = p.process_one_file('access', 'a', 'c', 'o') expected = { ('acct', '2010', '07', '09', '04'): { ('public', 'object', 'GET', '2xx'): 1, ('public', 'bytes_out'): 95, 'marker_query': 0, 'format_query': 1, 'delimiter_query': 0, 'path_query': 0, ('public', 'bytes_in'): 6, 'prefix_query': 0 } } self.assertEquals(result, expected)
def test_get_object_data(self): p = log_processor.LogProcessor(self.proxy_config, DumbLogger()) p._internal_proxy = DumbInternalProxy() result = list(p.get_object_data('a', 'c', 'o', False)) expected = ['obj', 'data'] self.assertEquals(result, expected) result = list(p.get_object_data('a', 'c', 'o.gz', True)) self.assertEquals(result, expected)
def test_get_object_data_errors(self): p = log_processor.LogProcessor(self.proxy_config, DumbLogger()) p._internal_proxy = DumbInternalProxy(code=500) result = p.get_object_data('a', 'c', 'o') self.assertRaises(log_common.BadFileDownload, list, result) p._internal_proxy = DumbInternalProxy(bad_compressed=True) result = p.get_object_data('a', 'c', 'o.gz', True) self.assertRaises(log_common.BadFileDownload, list, result) p._internal_proxy = DumbInternalProxy(timeout=True) result = p.get_object_data('a', 'c', 'o') self.assertRaises(log_common.BadFileDownload, list, result)
def test_process_one_access_file_error(self): access_proxy_config = self.proxy_config.copy() access_proxy_config.update({ 'log-processor-access': { 'source_filename_format': '%Y%m%d%H*', 'class_path': 'slogging.access_processor.AccessLogProcessor' } }) p = log_processor.LogProcessor(access_proxy_config, DumbLogger()) p._internal_proxy = DumbInternalProxy(code=500) self.assertRaises(log_common.BadFileDownload, p.process_one_file, 'access', 'a', 'c', 'o')
def test_stats_keylist_mapping_format(self): proxy_config = self.proxy_config.copy() proxy_config.update({ 'log-processor-stats': { 'class_path': 'slogging.stats_processor.StatsLogProcessor' } }) p = log_processor.LogProcessor(proxy_config, DumbLogger()) mapping = p.generate_keylist_mapping() for k, v in mapping.items(): # these only work for Py2.7+ #self.assertIsInstance(k, str) self.assertTrue(isinstance(k, str), type(k))
def test_generate_keylist_mapping_with_dummy_plugins(self): class Plugin1(object): def keylist_mapping(self): return {'a': 'b', 'c': 'd', 'e': ['f', 'g']} class Plugin2(object): def keylist_mapping(self): return {'a': '1', 'e': '2', 'h': '3'} p = log_processor.LogProcessor(self.proxy_config, DumbLogger()) p.plugins['plugin1'] = {'instance': Plugin1()} p.plugins['plugin2'] = {'instance': Plugin2()} result = p.generate_keylist_mapping() expected = { 'a': set(['b', '1']), 'c': 'd', 'e': set(['2', 'f', 'g']), 'h': '3' } self.assertEquals(result, expected)
def test_access_log_line_parser(self): access_proxy_config = self.proxy_config.copy() access_proxy_config.update({ 'log-processor-access': { 'source_filename_format': '%Y%m%d%H*', 'class_path': 'slogging.access_processor.AccessLogProcessor' } }) p = log_processor.LogProcessor(access_proxy_config, DumbLogger()) result = p.plugins['access']['instance'].log_line_parser( self.access_test_line) self.assertEquals( result, { 'code': 200, 'processing_time': '0.0262', 'auth_token': 'tk4e350daf-9338-4cc6-aabb-090e49babfbd', 'month': '07', 'second': '30', 'year': '2010', 'query': 'format=json&foo', 'tz': '+0000', 'http_version': 'HTTP/1.0', 'object_name': 'bar', 'etag': '-', 'method': 'GET', 'trans_id': 'txfa431231-7f07-42fd-8fc7-7da9d8cc1f90', 'client_ip': '1.2.3.4', 'format': 1, 'bytes_out': 95, 'container_name': 'foo', 'day': '09', 'minute': '14', 'account': 'acct', 'hour': '04', 'referrer': '-', 'request': '/v1/acct/foo/bar', 'user_agent': 'curl', 'bytes_in': 6, 'lb_ip': '4.5.6.7', 'log_source': None })
def test_get_stat_totals(self): stats_proxy_config = self.proxy_config.copy() stats_proxy_config.update({ 'log-processor-stats': { 'class_path': 'slogging.stats_processor.StatsLogProcessor' } }) p = log_processor.LogProcessor(stats_proxy_config, DumbLogger()) p._internal_proxy = DumbInternalProxy() def get_object_data(*a, **kw): return [self.stats_test_line] p.get_object_data = get_object_data result = p.process_one_file('stats', 'a', 'c', 'y/m/d/h/o') expected = { ('account', 'y', 'm', 'd', 'h'): { 'replica_count': 1, 'object_count': 2, 'container_count': 1, 'bytes_used': 3 } } self.assertEquals(result, expected)
def test_generate_keylist_mapping(self): p = log_processor.LogProcessor(self.proxy_config, DumbLogger()) result = p.generate_keylist_mapping() expected = {} self.assertEquals(result, expected)