def handle(self, fn, retry, **options):
result = parse_logs(
os.path.join(get_setting('ROOT'), '%(key)s', fn),
progress=progress_bar,
retry=retry,
)
process_results(result, progress=progress_bar)
def test_search_log(self):
"""Search logs are parsed for any data"""
res = parse_logs(self.get_log('search'))[None][date(2016, 1, 5)]
self.assertEqual(res['refer'], [
'search://yahoo',
'search://thesmartsearch',
'search://google',
'search://duckduckgo',
'search://yandex',
'search://ecosia',
'search://qwant',
])
self.assertEqual(res['search'], [
'inkscape+\xe7\x84\xa1\xe6\x96\x99\xe3\x80\x80\xe3\x83\x80\xe3\x82\xa6\xe3\x83\xb3\xe3\x83\xad\xe3\x83\xbc\xe3\x83\x89',
'inkscape',
'inkscape+manual',
'inkscape filter texture copyright',
'unknown',
'inkscape',
'inkscape'
])
def test_long_log(self):
"""A log line with long names"""
AGENT_CACHE[hash('LENTEST')] = [
('os', ("FAM" * 300, "VER" * 300)),
('device', ("FAM" * 300, "BRAND" * 100)),
('browser', ("FAM" * 300, "VER" * 300)),
]
ret = parse_logs(self.get_log('long'))[None].values()[0]
self.assertEqual(ret['count'][0][:10], 'folderuri/')
self.assertEqual(len(ret['count'][0]), 255)
self.assertEqual(ret['link'][0][:10], 'refer_uri/')
self.assertEqual(len(ret['link'][0]), 255)
self.assertEqual(ret['browser'][0][1][:3], 'VER')
self.assertEqual(len(ret['browser'][0][1]), 255)
self.assertEqual(ret['browser'][0][0][:3], 'FAM')
self.assertEqual(len(ret['browser'][0][0]), 128)
self.assertEqual(ret['device'][0][1][:3], 'BRA')
self.assertEqual(len(ret['device'][0][1]), 255)
self.assertEqual(ret['device'][0][0][:3], 'FAM')
self.assertEqual(len(ret['device'][0][0]), 128)
def test_agent_filters(self):
"""The user agents should be filtered"""
process_results(parse_logs(self.get_log('agents')))
self.assertObjects(LogValue.objects.filter(metric__name='os'),
['name__family', 'name__name'],
('Linux', '3.13'),
('Android', '5.1'),
)
self.assertObjects(LogValue.objects.filter(metric__name='device'),
['name__family', 'name__name'],
('Samsung', 'SM-N920C'),
)
self.assertObjects(LogValue.objects.filter(metric__name='browser'),
['name__family', 'name__name'],
('Samsung Internet', u'3.4'),
('Bot', 'Apache-HttpClient'),
('Bot', 'Baiduspider'),
('Bot', 'bingbot'),
('Bot', 'Googlebot'),
('Bot', 'Googlebot-Image'),
('Bot', 'Python Requests'),
('Bot', 'Sogou web spider'),
('Bot', 'ZEEFscraper'),
)
def assertParseLog(self, src, result):
"""Parse a log file in fixtures and compare to result"""
return self.assertStructure(parse_logs(self.get_log(src)), result)
def setUp(self):
"""Parse a log file in fixtures, analyse and compare to result"""
super(BasicAnalysisTests, self).setUp()
process_results(parse_logs(self.get_log('basic')))
def test_file_protection(self):
"""We refuse to process the same file"""
parse_logs(self.get_log('basic'))
self.assertRaises(IOError, parse_logs, self.get_log('basic'))
def setUp(self):
super(DataReportTests, self).setUp()
process_results(parse_logs(self.get_log('reportable')))
self.metric = LogMetric.objects.get(name='browser')