def test_parse_log_from_distinct_channels(self):
"""Parse files from distinct channels"""
ch1 = self.mock_db.db.get_channel_id('ch1')
nmsg, nmsg_new = parse_irc_file('data/#mediawiki-20010101.log', ch1,
'plain', self.mock_db.db)
actions = self.mock_db.count_num_messages(ch1)
self.assertEqual(nmsg, 79)
self.assertEqual(nmsg_new, 79)
self.assertEqual(len(actions.keys()), 2)
self.assertEqual(actions[str(LogParser.COMMENT)], 75)
self.assertEqual(actions[str(LogParser.ACTION)], 4)
ch2 = self.mock_db.db.get_channel_id('ch2')
nmsg, nmsg_new = parse_irc_file('data/#ceph.20010101.log', ch2,
'plain', self.mock_db.db)
actions = self.mock_db.count_num_messages(ch2)
self.assertEqual(nmsg, 95)
self.assertEqual(nmsg_new, 95)
self.assertEqual(len(actions.keys()), 4)
self.assertEqual(actions[str(LogParser.COMMENT)], 49)
self.assertEqual(actions[str(LogParser.JOIN)], 22)
self.assertEqual(actions[str(LogParser.PART)], 21)
self.assertEqual(actions[str(LogParser.NICKCHANGE)], 3)
# Check the total count of messages by type
actions = self.mock_db.count_num_messages()
self.assertEqual(len(actions.keys()), 5)
self.assertEqual(actions[str(LogParser.COMMENT)], 124)
self.assertEqual(actions[str(LogParser.ACTION)], 4)
self.assertEqual(actions[str(LogParser.JOIN)], 22)
self.assertEqual(actions[str(LogParser.PART)], 21)
self.assertEqual(actions[str(LogParser.NICKCHANGE)], 3)
def test_parse_log_from_same_channel(self):
"""Parse files from the same channel"""
ch1 = self.mock_db.db.get_channel_id('ch1')
parse_irc_file('data/#texthtml-20071217.log', ch1, 'html',
self.mock_db.db)
parse_irc_file('data/#tablehtml-20131211.log', ch1, 'html',
self.mock_db.db)
actions = self.mock_db.count_num_messages()
self.assertEqual(len(actions.keys()), 5)
self.assertEqual(actions[str(LogParser.COMMENT)], 98)
self.assertEqual(actions[str(LogParser.JOIN)], 105)
self.assertEqual(actions[str(LogParser.PART)], 115)
self.assertEqual(actions[str(LogParser.NICKCHANGE)], 5)
self.assertEqual(actions[str(LogParser.SERVER)], 3)
def test_mediawiki_log(self):
"""Parse MediaWiki log files"""
import os
ch = self.mock_db.db.get_channel_id('mediawiki')
files = os.listdir(MEDIAWIKI_LOG_PATH)
files.sort()
for logfile in files:
date = parse_irc_filename(logfile)
filepath = os.path.join(MEDIAWIKI_LOG_PATH, logfile)
parse_irc_file(filepath, ch, 'plain', self.mock_db.db, date)
actions = self.mock_db.count_num_messages()
self.assertEqual(len(actions.keys()), 2)
self.assertEqual(actions[str(LogParser.COMMENT)], 5443)
self.assertEqual(actions[str(LogParser.ACTION)], 109)
def test_parse_plain_text_logfile(self):
"""Parse a plain text log"""
ch1 = self.mock_db.db.get_channel_id('ch1')
nmsg, nmsg_new = parse_irc_file('data/#mediawiki-20010101.log', ch1,
'plain', self.mock_db.db)
actions = self.mock_db.count_num_messages(ch1)
self.assertEqual(nmsg, 79)
self.assertEqual(nmsg_new, 79)
self.assertEqual(len(actions.keys()), 2)
self.assertEqual(actions[str(LogParser.COMMENT)], 75)
self.assertEqual(actions[str(LogParser.ACTION)], 4)
def test_override_log_date(self):
"""Override the date of the filename
If the date is overridden, new messages will be inserted although
the two logs have the same date in their filename or stored
in log messages.
"""
force_date = string_to_datetime('2008-01-01', '%Y-%m-%d')
ch1 = self.mock_db.db.get_channel_id('ch1')
parse_irc_file('data/#texthtml-20071217.log', ch1, 'html',
self.mock_db.db)
parse_irc_file('data/#texthtml-20071217.log', ch1, 'html',
self.mock_db.db, force_date)
actions = self.mock_db.count_num_messages()
self.assertEqual(len(actions.keys()), 5)
self.assertEqual(actions[str(LogParser.COMMENT)], 118)
self.assertEqual(actions[str(LogParser.JOIN)], 124)
self.assertEqual(actions[str(LogParser.PART)], 94)
self.assertEqual(actions[str(LogParser.NICKCHANGE)], 4)
self.assertEqual(actions[str(LogParser.SERVER)], 6)
def test_parse_table_html_logfile(self):
"""Parse a table html log"""
ch1 = self.mock_db.db.get_channel_id('ch1')
nmsg, nmsg_new = parse_irc_file('data/#tablehtml-20131211.log', ch1,
'html', self.mock_db.db)
actions = self.mock_db.count_num_messages(ch1)
self.assertEqual(nmsg, 153)
self.assertEqual(nmsg_new, 153)
self.assertEqual(len(actions.keys()), 4)
self.assertEqual(actions[str(LogParser.COMMENT)], 39)
self.assertEqual(actions[str(LogParser.JOIN)], 43)
self.assertEqual(actions[str(LogParser.PART)], 68)
self.assertEqual(actions[str(LogParser.NICKCHANGE)], 3)
def test_parse_text_html_logfile(self):
"""Parse a text html log"""
ch1 = self.mock_db.db.get_channel_id('ch1')
nmsg, nmsg_new = parse_irc_file('data/#texthtml-20071217.log', ch1,
'html', self.mock_db.db)
actions = self.mock_db.count_num_messages(ch1)
self.assertEqual(nmsg, 173)
self.assertEqual(nmsg_new, 173)
self.assertEqual(len(actions.keys()), 5)
self.assertEqual(actions[str(LogParser.COMMENT)], 59)
self.assertEqual(actions[str(LogParser.JOIN)], 62)
self.assertEqual(actions[str(LogParser.PART)], 47)
self.assertEqual(actions[str(LogParser.NICKCHANGE)], 2)
self.assertEqual(actions[str(LogParser.SERVER)], 3)