def test_ignore_messages(self): """Test if it ignores some messages without mandatory fields""" backend = MBox('http://example.com/', self.tmp_error_path) messages = [m for m in backend.fetch()] # There are only two valid message on the mbox self.assertEqual(len(messages), 2) expected = { 'From': 'goran at domain.com ( Göran Lastname )', 'Date': 'Wed, 01 Dec 2010 14:26:40 +0100', 'Subject': '[List-name] Protocol Buffers anyone?', 'Message-ID': '<*****@*****.**>', 'unixfrom': 'goran at domain.com Wed Dec 1 08:26:40 2010', 'body': { 'plain': "Hi!\n\nA message in English, with a signature " "with a different encoding.\n\nregards, G?ran" "\n", } } message = messages[0]['data'] self.assertDictEqual(message, expected) # On the second message, the only change is that 'Message-id' # is replaced by 'Message-ID' message = messages[1]['data'] self.assertDictEqual(message, expected)
def test_fetch(self): """Test whether it parses a set of mbox files""" backend = MBox('http://example.com/', self.tmp_path) messages = [m for m in backend.fetch()] expected = [ ('<*****@*****.**>', '86315b479b4debe320b59c881c1e375216cbf333', 1291210000.0), ('<*****@*****.**>', '86315b479b4debe320b59c881c1e375216cbf333', 1291210000.0), ('<*****@*****.**>', 'bd0185317b013beb21ad3ea04635de3db72496ad', 1095843820.0), ('<*****@*****.**>', '51535703010a3e63d5272202942c283394cdebca', 1205746505.0), ('<019801ca633f$f4376140$dca623c0$@[email protected]>', '302e314c07242bb4750351286862f49e758f3e17', 1257992964.0), ('<*****@*****.**>', 'ddda42422c55d08d56c017a6f128fcd7447484ea', 1043881350.0), ('<*****@*****.**>', '86315b479b4debe320b59c881c1e375216cbf333', 1291210000.0), ('<*****@*****.**>', 'ad3116ae93c0df50436f7c84bfc94000e990996c', 1421328145.0), ('<*****@*****.**>', '4e255acab6442424ecbf05cb0feb1eccb587f7de', 1030123489.0), ] self.assertEqual(len(messages), len(expected)) for x in range(len(messages)): message = messages[x] self.assertEqual(message['data']['Message-ID'], expected[x][0]) self.assertEqual(message['origin'], 'http://example.com/') self.assertEqual(message['uuid'], expected[x][1]) self.assertEqual(message['updated_on'], expected[x][2]) self.assertEqual(message['category'], 'message') self.assertEqual(message['tag'], 'http://example.com/')
def test_fetch_from_date(self): """Test whether a list of messages is returned since a given date""" from_date = datetime.datetime(2008, 1, 1) backend = MBox('http://example.com/', self.tmp_path) messages = [m for m in backend.fetch(from_date=from_date)] expected = [ ('<*****@*****.**>', '86315b479b4debe320b59c881c1e375216cbf333', 1291210000.0), ('<*****@*****.**>', '86315b479b4debe320b59c881c1e375216cbf333', 1291210000.0), ('<*****@*****.**>', '51535703010a3e63d5272202942c283394cdebca', 1205746505.0), ('<019801ca633f$f4376140$dca623c0$@[email protected]>', '302e314c07242bb4750351286862f49e758f3e17', 1257992964.0), ('<*****@*****.**>', '86315b479b4debe320b59c881c1e375216cbf333', 1291210000.0), ('<*****@*****.**>', 'ad3116ae93c0df50436f7c84bfc94000e990996c', 1421328145.0) ] self.assertEqual(len(messages), len(expected)) for x in range(len(messages)): message = messages[x] self.assertEqual(message['data']['Message-ID'], expected[x][0]) self.assertEqual(message['origin'], 'http://example.com/') self.assertEqual(message['uuid'], expected[x][1]) self.assertEqual(message['updated_on'], expected[x][2]) self.assertEqual(message['category'], 'message') self.assertEqual(message['tag'], 'http://example.com/')
def list_mailers(self, url, directory="files/mbox"): repo = MBox(uri=url, dirpath=directory) count = 0 list_mailers = [] for message in repo.fetch(): list_mailers.append(message['data']['From']) return list_mailers
def test_ignore_file_errors(self): """Files with IO errors should be ignored""" tmp_path_ign = tempfile.mkdtemp(prefix='perceval_') shutil.copy( os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/mbox/mbox_single.mbox'), tmp_path_ign) shutil.copy( os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/mbox/mbox_multipart.mbox'), tmp_path_ign) # Update file mode to make it unable to access os.chmod(os.path.join(tmp_path_ign, 'mbox_multipart.mbox'), 0o000) backend = MBox('http://example.com/', tmp_path_ign) messages = [m for m in backend.fetch()] # Only one message is read self.assertEqual(len(messages), 1) self.assertEqual(messages[0]['data']['Message-ID'], '<*****@*****.**>') self.assertEqual(messages[0]['data']['Date'], 'Wed, 01 Dec 2010 14:26:40 +0100') shutil.rmtree(tmp_path_ign)
def test_search_fields(self): """Test whether the search_fields is properly set""" backend = MBox('http://example.com/', self.tmp_path) messages = [m for m in backend.fetch(from_date=None)] for message in messages: self.assertEqual(backend.metadata_id(message['data']), message['search_fields']['item_id'])
def test_fetch_exception(self, mock_str_to_datetime): """Test whether an exception is thrown when the the fetch_items method fails""" mock_str_to_datetime.side_effect = Exception backend = MBox('http://example.com/', self.tmp_path) with self.assertRaises(Exception): _ = [m for m in backend.fetch(from_date=None)]
def test_initialization(self): """Test whether attributes are initializated""" backend = MBox('http://example.com/', self.tmp_path, tag='test') self.assertEqual(backend.uri, 'http://example.com/') self.assertEqual(backend.dirpath, self.tmp_path) self.assertEqual(backend.origin, 'http://example.com/') self.assertEqual(backend.tag, 'test') # When origin is empty or None it will be set to # the value in uri backend = MBox('http://example.com/', self.tmp_path) self.assertEqual(backend.origin, 'http://example.com/') self.assertEqual(backend.tag, 'http://example.com/') backend = MBox('http://example.com/', self.tmp_path, tag='') self.assertEqual(backend.origin, 'http://example.com/') self.assertEqual(backend.tag, 'http://example.com/')
def test_ignore_file_errors(self): """Files with IO errors should be ignored""" tmp_path_ign = tempfile.mkdtemp(prefix='perceval_') def copy_mbox_side_effect(*args, **kwargs): """Copy a mbox archive or raise IO error for 'mbox_multipart.mbox' archive""" error_file = os.path.join(tmp_path_ign, 'mbox_multipart.mbox') mbox = args[0] if mbox.filepath == error_file: raise OSError('Mock error') tmp_path = tempfile.mktemp(prefix='perceval_') with mbox.container as f_in: with open(tmp_path, mode='wb') as f_out: for l in f_in: f_out.write(l) return tmp_path shutil.copy( os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/mbox/mbox_single.mbox'), tmp_path_ign) shutil.copy( os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/mbox/mbox_multipart.mbox'), tmp_path_ign) # Mock 'copy_mbox' method for forcing to raise an OSError # with file 'data/mbox/mbox_multipart.mbox' to check if # the code ignores this file with unittest.mock.patch('perceval.backends.core.mbox.MBox._copy_mbox' ) as mock_copy_mbox: mock_copy_mbox.side_effect = copy_mbox_side_effect backend = MBox('http://example.com/', tmp_path_ign) messages = [m for m in backend.fetch()] # Only one message is read self.assertEqual(len(messages), 1) self.assertEqual(messages[0]['data']['Message-ID'], '<*****@*****.**>') self.assertEqual(messages[0]['data']['Date'], 'Wed, 01 Dec 2010 14:26:40 +0100') shutil.rmtree(tmp_path_ign)
def get_content(self, max_items: Optional[int] = None) -> List[str]: """ Получает только содержимое писем в MBOX хранилище писем игнорируя все заголовки :param max_items: Максимальное количество сообщений :return: генератор с содержимым писем архива MBOX """ repo = MBox(self.mbox_path, self.mbox_path) result = [] for index, msg in enumerate(repo.fetch()): if max_items and index >= max_items: break result.append(msg['data']['body'].get( 'html', msg['data']['body'].get('plain', ''))) return result
mbox_uri = 'http://mail-archives.apache.org/mod_mbox/httpd-announce/' # directory for letting Perceval where mbox archives are # you need to have the archives to analyzed there before running the script mbox_dir = 'archives' # ElasticSearch instance (url) es = elasticsearch.Elasticsearch(['http://localhost:9200/']) # Create the 'messages' index in ElasticSearch try: es.indices.create('messages') except elasticsearch.exceptions.RequestError: print('Index already exisits, remove it before running this script again.') exit() # create a mbox object, using mbox_uri as label, mbox_dir as directory to scan repo = MBox(uri=mbox_uri, dirpath=mbox_dir) # Fetch all commits as an iteratoir, and iterate it uploading to ElasticSearch print('Analyzing mbox archives...') # fetch all messages as an iteratoir for message in repo.fetch(): # Create the object (dictionary) to upload to ElasticSearch summary = { 'from': message['data']['From'], 'subject': message['data']['Subject'], 'date': email.utils.parsedate_to_datetime(message['data']['Date']) } print('.', end='') # Upload the object to ElasticSearch es.index(index='messages', doc_type='summary', body=summary)
def getmbox(self, mbox_files): mbox_parser = MBox(uri=mbox_files, dirpath='./mboxes') return mbox_parser.fetch()
def createRepo(self, dir, mailList): mbox_uri = mailList + '@iova.net' mbox_dir = dir repo = MBox(uri=mbox_uri, dirpath=mbox_dir) return repo
def numMails(self, url, directory="files/mbox"): repo = MBox(uri=url, dirpath=directory) count = 0 for message in repo.fetch(): count += 1 return count
def createRepo(self): mbox_uri = '*****@*****.**' mbox_dir = '../mails' repo = MBox(uri=mbox_uri, dirpath=mbox_dir) return repo
from perceval.backends.core.mbox import MBox from elasticsearch import Elasticsearch import hashlib from jwzthreading import thread, Message es = Elasticsearch() src_url = 'https://lists.xenproject.org/archives/html/mbox/' dest_dir = '/home/heather/dev/xen/mboxes' repo = MBox(uri=src_url, dirpath=dest_dir) logfile = open('mboxAnalysis.log', 'w') def create_index(): mapping = { "properties": { "data": { "properties": { "Message-ID": { "type": "keyword" } } } } } es.indices.create(index='xenmbox', ignore=400) es.indices.put_mapping(index='xenmbox', doc_type='unknown', body=mapping, update_all_types=True)