def test_encrypted_mail(self):
"""Test parsing of a pgp encrypted mail."""
data = load_mail('pgp_crypted_1.eml')
mail = MailMessage(data)
self.assertTrue(verifyObject(IMessageParser, mail))
self.assertTrue(len(mail.participants) > 1)
self.assertEqual(len(mail.attachments), 2)
self.assertEqual(mail.subject, 'crypted content')
self.assertTrue(isinstance(mail.date, datetime))
def test_transport_signature(self):
mail = load_mail('dkim1.eml')
message = MailMessage(mail)
param = NewMessage()
user = FakeUser()
featurer = InboundMailFeature(message, {})
featurer.process(user, param, ([]))
feats = param.privacy_features
self.assertEqual(feats.get('transport_signed'), True)
def test_encrypted_message(self):
mail = load_mail('pgp_crypted_1.eml')
message = MailMessage(mail)
param = NewMessage()
user = FakeUser()
featurer = InboundMailFeature(message, {})
featurer.process(user, param, ([]))
feats = param.privacy_features
self.assertEqual(feats.get('message_encrypted'), True)
self.assertEqual(feats.get('message_encryption_method'), 'pgp')
def test_no_encryption_signature(self):
mail = load_mail('spam1.eml')
message = MailMessage(mail)
param = NewMessage()
user = FakeUser()
featurer = InboundMailFeature(message, {})
featurer.process(user, param, ([]))
feats = param.privacy_features
self.assertEqual(feats.get('message_signed'), False)
self.assertEqual(feats.get('message_encrypted'), False)
self.assertEqual(feats.get('message_encryption_method'), '')
self.assertEqual(feats.get('message_signature_type'), '')
self.assertEqual(feats.get('transport_signed'), False)
def import_email(email, import_path, format, contact_probability, **kwargs):
"""Import emails for an user."""
from caliopen_main.user.core import User
from caliopen_main.contact.core import Contact, ContactLookup
from caliopen_main.message.parsers.mail import MailMessage
from caliopen_main.contact.parameters import NewContact, NewEmail
from caliopen_nats.delivery import UserMessageDelivery
from caliopen_main.message.core import RawMessage
from caliopen_storage.config import Configuration
max_size = int(Configuration("global").get("object_store.db_size_limit"))
if format == 'maildir':
emails = Maildir(import_path, factory=message_from_file)
mode = 'maildir'
else:
if os.path.isdir(import_path):
mode = 'mbox_directory'
emails = {}
files = [
f for f in os.listdir(import_path)
if os.path.isfile(os.path.join(import_path, f))
]
for f in files:
try:
log.debug('Importing mail from file {}'.format(f))
with open('%s/%s' % (import_path, f)) as fh:
emails[f] = message_from_file(fh)
except Exception as exc:
log.error('Error importing email {}'.format(exc))
else:
mode = 'mbox'
emails = mbox(import_path)
user = User.by_local_identity(email)
log.info("Processing mode %s" % mode)
for key, data in emails.iteritems():
# Prevent creating message too large to fit in db.
# (should use inject cmd for large messages)
size = len(data.as_string())
if size > max_size:
log.warn("Message too large to fit into db. \
Please, use 'inject' cmd for importing large emails.")
continue
raw = RawMessage.create(data.as_string())
log.debug('Created raw message {}'.format(raw.raw_msg_id))
message = MailMessage(data.as_string())
dice = random()
if dice <= contact_probability:
for participant in message.participants:
try:
ContactLookup.get(user, participant.address)
except NotFound:
log.info('Creating contact %s' % participant.address)
name, domain = participant.address.split('@')
contact_param = NewContact()
contact_param.family_name = name
if participant.address:
e_mail = NewEmail()
e_mail.address = participant.address
contact_param.emails = [e_mail]
Contact.create(user, contact_param)
log.info('No contact associated to raw {} '.format(raw.raw_msg_id))
processor = UserMessageDelivery(user)
obj_message = processor.process_raw(raw.raw_msg_id)
log.info('Created message {}'.format(obj_message.message_id))
def process_inbound(self, raw):
"""Process inbound message.
@param raw: a RawMessage object
@rtype: NewMessage
"""
message = MailMessage(raw.raw_data)
new_message = NewInboundMessage()
new_message.raw_msg_id = raw.raw_msg_id
new_message.subject = message.subject
new_message.body_html = message.body_html
new_message.body_plain = message.body_plain
new_message.date = message.date
new_message.size = message.size
new_message.protocol = message.message_protocol
new_message.is_unread = True
new_message.is_draft = False
new_message.is_answered = False
new_message.is_received = True
new_message.importance_level = 0 # XXX tofix on parser
new_message.external_references = message.external_references
participants = []
for p in message.participants:
participant, contact = self.get_participant(message, p)
new_message.participants.append(participant)
participants.append((participant, contact))
if not participants:
raise Exception("no participant found in raw message {}".format(
raw.raw_msg_id))
for a in message.attachments:
attachment = Attachment()
attachment.content_type = a.content_type
attachment.file_name = a.filename
attachment.size = a.size
attachment.mime_boundary = a.mime_boundary
if hasattr(a, "is_inline"):
attachment.is_inline = a.is_inline
new_message.attachments.append(attachment)
# Compute PI !!
conf = Configuration('global').configuration
extractor = InboundMailFeature(message, conf)
extractor.process(self.user, new_message, participants)
# compute tags
self._get_tags(new_message)
if new_message.tags:
log.debug('Resolved tags {}'.format(new_message.tags))
# lookup by external references
lookup_sequence = self.lookup_discussion_sequence(message, new_message)
lkp = self.lookup(lookup_sequence)
log.debug('Lookup with sequence {} give {}'.
format(lookup_sequence, lkp))
if lkp:
new_message.discussion_id = lkp.discussion_id
else:
discussion = Discussion.create_from_message(self.user, message)
log.debug('Created discussion {}'.format(discussion.discussion_id))
new_message.discussion_id = discussion.discussion_id
self.create_lookups(lookup_sequence, new_message)
# Format features
new_message.privacy_features = \
marshal_features(new_message.privacy_features)
try:
new_message.validate()
except Exception as exc:
log.error(
"validation failed with error : « {} » \
for new_message {}[dump : {}]".format(
exc, new_message, vars(new_message)))
raise exc
return new_message
def parse(self):
"""Parse raw message to get a formatted object."""
return MailMessage(self)
def import_email(email, import_path, format, contact_probability, **kwargs):
"""Import emails for an user."""
from caliopen_main.user.core import User
from caliopen_main.contact.core import Contact, ContactLookup
from caliopen_main.message.parsers.mail import MailMessage
from caliopen_main.contact.parameters import NewContact, NewEmail
from caliopen_nats.delivery import UserMailDelivery
from caliopen_main.message.core import RawMessage
from caliopen_storage.config import Configuration
max_size = int(Configuration("global").get("object_store.db_size_limit"))
if 'to' in kwargs and kwargs['to']:
dest_email = kwargs['to']
else:
dest_email = email
if format == 'maildir':
if dest_email != email:
raise Exception('Cannot change To email using maildir format')
emails = Maildir(import_path, factory=message_from_file)
mode = 'maildir'
else:
if os.path.isdir(import_path):
mode = 'mbox_directory'
emails = {}
files = [
f for f in os.listdir(import_path)
if os.path.isfile(os.path.join(import_path, f))
]
for f in files:
try:
log.debug('Importing mail from file {}'.format(f))
with open('%s/%s' % (import_path, f)) as fh:
data = fh.read()
data = re.sub('^To: (.*)',
'To: %s' % dest_email,
data,
flags=re.MULTILINE)
emails[f] = message_from_string(data)
except Exception as exc:
log.error('Error importing email {}'.format(exc))
else:
mode = 'mbox'
emails = mbox(import_path)
user = User.by_local_identifier(dest_email, 'email')
log.info("Processing mode %s" % mode)
for key, data in emails.iteritems():
# Prevent creating message too large to fit in db.
# (should use inject cmd for large messages)
size = len(data.as_string())
if size > max_size:
log.warn("Message too large to fit into db. \
Please, use 'inject' cmd for importing large emails.")
continue
raw = RawMessage.create(data.as_string())
log.debug('Created raw message {}'.format(raw.raw_msg_id))
message = MailMessage(data.as_string())
dice = random()
if dice <= contact_probability:
for participant in message.participants:
try:
ContactLookup.get(user, participant.address)
except NotFound:
log.info('Creating contact %s' % participant.address)
name, domain = participant.address.split('@')
contact_param = NewContact()
contact_param.family_name = name
if participant.address:
e_mail = NewEmail()
e_mail.address = participant.address
contact_param.emails = [e_mail]
Contact.create(user, contact_param)
else:
log.info('No contact associated to raw {} '.format(raw.raw_msg_id))
processor = UserMailDelivery(
user, user.local_identities[0]) # assume one local identity
try:
obj_message = processor.process_raw(raw.raw_msg_id)
except Exception as exc:
if isinstance(exc, DuplicateMessage):
log.info('duplicate message {}, not imported'.format(
raw.raw_msg_id))
else:
log.exception(exc)
else:
log.info('Created message {}'.format(obj_message.message_id))
def process_inbound(self, raw):
"""Process inbound message.
@param raw: a RawMessage object
@rtype: NewMessage
"""
email = MailMessage(raw.raw_data)
new_message = NewInboundMessage()
new_message.raw_msg_id = raw.raw_msg_id
new_message.subject = email.subject
new_message.body_html = email.body_html
new_message.body_plain = email.body_plain
new_message.date = email.date
new_message.size = email.size
new_message.protocol = email.message_protocol
new_message.is_unread = True
new_message.is_draft = False
new_message.is_answered = False
new_message.is_received = True
new_message.importance_level = 0 # XXX tofix on parser
new_message.external_references = email.external_references
participants = []
for p in email.participants:
p.address = p.address.lower()
try:
participant, contact = self.get_participant(email, p)
new_message.participants.append(participant)
participants.append((participant, contact))
except Exception as exc:
log.error(
"process_inbound failed to lookup participant for email {} : {}"
.format(vars(email), exc))
raise exc
if not participants:
raise Exception("no participant found in raw email {}".format(
raw.raw_msg_id))
for a in email.attachments:
attachment = Attachment()
attachment.content_type = a.content_type
attachment.file_name = a.filename
attachment.size = a.size
attachment.mime_boundary = a.mime_boundary
if hasattr(a, "is_inline"):
attachment.is_inline = a.is_inline
new_message.attachments.append(attachment)
# Compute PI !!
conf = Configuration('global').configuration
extractor = InboundMailFeature(email, conf)
extractor.process(self.user, new_message, participants)
# compute user tags
self._get_tags(new_message)
# embed external flags if any
new_message.ext_tags = email.external_flags
if new_message.tags:
log.debug('Resolved tags {}'.format(new_message.tags))
# build discussion_id from lookup_sequence
lookup_sequence, discussion_id = self.lookup_discussion_sequence(
email, new_message)
log.debug('Lookup with sequence {} gives {}'.format(
lookup_sequence, discussion_id))
new_message.discussion_id = discussion_id
# upsert lookup tables
discuss = Discussion(self.user)
discuss.upsert_lookups_for_participants(new_message.participants)
# Format features
new_message.privacy_features = \
marshal_features(new_message.privacy_features)
try:
new_message.validate()
except Exception as exc:
log.error("validation failed with error : « {} » \
for new_message {}[dump : {}]".format(exc, new_message,
vars(new_message)))
raise exc
return new_message