def get(self):
# XXX how to check privacy_index ?
raw_msg_id = self.request.matchdict.get('raw_msg_id')
raw = RawMessage.get_for_user(self.user.user_id, raw_msg_id)
if raw:
return raw.raw_data
raise ResourceNotFound('No such message')
def get(self):
# XXX how to check privacy_index ?
raw_msg_id = self.request.matchdict.get('raw_msg_id')
raw = RawMessage.get_for_user(self.user.user_id, raw_msg_id)
if raw:
return raw.raw_data
raise ResourceNotFound('No such message')
def process_raw(self, raw_msg_id):
"""Process a raw message for an user, ie makes it a rich 'message'."""
raw = RawMessage.get(raw_msg_id)
if not raw:
log.error('Raw message <{}> not found'.format(raw_msg_id))
raise NotFound
log.debug('Retrieved raw message {}'.format(raw_msg_id))
message = self.qualifier.process_inbound(raw)
if message.external_msg_id:
external_refs = Merl._model_class.filter(
user_id=self.user.user_id,
external_msg_id=message.external_msg_id)
if external_refs:
msg = external_refs[0]
# message already imported, should update it with identity_id ?
obj = Message(user=self.user, message_id=msg.message_id)
if str(msg.identity_id) != self.identity.identity_id:
obj.get_db()
obj.unmarshall_db()
obj.user_identities.append(self.identity.identity_id)
obj.marshall_db()
obj.save_db()
obj.marshall_index()
obj.save_index()
Merl.create(self.user,
external_msg_id=msg.external_msg_id,
identity_id=self.identity.identity_id,
message_id=msg.message_id)
raise DuplicateObject(DUPLICATE_MESSAGE_EXC)
else:
log.warn('Message without external message_id for raw {}'.format(
raw.raw_msg_id))
# store and index Message
obj = Message(user=self.user)
obj.unmarshall_dict(message.to_native())
obj.user_id = uuid.UUID(self.user.user_id)
obj.user_identities = [uuid.UUID(self.identity.identity_id)]
obj.message_id = uuid.uuid4()
obj.date_insert = datetime.datetime.now(tz=pytz.utc)
obj.date_sort = obj.date_insert
obj.marshall_db()
obj.save_db()
obj.marshall_index()
obj.save_index()
if message.external_msg_id:
# store external_msg_id in lookup table
# but do not abort if it failed
try:
Merl.create(self.user,
external_msg_id=obj.external_msg_id,
identity_id=obj.user_identity,
message_id=obj.message_id)
except Exception as exc:
log.exception("UserMessageDelivery failed "
"to store message_external_ref : {}".format(exc))
return obj
def process_raw(self, raw_msg_id):
"""Process a raw message for an user, ie makes it a rich 'message'."""
raw = RawMessage.get(raw_msg_id)
if not raw:
log.error('Raw message <{}> not found'.format(raw_msg_id))
raise NotFound
log.debug('Retrieved raw message {}'.format(raw_msg_id))
message = self.qualifier.process_inbound(raw)
external_refs = ModelMessageExternalRefLookup.filter(
user_id=self.user.user_id,
external_msg_id=message.external_msg_id)
if external_refs:
# message already imported, update it with identity_id if needed
for external_ref in external_refs:
obj = Message(user=self.user,
message_id=external_ref.message_id)
if str(external_ref.identity_id) != self.identity.identity_id:
obj.get_db()
obj.unmarshall_db()
obj.user_identities.append(self.identity.identity_id)
obj.marshall_db()
obj.save_db()
obj.marshall_index()
obj.save_index()
MessageExternalRefLookup.create(self.user,
external_msg_id=external_ref.external_msg_id,
identity_id=self.identity.identity_id,
message_id=external_ref.message_id)
raise DuplicateObject(DUPLICATE_MESSAGE_EXC)
# store and index Message
obj = Message(user=self.user)
obj.unmarshall_dict(message.to_native())
obj.user_id = uuid.UUID(self.user.user_id)
obj.user_identities = [uuid.UUID(self.identity.identity_id)]
obj.message_id = uuid.uuid4()
obj.date_insert = datetime.datetime.now(tz=pytz.utc)
obj.date_sort = obj.date_insert
obj.marshall_db()
obj.save_db()
obj.marshall_index()
obj.save_index()
# store external_msg_id in lookup table
# but do not abort if it failed
try:
MessageExternalRefLookup.create(self.user,
external_msg_id=obj.external_msg_id,
identity_id=obj.user_identity,
message_id=obj.message_id)
except Exception as exc:
log.exception("UserMessageDelivery failed "
"to store message_external_ref : {}".format(exc))
return obj
def process_raw(self, raw_msg_id):
"""Process a raw message for an user, ie makes it a rich 'message'."""
raw = RawMessage.get(raw_msg_id)
if not raw:
log.error('Raw message <{}> not found'.format(raw_msg_id))
raise NotFound
log.debug('Retrieved raw message {}'.format(raw_msg_id))
qualifier = UserMessageQualifier(self.user)
message = qualifier.process_inbound(raw)
# store and index message
obj = Message(self.user)
obj.unmarshall_dict(message.to_native())
obj.user_id = uuid.UUID(self.user.user_id)
obj.message_id = uuid.uuid4()
obj.date_insert = datetime.datetime.now(tz=pytz.utc)
obj.marshall_db()
obj.save_db()
obj.marshall_index()
obj.save_index()
return obj
def import_email(email, import_path, format, contact_probability, **kwargs):
"""Import emails for an user."""
from caliopen_main.user.core import User
from caliopen_main.contact.core import Contact, ContactLookup
from caliopen_main.message.parsers.mail import MailMessage
from caliopen_main.contact.parameters import NewContact, NewEmail
from caliopen_nats.delivery import UserMessageDelivery
from caliopen_main.message.core import RawMessage
from caliopen_storage.config import Configuration
max_size = int(Configuration("global").get("object_store.db_size_limit"))
if format == 'maildir':
emails = Maildir(import_path, factory=message_from_file)
mode = 'maildir'
else:
if os.path.isdir(import_path):
mode = 'mbox_directory'
emails = {}
files = [
f for f in os.listdir(import_path)
if os.path.isfile(os.path.join(import_path, f))
]
for f in files:
try:
log.debug('Importing mail from file {}'.format(f))
with open('%s/%s' % (import_path, f)) as fh:
emails[f] = message_from_file(fh)
except Exception as exc:
log.error('Error importing email {}'.format(exc))
else:
mode = 'mbox'
emails = mbox(import_path)
user = User.by_local_identity(email)
log.info("Processing mode %s" % mode)
for key, data in emails.iteritems():
# Prevent creating message too large to fit in db.
# (should use inject cmd for large messages)
size = len(data.as_string())
if size > max_size:
log.warn("Message too large to fit into db. \
Please, use 'inject' cmd for importing large emails.")
continue
raw = RawMessage.create(data.as_string())
log.debug('Created raw message {}'.format(raw.raw_msg_id))
message = MailMessage(data.as_string())
dice = random()
if dice <= contact_probability:
for participant in message.participants:
try:
ContactLookup.get(user, participant.address)
except NotFound:
log.info('Creating contact %s' % participant.address)
name, domain = participant.address.split('@')
contact_param = NewContact()
contact_param.family_name = name
if participant.address:
e_mail = NewEmail()
e_mail.address = participant.address
contact_param.emails = [e_mail]
Contact.create(user, contact_param)
log.info('No contact associated to raw {} '.format(raw.raw_msg_id))
processor = UserMessageDelivery(user)
obj_message = processor.process_raw(raw.raw_msg_id)
log.info('Created message {}'.format(obj_message.message_id))
def import_email(email, import_path, format, contact_probability, **kwargs):
"""Import emails for an user."""
from caliopen_main.user.core import User
from caliopen_main.contact.core import Contact, ContactLookup
from caliopen_main.message.parsers.mail import MailMessage
from caliopen_main.contact.parameters import NewContact, NewEmail
from caliopen_nats.delivery import UserMailDelivery
from caliopen_main.message.core import RawMessage
from caliopen_storage.config import Configuration
max_size = int(Configuration("global").get("object_store.db_size_limit"))
if 'to' in kwargs and kwargs['to']:
dest_email = kwargs['to']
else:
dest_email = email
if format == 'maildir':
if dest_email != email:
raise Exception('Cannot change To email using maildir format')
emails = Maildir(import_path, factory=message_from_file)
mode = 'maildir'
else:
if os.path.isdir(import_path):
mode = 'mbox_directory'
emails = {}
files = [
f for f in os.listdir(import_path)
if os.path.isfile(os.path.join(import_path, f))
]
for f in files:
try:
log.debug('Importing mail from file {}'.format(f))
with open('%s/%s' % (import_path, f)) as fh:
data = fh.read()
data = re.sub('^To: (.*)',
'To: %s' % dest_email,
data,
flags=re.MULTILINE)
emails[f] = message_from_string(data)
except Exception as exc:
log.error('Error importing email {}'.format(exc))
else:
mode = 'mbox'
emails = mbox(import_path)
user = User.by_local_identifier(dest_email, 'email')
log.info("Processing mode %s" % mode)
for key, data in emails.iteritems():
# Prevent creating message too large to fit in db.
# (should use inject cmd for large messages)
size = len(data.as_string())
if size > max_size:
log.warn("Message too large to fit into db. \
Please, use 'inject' cmd for importing large emails.")
continue
raw = RawMessage.create(data.as_string())
log.debug('Created raw message {}'.format(raw.raw_msg_id))
message = MailMessage(data.as_string())
dice = random()
if dice <= contact_probability:
for participant in message.participants:
try:
ContactLookup.get(user, participant.address)
except NotFound:
log.info('Creating contact %s' % participant.address)
name, domain = participant.address.split('@')
contact_param = NewContact()
contact_param.family_name = name
if participant.address:
e_mail = NewEmail()
e_mail.address = participant.address
contact_param.emails = [e_mail]
Contact.create(user, contact_param)
else:
log.info('No contact associated to raw {} '.format(raw.raw_msg_id))
processor = UserMailDelivery(
user, user.local_identities[0]) # assume one local identity
try:
obj_message = processor.process_raw(raw.raw_msg_id)
except Exception as exc:
if isinstance(exc, DuplicateMessage):
log.info('duplicate message {}, not imported'.format(
raw.raw_msg_id))
else:
log.exception(exc)
else:
log.info('Created message {}'.format(obj_message.message_id))
def import_email(email, import_path, format, contact_probability,
**kwargs):
"""Import emails for an user."""
from caliopen_main.user.core import User, UserIdentity
from caliopen_main.contact.core import Contact, ContactLookup
from caliopen_main.message.parsers.mail import MailMessage
from caliopen_main.contact.parameters import NewContact, NewEmail
from caliopen_nats.delivery import UserMailDelivery
from caliopen_main.message.core import RawMessage
from caliopen_storage.config import Configuration
max_size = int(Configuration("global").get("object_store.db_size_limit"))
if 'to' in kwargs and kwargs['to']:
dest_email = kwargs['to']
else:
dest_email = email
if format == 'maildir':
if dest_email != email:
raise Exception('Cannot change To email using maildir format')
emails = Maildir(import_path, factory=message_from_file)
mode = 'maildir'
else:
if os.path.isdir(import_path):
mode = 'mbox_directory'
emails = {}
files = [f for f in os.listdir(import_path) if
os.path.isfile(os.path.join(import_path, f))]
for f in files:
try:
log.debug('Importing mail from file {}'.format(f))
with open('%s/%s' % (import_path, f)) as fh:
data = fh.read()
data = re.sub('^To: (.*)', 'To: %s' % dest_email,
data, flags=re.MULTILINE)
emails[f] = message_from_string(data)
except Exception as exc:
log.error('Error importing email {}'.format(exc))
else:
mode = 'mbox'
emails = mbox(import_path)
user = User.by_local_identifier(dest_email, 'email')
log.info("Processing mode %s" % mode)
for key, data in emails.iteritems():
# Prevent creating message too large to fit in db.
# (should use inject cmd for large messages)
size = len(data.as_string())
if size > max_size:
log.warn("Message too large to fit into db. \
Please, use 'inject' cmd for importing large emails.")
continue
raw = RawMessage.create(data.as_string())
log.debug('Created raw message {}'.format(raw.raw_msg_id))
message = MailMessage(data.as_string())
dice = random()
if dice <= contact_probability:
for participant in message.participants:
try:
ContactLookup.get(user, participant.address)
except NotFound:
log.info('Creating contact %s' % participant.address)
name, domain = participant.address.split('@')
contact_param = NewContact()
contact_param.family_name = name
if participant.address:
e_mail = NewEmail()
e_mail.address = participant.address
contact_param.emails = [e_mail]
Contact.create(user, contact_param)
log.info('No contact associated to raw {} '.format(raw.raw_msg_id))
processor = UserMailDelivery(user,
user.local_identities[0]) # assume one local identity
try:
obj_message = processor.process_raw(raw.raw_msg_id)
except Exception as exc:
log.exception(exc)
else:
log.info('Created message {}'.format(obj_message.message_id))