def collection_post(self):
"""API to import an user file (vcard at this time)."""
# need to check by ourself if <file> param is present
# because swagger lib failed to do it correctly :(
try:
self.request.POST.getone("file")
except Exception as exc:
raise ValidationError(exc)
data = self.request.POST['file'].file
try:
parser = VcardParser(data)
except Exception as exc:
log.exception('Exception during vcard file parsing %r' % exc)
raise ValidationError(exc)
try:
new_contacts = parser.parse()
except Exception as exc:
log.error('Syntax error: {}'.format(exc))
raise ValidationError(exc)
try:
for contact in new_contacts:
CoreContact.create(self.user, contact.contact)
except Exception as exc:
log.error(
'File valid but we can create the new contact: {}'.format(exc))
raise Unprocessable(detail=exc.message)
return Response(status=200)
def import_vcard(username, directory, file_vcard, **kwargs):
from caliopen_main.contact.core import Contact as CoreContact
from caliopen_main.user.core.user import User as CoreUser
from caliopen_main.contact.parsers import VcardParser
new_contacts = []
if directory:
files = [
f for f in os.listdir(directory)
if os.path.isfile(os.path.join(directory, f))
]
for f in files:
ext = f.split('.')[-1]
if ext == 'vcard' or ext == 'vcf':
file = '{directory}/{file}'.format(directory=directory, file=f)
parser = VcardParser(file)
new_contacts.extend(parser.parse())
else:
log.warn("Not valid file extension for vcard %s" % f)
if file_vcard:
parser = VcardParser(file_vcard)
new_contacts = parser.parse()
user = CoreUser.by_name(username)
for contact in new_contacts:
CoreContact.create(user, contact.contact)
def import_vcard(username, directory, file_vcard, **kwargs):
from caliopen_main.contact.core import Contact as CoreContact
from caliopen_main.user.core.user import User as CoreUser
from caliopen_main.contact.parsers import VcardParser
new_contacts = []
if directory:
files = [f for f in os.listdir(directory) if
os.path.isfile(os.path.join(directory, f))]
for f in files:
ext = f.split('.')[-1]
if ext == 'vcard' or ext == 'vcf':
file = '{directory}/{file}'.format(directory=directory, file=f)
parser = VcardParser(file)
new_contacts.extend(parser.parse())
else:
log.warn("Not valid file extension for vcard %s" % f)
if file_vcard:
parser = VcardParser(file_vcard)
new_contacts = parser.parse()
user = CoreUser.by_name(username)
for contact in new_contacts:
CoreContact.create(user, contact.contact)
def collection_post(self):
# need to check by ourself if <file> param is present
# because swagger lib failed to do it correctly :(
try:
self.request.POST.getone("file")
except Exception as exc:
raise ValidationError(exc)
data = self.request.POST['file'].file
vcards = read_file(data, False)
try:
new_contacts = parse_vcards(vcards)
except Exception as exc:
log.error('Syntax error: {}'.format(exc))
raise ValidationError(exc)
try:
for i in new_contacts:
CoreContact.create(self.user, i)
except Exception as exc:
log.error(
'File valid but we can create the new contact: {}'.format(exc))
raise Unprocessable(exc)
return Response(status=200)
def collection_post(self):
"""API to import an user file (vcard at this time)."""
# need to check by ourself if <file> param is present
# because swagger lib failed to do it correctly :(
try:
self.request.POST.getone("file")
except Exception as exc:
raise ValidationError(exc)
data = self.request.POST['file'].file
try:
parser = VcardParser(data)
except Exception as exc:
log.exception('Exception during vcard file parsing %r' % exc)
raise ValidationError(exc)
try:
new_contacts = parser.parse()
except Exception as exc:
log.error('Syntax error: {}'.format(exc))
raise ValidationError(exc)
try:
for contact in new_contacts:
CoreContact.create(self.user, contact.contact)
except Exception as exc:
log.error(
'File valid but we can create the new contact: {}'.format(exc))
raise Unprocessable(detail=exc.message)
return Response(status=200)
def _create(self, contact_id, params, add_func, return_obj):
"""Create sub object from param using add_func."""
contact = CoreContact.get(self.user, contact_id)
created = getattr(contact, add_func)(params)
log.debug('Created object {} for contact {}'.format(
created.address_id, contact.contact_id))
return return_obj.build(created).serialize()
def contact(self):
"""User is a contact."""
if self.contact_id is None:
return None
try:
return CoreContact.get(self, self.contact_id)
except NotFound:
log.warn("contact {} not found for user {}".format(
self.contact_id, self.user_id))
return None
def contact(self):
"""User is a contact."""
if self.contact_id is None:
return None
try:
return CoreContact.get(self, self.contact_id)
except NotFound:
log.warn("contact {} not found for user {}".
format(self.contact_id, self.user_id))
return None
def get_participant(self, message, participant):
"""Try to find a related contact and return a Participant instance."""
p = Participant()
p.address = participant.address
p.type = participant.type
p.label = participant.label
p.protocol = message.message_type
log.debug('Will lookup contact {} for user {}'.format(
participant.address, self.user.user_id))
c = Contact.lookup(self.user, participant.address)
if c:
p.contact_ids = [c.contact_id]
return p, c
def collection_post(self):
"""Create a new contact from json post data structure."""
data = self.request.json
contact_param = NewContactParam(data)
try:
contact_param.validate()
except Exception as exc:
raise ValidationError(exc)
contact = CoreContact.create(self.user, contact_param)
contact_url = self.request.route_path('contact',
contact_id=contact.contact_id)
self.request.response.location = contact_url.encode('utf-8')
# XXX return a Location to get contact not send it direct
return {'location': contact_url}
def import_vcard(username, directory, file_vcard, **kwargs):
from caliopen_main.contact.parameters import NewContact
from caliopen_main.contact.core import Contact as CoreContact
from caliopen_main.user.core.user import User as CoreUser
from caliopen_main.contact.parsers import parse_vcards
from caliopen_main.contact.parsers import read_file, read_directory
vcards = []
if directory:
vcards = read_directory(directory)
if file_vcard:
vcards = read_file(file_vcard, True)
user = CoreUser.by_name(username)
new_contacts = parse_vcards(vcards)
for i in new_contacts:
CoreContact.create(user, i)
def collection_get(self):
filter_params = {'limit': self.get_limit(),
'offset': self.get_offset()}
log.debug('Filter parameters {}'.format(filter_params))
results = CoreContact._model_class.search(self.user, **filter_params)
data = []
for item in results:
try:
c = ReturnContact.build(
CoreContact.get(self.user, item.contact_id)). \
serialize()
data.append(c)
except Exception as exc:
log.error("unable to serialize contact : {}".format(exc))
return {'contacts': data, 'total': results.hits.total}
def collection_get(self):
filter_params = {'limit': self.get_limit(),
'offset': self.get_offset()}
log.debug('Filter parameters {}'.format(filter_params))
results = CoreContact._model_class.search(self.user, **filter_params)
data = []
for item in results:
try:
c = ReturnContact.build(
CoreContact.get(self.user, item.contact_id)). \
serialize()
data.append(c)
except Exception as exc:
log.error("unable to serialize contact : {}".format(exc))
return {'contacts': data, 'total': results.hits.total}
def get_participant(self, message, participant):
"""Try to find a related contact and return a Participant instance."""
p = Participant()
p.address = participant.address
p.type = participant.type
p.label = participant.label
p.protocol = message.message_protocol
log.debug('Will lookup contact {} for user {}'.
format(participant.address, self.user.user_id))
c = Contact.lookup(self.user, participant.address)
if c:
p.contact_ids = [c.contact_id]
else:
if p.address == self.identity.identifier:
p.contact_ids = [self.user.contact_id]
return p, c
def collection_post(self):
"""Create a new contact from json post data structure."""
data = self.request.json
contact_param = NewContactParam(data)
try:
contact_param.validate()
if hasattr(contact_param, "tags") and contact_param.tags:
raise ValidationError(
"adding tags through parent object is forbidden")
except Exception as exc:
raise ValidationError(exc)
contact = CoreContact.create(self.user, contact_param)
contact_url = self.request.route_path('contact',
contact_id=contact.contact_id)
self.request.response.location = contact_url.encode('utf-8')
# XXX return a Location to get contact not send it direct
return {'location': contact_url}
def get_participant(self, message, participant):
"""Try to find a related contact and return a Participant instance."""
p = Participant()
p.address = participant.address.lower()
p.type = participant.type
p.label = participant.label
p.protocol = message.message_protocol
log.debug('Will lookup contact {} for user {}'.format(
participant.address, self.user.user_id))
try:
c = Contact.lookup(self.user, participant.address)
except Exception as exc:
log.error(
"Contact lookup failed in get_participant for participant {} : {}"
.format(vars(participant), exc))
raise exc
if c:
p.contact_ids = [c.contact_id]
else:
if p.address == self.identity.identifier and self.user.contact_id:
p.contact_ids = [self.user.contact_id]
return p, c
def __init__(self, request):
self.request = request
self.user = request.authenticated_userid
contact_id = self.request.matchdict.get('contact_id')
self.contact = CoreContact.get(self.user, contact_id)
def _delete(self, relation_id, delete_func):
"""Delete sub object relation_id using delete_fund."""
contact_id = self.request.validated['contact_id']
contact = CoreContact.get(self.user, contact_id)
return getattr(contact, delete_func)(relation_id)
def import_email(email, import_path, format, contact_probability, **kwargs):
"""Import emails for an user."""
from caliopen_main.user.core import User
from caliopen_main.contact.core import Contact, ContactLookup
from caliopen_main.message.parsers.mail import MailMessage
from caliopen_main.contact.parameters import NewContact, NewEmail
from caliopen_nats.delivery import UserMailDelivery
from caliopen_main.message.core import RawMessage
from caliopen_storage.config import Configuration
max_size = int(Configuration("global").get("object_store.db_size_limit"))
if 'to' in kwargs and kwargs['to']:
dest_email = kwargs['to']
else:
dest_email = email
if format == 'maildir':
if dest_email != email:
raise Exception('Cannot change To email using maildir format')
emails = Maildir(import_path, factory=message_from_file)
mode = 'maildir'
else:
if os.path.isdir(import_path):
mode = 'mbox_directory'
emails = {}
files = [
f for f in os.listdir(import_path)
if os.path.isfile(os.path.join(import_path, f))
]
for f in files:
try:
log.debug('Importing mail from file {}'.format(f))
with open('%s/%s' % (import_path, f)) as fh:
data = fh.read()
data = re.sub('^To: (.*)',
'To: %s' % dest_email,
data,
flags=re.MULTILINE)
emails[f] = message_from_string(data)
except Exception as exc:
log.error('Error importing email {}'.format(exc))
else:
mode = 'mbox'
emails = mbox(import_path)
user = User.by_local_identifier(dest_email, 'email')
log.info("Processing mode %s" % mode)
for key, data in emails.iteritems():
# Prevent creating message too large to fit in db.
# (should use inject cmd for large messages)
size = len(data.as_string())
if size > max_size:
log.warn("Message too large to fit into db. \
Please, use 'inject' cmd for importing large emails.")
continue
raw = RawMessage.create(data.as_string())
log.debug('Created raw message {}'.format(raw.raw_msg_id))
message = MailMessage(data.as_string())
dice = random()
if dice <= contact_probability:
for participant in message.participants:
try:
ContactLookup.get(user, participant.address)
except NotFound:
log.info('Creating contact %s' % participant.address)
name, domain = participant.address.split('@')
contact_param = NewContact()
contact_param.family_name = name
if participant.address:
e_mail = NewEmail()
e_mail.address = participant.address
contact_param.emails = [e_mail]
Contact.create(user, contact_param)
else:
log.info('No contact associated to raw {} '.format(raw.raw_msg_id))
processor = UserMailDelivery(
user, user.local_identities[0]) # assume one local identity
try:
obj_message = processor.process_raw(raw.raw_msg_id)
except Exception as exc:
if isinstance(exc, DuplicateMessage):
log.info('duplicate message {}, not imported'.format(
raw.raw_msg_id))
else:
log.exception(exc)
else:
log.info('Created message {}'.format(obj_message.message_id))
def create(cls, new_user):
"""Create a new user.
@param: new_user is a parameters/user.py.NewUser object
# 1.check username regex
# 2.check username is not in reserved_name table
# 3.check recovery email validity (TODO : check if email is not within
# the current Caliopen's instance)
# 4.check username availability
# 5.add username to user cassa user_name table (to block the
# availability)
# 6.check password strength (and regex?)
# then
# create user and linked contact
"""
def rollback_username_storage(username):
UserName.get(username).delete()
# 0. check for user email white list and max number of users
cls._check_whitelistes(new_user)
cls._check_max_users()
# 1.
try:
validators.is_valid_username(new_user.name)
except SyntaxError:
raise ValueError("Malformed username")
# 2.
try:
ReservedName.get(new_user.name)
raise ValueError('Reserved user name')
except NotFound:
pass
user_id = uuid.uuid4()
# 3.
if not new_user.recovery_email:
raise ValueError("Missing recovery email")
try:
cls.validate_recovery_email(new_user.recovery_email)
except Exception as exc:
log.info("recovery email failed validation : {}".format(exc))
raise ValueError(exc)
# 4. & 5.
if User.is_username_available(new_user.name.lower()):
# save username immediately to prevent concurrent creation
UserName.create(name=new_user.name.lower(), user_id=user_id)
# NB : need to rollback this username creation if the below
# User creation failed for any reason
else:
raise ValueError("Username already exist")
# 6.
try:
user_inputs = [
new_user.name.encode("utf-8"),
new_user.recovery_email.encode("utf-8")
]
# TODO: add contact inputs if any
password_strength = zxcvbn(new_user.password,
user_inputs=user_inputs)
privacy_features = {
"password_strength": str(password_strength["score"])
}
passwd = new_user.password.encode('utf-8')
new_user.password = bcrypt.hashpw(passwd, bcrypt.gensalt())
except Exception as exc:
log.exception(exc)
rollback_username_storage(new_user.name)
raise exc
try:
new_user.validate() # schematic model validation
except Exception as exc:
rollback_username_storage(new_user.name)
log.info("schematics validation error: {}".format(exc))
raise ValueError("new user malformed")
try:
recovery = new_user.recovery_email
if hasattr(new_user, "contact"):
family_name = new_user.contact.family_name
given_name = new_user.contact.given_name
else:
family_name = ""
given_name = ""
# XXX PI compute
pi = PIModel()
pi.technic = 0
pi.comportment = 0
pi.context = 0
pi.version = 0
shard_id = allocate_user_shard(user_id)
core = super(User, cls).create(
user_id=user_id,
name=new_user.name,
password=new_user.password,
recovery_email=recovery,
params=new_user.params,
date_insert=datetime.datetime.now(tz=pytz.utc),
privacy_features=privacy_features,
pi=pi,
family_name=family_name,
given_name=given_name,
shard_id=shard_id)
except Exception as exc:
log.info(exc)
rollback_username_storage(new_user.name)
raise exc
# **** operations below do not raise fatal error and rollback **** #
# Setup index
setup_index(core)
# Setup others entities related to user
setup_system_tags(core)
setup_settings(core, new_user.settings)
UserRecoveryEmail.create(recovery_email=recovery, user_id=user_id)
# Add a default local identity on a default configured domain
default_domain = Configuration('global').get('default_domain')
default_local_id = '{}@{}'.format(core.name, default_domain)
if not core.add_local_identity(default_local_id):
log.warn('Impossible to create default local identity {}'.format(
default_local_id))
# save and index linked contact
if hasattr(new_user, "contact"):
# add local email to contact
local_mail = NewEmail()
local_mail.address = default_local_id
new_user.contact.emails.append(local_mail)
# create default contact for user
contact = CoreContact.create(core, new_user.contact)
core.model.contact_id = contact.contact_id
log.info("contact id {} for new user {}".format(
contact.contact_id, core.user_id))
else:
log.error("missing contact in new_user params for user {}. "
"Can't create related tables".format(core.user_id))
core.save()
return core
def import_email(email, import_path, format, contact_probability, **kwargs):
"""Import emails for an user."""
from caliopen_main.user.core import User
from caliopen_main.contact.core import Contact, ContactLookup
from caliopen_main.message.parsers.mail import MailMessage
from caliopen_main.contact.parameters import NewContact, NewEmail
from caliopen_nats.delivery import UserMessageDelivery
from caliopen_main.message.core import RawMessage
from caliopen_storage.config import Configuration
max_size = int(Configuration("global").get("object_store.db_size_limit"))
if format == 'maildir':
emails = Maildir(import_path, factory=message_from_file)
mode = 'maildir'
else:
if os.path.isdir(import_path):
mode = 'mbox_directory'
emails = {}
files = [
f for f in os.listdir(import_path)
if os.path.isfile(os.path.join(import_path, f))
]
for f in files:
try:
log.debug('Importing mail from file {}'.format(f))
with open('%s/%s' % (import_path, f)) as fh:
emails[f] = message_from_file(fh)
except Exception as exc:
log.error('Error importing email {}'.format(exc))
else:
mode = 'mbox'
emails = mbox(import_path)
user = User.by_local_identity(email)
log.info("Processing mode %s" % mode)
for key, data in emails.iteritems():
# Prevent creating message too large to fit in db.
# (should use inject cmd for large messages)
size = len(data.as_string())
if size > max_size:
log.warn("Message too large to fit into db. \
Please, use 'inject' cmd for importing large emails.")
continue
raw = RawMessage.create(data.as_string())
log.debug('Created raw message {}'.format(raw.raw_msg_id))
message = MailMessage(data.as_string())
dice = random()
if dice <= contact_probability:
for participant in message.participants:
try:
ContactLookup.get(user, participant.address)
except NotFound:
log.info('Creating contact %s' % participant.address)
name, domain = participant.address.split('@')
contact_param = NewContact()
contact_param.family_name = name
if participant.address:
e_mail = NewEmail()
e_mail.address = participant.address
contact_param.emails = [e_mail]
Contact.create(user, contact_param)
log.info('No contact associated to raw {} '.format(raw.raw_msg_id))
processor = UserMessageDelivery(user)
obj_message = processor.process_raw(raw.raw_msg_id)
log.info('Created message {}'.format(obj_message.message_id))
def import_email(email, import_path, format, contact_probability,
**kwargs):
"""Import emails for an user."""
from caliopen_main.user.core import User, UserIdentity
from caliopen_main.contact.core import Contact, ContactLookup
from caliopen_main.message.parsers.mail import MailMessage
from caliopen_main.contact.parameters import NewContact, NewEmail
from caliopen_nats.delivery import UserMailDelivery
from caliopen_main.message.core import RawMessage
from caliopen_storage.config import Configuration
max_size = int(Configuration("global").get("object_store.db_size_limit"))
if 'to' in kwargs and kwargs['to']:
dest_email = kwargs['to']
else:
dest_email = email
if format == 'maildir':
if dest_email != email:
raise Exception('Cannot change To email using maildir format')
emails = Maildir(import_path, factory=message_from_file)
mode = 'maildir'
else:
if os.path.isdir(import_path):
mode = 'mbox_directory'
emails = {}
files = [f for f in os.listdir(import_path) if
os.path.isfile(os.path.join(import_path, f))]
for f in files:
try:
log.debug('Importing mail from file {}'.format(f))
with open('%s/%s' % (import_path, f)) as fh:
data = fh.read()
data = re.sub('^To: (.*)', 'To: %s' % dest_email,
data, flags=re.MULTILINE)
emails[f] = message_from_string(data)
except Exception as exc:
log.error('Error importing email {}'.format(exc))
else:
mode = 'mbox'
emails = mbox(import_path)
user = User.by_local_identifier(dest_email, 'email')
log.info("Processing mode %s" % mode)
for key, data in emails.iteritems():
# Prevent creating message too large to fit in db.
# (should use inject cmd for large messages)
size = len(data.as_string())
if size > max_size:
log.warn("Message too large to fit into db. \
Please, use 'inject' cmd for importing large emails.")
continue
raw = RawMessage.create(data.as_string())
log.debug('Created raw message {}'.format(raw.raw_msg_id))
message = MailMessage(data.as_string())
dice = random()
if dice <= contact_probability:
for participant in message.participants:
try:
ContactLookup.get(user, participant.address)
except NotFound:
log.info('Creating contact %s' % participant.address)
name, domain = participant.address.split('@')
contact_param = NewContact()
contact_param.family_name = name
if participant.address:
e_mail = NewEmail()
e_mail.address = participant.address
contact_param.emails = [e_mail]
Contact.create(user, contact_param)
log.info('No contact associated to raw {} '.format(raw.raw_msg_id))
processor = UserMailDelivery(user,
user.local_identities[0]) # assume one local identity
try:
obj_message = processor.process_raw(raw.raw_msg_id)
except Exception as exc:
log.exception(exc)
else:
log.info('Created message {}'.format(obj_message.message_id))
