def process_key_discovery(self, msg, payload): """Discover public keys related to a new contact identifier.""" if 'user_id' not in payload or 'contact_id' not in payload: raise Exception('Invalid contact_update structure') user = User.get(payload['user_id']) contact = Contact(user.user_id, contact_id=payload['contact_id']) contact.get_db() contact.unmarshall_db() manager = ContactPublicKeyManager() founds = [] for ident in payload.get('emails', []): log.info('Process email identity {0}'.format(ident['address'])) discovery = manager.process_identity(user, contact, ident['address'], 'email') if discovery: founds.append(discovery) for ident in payload.get('identities', []): log.info('Process identity {0}:{1}'. format(ident['type'], ident['name'])) discovery = manager.process_identity(user, contact, ident['name'], ident['type']) if discovery: founds.append(discovery) if founds: log.info('Found %d results' % len(founds)) self._process_results(user, contact, founds)
def process_raw(self, msg, payload): """Process an inbound raw message.""" nats_error = { 'error': '', 'message': 'inbound email message process failed' } nats_success = { 'message': 'OK : inbound email message proceeded' } try: user = User.get(payload['user_id']) identity = UserIdentity.get(user, payload['identity_id']) deliver = UserMailDelivery(user, identity) new_message = deliver.process_raw(payload['message_id']) nats_success['message_id'] = str(new_message.message_id) self.natsConn.publish(msg.reply, json.dumps(nats_success)) except DuplicateObject: log.info("Message already imported : {}".format(payload)) nats_success['message_id'] = str(payload['message_id']) nats_success['message'] = 'raw message already imported' self.natsConn.publish(msg.reply, json.dumps(nats_success)) except Exception as exc: # TODO: handle abort exception and report it as special case log.error("deliver process failed for raw {}: {}". format(payload, exc)) nats_error['error'] = str(exc.message) self.natsConn.publish(msg.reply, json.dumps(nats_error)) return exc
def create_user(**kwargs): """Create user in Caliopen instance.""" from caliopen_main.user.core import User from caliopen_main.user.parameters import NewUser from caliopen_main.contact.parameters import NewContact from caliopen_main.contact.parameters import NewEmail # Fill core registry from caliopen_main.message.objects.message import Message param = NewUser() param.name = kwargs['email'] if '@' in param.name: username, domain = param.name.split('@') param.name = username # Monkey patch configuration local_domain with provided value conf = Configuration('global').configuration conf['default_domain'] = domain param.password = kwargs['password'] param.recovery_email = u'{}@recovery-caliopen.local'.format(param.name) contact = NewContact() contact.given_name = kwargs.get('given_name') contact.family_name = kwargs.get('family_name') email = NewEmail() email.address = param.recovery_email contact.emails = [email] param.contact = contact user = User.create(param) log.info('User %s (%s) created' % (user.user_id, user.name))
def user(self): """Return user related to this object.""" from caliopen_main.user.core import User if not self._user: self._user = User.get(self.user_id) return self._user
def process_update(self, msg, payload): """Process a contact update message.""" # XXX validate payload structure if 'user_id' not in payload or 'contact_id' not in payload: raise Exception('Invalid contact_update structure') user = User.get(payload['user_id']) contact = Contact(user, contact_id=payload['contact_id']) contact.get_db() contact.unmarshall_db() qualifier = ContactMessageQualifier(user) log.info('Will process update for contact {0} of user {1}'. format(contact.contact_id, user.user_id)) # TODO: (re)discover GPG keys qualifier.process(contact)
def basic_compute(username, job, ** kwargs): """Import emails for an user.""" from caliopen_main.user.core import User from caliopen_main.contact.objects import Contact from caliopen_pi.qualifiers import ContactMessageQualifier user = User.by_name(username) qualifier = ContactMessageQualifier(user) contacts = Contact.list_db(user.user_id) if job == 'contact_privacy': for contact in contacts: log.info('Processing contact {0}'.format(contact.contact_id)) qualifier.process(contact)
def no_such_user(request): """Validator that an user does not exist.""" username = request.swagger_data['user']['username'] if not User.is_username_available(username): raise AuthenticationError('User already exist')
def _load_user(self): if self._user: return self._user = User.get(self.user_id)
def _check_discussion_consistency(self, user_id): from caliopen_main.message.objects.message import Message new_discussion = False if not hasattr(self, 'discussion_id') or self.discussion_id == "" \ or self.discussion_id is None: # no discussion_id provided. Try to find one with draft's parent_id # or create new discussion if hasattr(self, 'parent_id') \ and self.parent_id is not None \ and self.parent_id != "": parent_msg = Message(user_id, message_id=self.parent_id) try: parent_msg.get_db() parent_msg.unmarshall_db() except NotFound: raise err.PatchError(message="parent message not found") self.discussion_id = parent_msg.discussion_id else: user = User.get(user_id) discussion = Discussion.create_from_message(user, self) self.discussion_id = discussion.discussion_id new_discussion = True if not new_discussion: dim = DIM(user_id) d_id = self.discussion_id last_message = dim.get_last_message(d_id, -10, 10, True) if last_message == {}: raise err.PatchError( message='No such discussion {}'.format(d_id)) is_a_reply = (str(last_message.message_id) != str(self.message_id)) if is_a_reply: # check participants consistency if hasattr(self, "participants") and len(self.participants) > 0: participants = [p['address'] for p in self.participants] last_msg_participants = [ p['address'] for p in last_message.participants ] if len(participants) != len(last_msg_participants): raise err.PatchError( message="list of participants " "is not consistent for this discussion") participants.sort() last_msg_participants.sort() for i, participant in enumerate(participants): if participant != last_msg_participants[i]: raise err.PatchConflict( message="list of participants " "is not consistent for this discussion") else: self.build_participants_for_reply(user_id) # check parent_id consistency if 'parent_id' in self and self.parent_id != "" \ and self.parent_id is not None: if not dim.message_belongs_to( discussion_id=self.discussion_id, message_id=self.parent_id): raise err.PatchConflict(message="provided message " "parent_id does not belong " "to this discussion") else: self.parent_id = last_message.parent_id self.update_external_references(user_id) else: last_message = None else: last_message = None return last_message
def get(self): """Get information about logged user.""" user_id = self.request.authenticated_userid.user_id user = User.get(user_id) return ReturnUser.build(user).serialize()
def collection_post(self): """Create a new user.""" settings = Settings() settings.import_data(self.request.swagger_data['user']['settings']) try: settings.validate() except Exception as exc: raise Unprocessable(detail=exc.message) param = NewUser({'name': self.request.swagger_data['user']['username'], 'password': self.request.swagger_data['user'][ 'password'], 'recovery_email': self.request.swagger_data['user'][ 'recovery_email'], 'settings': settings, }) if self.request.swagger_data['user']['contact'] is not None: param.contact = self.request.swagger_data['user']['contact'] else: c = NewContact() c.given_name = param.name c.family_name = "" # can't guess it ! email = NewEmail() email.address = param.recovery_email c.emails = [email] param.contact = c try: user = User.create(param) except Exception as exc: log.exception('Error during user creation {0}'.format(exc)) raise NotAcceptable(detail=exc.message) log.info('Created user {} with name {}'. format(user.user_id, user.name)) # default device management in_device = self.request.swagger_data['user']['device'] if in_device: try: in_device['name'] = 'default' device = Device.create_from_parameter(user, in_device, self.request.headers) log.info('Device %r created' % device.device_id) except Exception as exc: log.exception('Error during default device creation %r' % exc) else: log.warn('Missing default device parameter') user_url = self.request.route_path('User', user_id=user.user_id) self.request.response.location = user_url.encode('utf-8') # send notification to apiv2 to trigger post-registration actions config = Configuration('global').get("message_queue") try: tornado.ioloop.IOLoop.current().run_sync( lambda: notify_new_user(user, config), timeout=5) except Exception as exc: log.exception( 'Error when sending new_user notification on NATS : {0}'. format(exc)) return {'location': user_url}
def no_such_user(request): """Validator that an user does not exist.""" username = request.swagger_data['user']['username'] if not User.is_username_available(username): raise NotAcceptable(detail='User already exist')
def collection_post(self): """ Api for user authentication. Store generated tokens in a cache entry related to user_id and return a structure with this tokens for client usage. """ params = self.request.json try: user = User.authenticate(params['username'], params['password']) log.info('Authenticate user {username}'.format(username=user.name)) except Exception as exc: log.info('Authentication error for {name} : {error}'.format( name=params['username'], error=exc)) raise AuthenticationError(detail=exc.message) # Device management in_device = self.request.swagger_data['authentication']['device'] key = None if in_device: try: device = Device.get(user, in_device['device_id']) log.info("Found device %s" % device.device_id) # Found a device, check if signature public key have X and Y key = get_device_sig_key(user, device) if not key: log.error('No signature key found for device %r' % device.device_id) else: if patch_device_key(key, in_device): log.info('Patch device key OK') else: log.warn('Patch device key does not work') except NotFound: devices = Device.find(user) if devices.get('objects', []): in_device['status'] = 'unverified' else: in_device['name'] = 'default' # we must declare a new device device = Device.create_from_parameter(user, in_device, self.request.headers) log.info("Created device %s" % device.device_id) key = get_device_sig_key(user, device) if not key: log.error('No signature key found for device %r' % device.device_id) else: raise ValidationError(detail='No device informations') try: device.login(self.request.headers.get('X-Forwarded-For')) except Exception as exc: log.exception('Device login failed: {0}'.format(exc)) tokens = make_user_device_tokens(self.request, user, device, key) return { 'user_id': user.user_id, 'username': user.name, 'tokens': tokens, 'device': { 'device_id': device.device_id, 'status': device.status } }
def collection_post(self): """Create a new user.""" settings = Settings() settings.import_data(self.request.swagger_data['user']['settings']) try: settings.validate() except Exception as exc: raise Unprocessable(detail=exc.message) param = NewUser({ 'name': self.request.swagger_data['user']['username'], 'password': self.request.swagger_data['user']['password'], 'recovery_email': self.request.swagger_data['user']['recovery_email'], 'settings': settings, }) if self.request.swagger_data['user']['contact'] is not None: param.contact = self.request.swagger_data['user']['contact'] else: c = NewContact() c.given_name = param.name c.family_name = "" # can't guess it ! email = NewEmail() email.address = param.recovery_email c.emails = [email] param.contact = c try: user = User.create(param) except Exception as exc: log.exception('Error during user creation {0}'.format(exc)) raise NotAcceptable(detail=exc.message) log.info('Created user {} with name {}'.format(user.user_id, user.name)) # default device management in_device = self.request.swagger_data['user']['device'] if in_device: try: in_device['name'] = 'default' device = Device.create_from_parameter(user, in_device, self.request.headers) log.info('Device %r created' % device.device_id) except Exception as exc: log.exception('Error during default device creation %r' % exc) else: log.warn('Missing default device parameter') user_url = self.request.route_path('User', user_id=user.user_id) self.request.response.location = user_url.encode('utf-8') # send notification to apiv2 to trigger post-registration actions config = Configuration('global').get("message_queue") try: tornado.ioloop.IOLoop.current().run_sync( lambda: notify_new_user(user, config), timeout=5) except Exception as exc: log.exception( 'Error when sending new_user notification on NATS : {0}'. format(exc)) return {'location': user_url}
def collection_post(self): """ Api for user authentication. Store generated tokens in a cache entry related to user_id and return a structure with this tokens for client usage. """ params = self.request.json try: user = User.authenticate(params['username'], params['password']) log.info('Authenticate user {username}'.format(username=user.name)) except Exception as exc: log.info('Authentication error for {name} : {error}'. format(name=params['username'], error=exc)) raise AuthenticationError(detail=exc.message) # Device management in_device = self.request.swagger_data['authentication']['device'] key = None if in_device: try: device = Device.get(user, in_device['device_id']) log.info("Found device %s" % device.device_id) # Found a device, check if signature public key have X and Y key = get_device_sig_key(user, device) if not key: log.error('No signature key found for device %r' % device.device_id) else: if patch_device_key(key, in_device): log.info('Patch device key OK') else: log.warn('Patch device key does not work') except NotFound: devices = Device.find(user) if devices.get('objects', []): in_device['status'] = 'unverified' else: in_device['name'] = 'default' # we must declare a new device device = Device.create_from_parameter(user, in_device, self.request.headers) log.info("Created device %s" % device.device_id) key = get_device_sig_key(user, device) if not key: log.error('No signature key found for device %r' % device.device_id) else: raise ValidationError(detail='No device informations') try: device.login(self.request.headers.get('X-Forwarded-For')) except Exception as exc: log.exception('Device login failed: {0}'.format(exc)) access_token = create_token() refresh_token = create_token(80) # ttl = self.request.cache.client.ttl # TODO: remove this ttl to go back to cache.client ttl = 86400 expires_at = (datetime.datetime.utcnow() + datetime.timedelta(seconds=ttl)) tokens = {'access_token': access_token, 'refresh_token': refresh_token, 'expires_in': ttl, # TODO : remove this value 'shard_id': user.shard_id, 'expires_at': expires_at.isoformat()} cache_key = '{}-{}'.format(user.user_id, device.device_id) session_data = tokens.copy() if key: session_data.update({'key_id': str(key.key_id), 'x': key.x, 'y': key.y, 'curve': key.crv}) else: raise MethodFailure(detail='No public key found for device') self.request.cache.set(cache_key, session_data) self.request.cache.set(user.user_id, tokens) tokens.pop('shard_id') return {'user_id': user.user_id, 'username': user.name, 'tokens': tokens, 'device': {'device_id': device.device_id, 'status': device.status}}
def collection_post(self): """ Api for user authentication. Store generated tokens in a cache entry related to user_id and return a structure with this tokens for client usage. """ params = self.request.json try: user = User.authenticate(params['username'], params['password']) log.info('Authenticate user {username}'.format(username=user.name)) except Exception as exc: log.info('Authentication error for {name} : {error}'.format( name=params['username'], error=exc)) raise AuthenticationError(detail=exc.message) # Device management in_device = self.request.swagger_data['authentication']['device'] if in_device: try: device = Device.get(user, in_device['device_id']) except NotFound: devices = Device.find(user) if devices.get('objects', []): in_device['status'] = 'unverified' else: in_device['name'] = 'default' # we must declare a new device device = Device.create_from_parameter(user, in_device, self.request.headers) else: device = FakeDevice() access_token = create_token() refresh_token = create_token(80) # ttl = self.request.cache.client.ttl # TODO: remove this ttl to go back to cache.client ttl = 86400 expires_at = (datetime.datetime.utcnow() + datetime.timedelta(seconds=ttl)) tokens = { 'access_token': access_token, 'refresh_token': refresh_token, 'expires_in': ttl, # TODO : remove this value 'expires_at': expires_at.isoformat() } cache_key = '{}-{}'.format(user.user_id, device.device_id) self.request.cache.set(cache_key, tokens) # XXX to remove when all authenticated API will use X-Device-ID self.request.cache.set(user.user_id, tokens) return { 'user_id': user.user_id, 'username': user.name, 'tokens': tokens, 'device': { 'device_id': device.device_id, 'status': device.status } }
def user(self): from caliopen_main.user.core import User return User.get(self.user_id)
def resync_index(**kwargs): """Resync an index for an user.""" from caliopen_main.user.core import User, allocate_user_shard from caliopen_main.user.core.setups import setup_index from caliopen_main.contact.store import Contact from caliopen_main.contact.objects import Contact as ContactObject from caliopen_main.message.store import Message from caliopen_main.message.objects import Message as MessageObject if 'user_name' in kwargs and kwargs['user_name']: user = User.by_name(kwargs['user_name']) elif 'user_id' in kwargs and kwargs['user_id']: user = User.get(kwargs['user_id']) else: print('Need user_name or user_id parameter') sys.exit(1) es_url = Configuration('global').get('elasticsearch.url') es_client = Elasticsearch(es_url) if 'delete' in kwargs and kwargs['delete']: del_msg, del_con = clean_index_user(es_client, user) log.info('Delete of {0} old contacts and {1} old messages'. format(del_con, del_msg)) user_id = uuid.UUID(user.user_id) shard_id = allocate_user_shard(user_id) if user.shard_id != shard_id: log.warn('Reallocate user index shard from {} to {}'. format(user.shard_id, shard_id)) # XXX fixme. attribute should be set without using model user.model.shard_id = shard_id user.save() setup_index(user) cpt_contact = 0 contacts = Contact.filter(user_id=user.user_id) for contact in contacts: log.debug('Reindex contact %r' % contact.contact_id) obj = ContactObject(user, contact_id=contact.contact_id) obj.get_db() obj.unmarshall_db() obj.create_index() cpt_contact += 1 cpt_message = 0 messages = Message.filter(user_id=user.user_id).timeout(None). \ allow_filtering() for message in messages: log.debug('Reindex message %r' % message.message_id) obj = MessageObject(user, message_id=message.message_id) obj.get_db() obj.unmarshall_db() obj.create_index() cpt_message += 1 log.info('Sync of {0} contacts, {1} messages'. format(cpt_contact, cpt_message)) log.info('Create index alias %r' % user.user_id) try: es_client.indices.put_alias(index=shard_id, name=user.user_id) except Exception as exc: log.exception('Error during alias creation : {}'.format(exc)) raise exc
def collection_post(self): """ Api for user authentication. Store generated tokens in a cache entry related to user_id and return a structure with this tokens for client usage. """ params = self.request.json try: user = User.authenticate(params['username'], params['password']) log.info('Authenticate user {username}'.format(username=user.name)) except Exception as exc: log.info('Authentication error for {name} : {error}'.format( name=params['username'], error=exc)) raise AuthenticationError(detail=exc.message) # Device management in_device = self.request.swagger_data['authentication']['device'] key = None if in_device: try: device = Device.get(user, in_device['device_id']) # Found a device, check if signature public key have X and Y key = get_device_sig_key(user, device) if not key: log.error('No signature key found for device %r' % device.device_id) else: if patch_device_key(key, in_device): log.info('Patch device key OK') else: log.warn('Patch device key does not work') except NotFound: devices = Device.find(user) if devices.get('objects', []): in_device['status'] = 'unverified' else: in_device['name'] = 'default' # we must declare a new device device = Device.create_from_parameter(user, in_device, self.request.headers) key = get_device_sig_key(user, device) if not key: log.error('No signature key found for device %r' % device.device_id) else: raise ValidationError(detail='No device informations') access_token = create_token() refresh_token = create_token(80) # ttl = self.request.cache.client.ttl # TODO: remove this ttl to go back to cache.client ttl = 86400 expires_at = (datetime.datetime.utcnow() + datetime.timedelta(seconds=ttl)) tokens = { 'access_token': access_token, 'refresh_token': refresh_token, 'expires_in': ttl, # TODO : remove this value 'shard_id': user.shard_id, 'expires_at': expires_at.isoformat() } cache_key = '{}-{}'.format(user.user_id, device.device_id) session_data = tokens.copy() if key: session_data.update({ 'key_id': str(key.key_id), 'x': key.x, 'y': key.y, 'curve': key.crv }) else: raise MethodFailure(detail='No public key found for device') self.request.cache.set(cache_key, session_data) self.request.cache.set(user.user_id, tokens) tokens.pop('shard_id') return { 'user_id': user.user_id, 'username': user.name, 'tokens': tokens, 'device': { 'device_id': device.device_id, 'status': device.status } }
def import_email(email, import_path, format, contact_probability, **kwargs): """Import emails for an user.""" from caliopen_main.user.core import User from caliopen_main.contact.core import Contact, ContactLookup from caliopen_main.message.parsers.mail import MailMessage from caliopen_main.contact.parameters import NewContact, NewEmail from caliopen_nats.delivery import UserMailDelivery from caliopen_main.message.core import RawMessage from caliopen_storage.config import Configuration max_size = int(Configuration("global").get("object_store.db_size_limit")) if 'to' in kwargs and kwargs['to']: dest_email = kwargs['to'] else: dest_email = email if format == 'maildir': if dest_email != email: raise Exception('Cannot change To email using maildir format') emails = Maildir(import_path, factory=message_from_file) mode = 'maildir' else: if os.path.isdir(import_path): mode = 'mbox_directory' emails = {} files = [ f for f in os.listdir(import_path) if os.path.isfile(os.path.join(import_path, f)) ] for f in files: try: log.debug('Importing mail from file {}'.format(f)) with open('%s/%s' % (import_path, f)) as fh: data = fh.read() data = re.sub('^To: (.*)', 'To: %s' % dest_email, data, flags=re.MULTILINE) emails[f] = message_from_string(data) except Exception as exc: log.error('Error importing email {}'.format(exc)) else: mode = 'mbox' emails = mbox(import_path) user = User.by_local_identifier(dest_email, 'email') log.info("Processing mode %s" % mode) for key, data in emails.iteritems(): # Prevent creating message too large to fit in db. # (should use inject cmd for large messages) size = len(data.as_string()) if size > max_size: log.warn("Message too large to fit into db. \ Please, use 'inject' cmd for importing large emails.") continue raw = RawMessage.create(data.as_string()) log.debug('Created raw message {}'.format(raw.raw_msg_id)) message = MailMessage(data.as_string()) dice = random() if dice <= contact_probability: for participant in message.participants: try: ContactLookup.get(user, participant.address) except NotFound: log.info('Creating contact %s' % participant.address) name, domain = participant.address.split('@') contact_param = NewContact() contact_param.family_name = name if participant.address: e_mail = NewEmail() e_mail.address = participant.address contact_param.emails = [e_mail] Contact.create(user, contact_param) else: log.info('No contact associated to raw {} '.format(raw.raw_msg_id)) processor = UserMailDelivery( user, user.local_identities[0]) # assume one local identity try: obj_message = processor.process_raw(raw.raw_msg_id) except Exception as exc: if isinstance(exc, DuplicateMessage): log.info('duplicate message {}, not imported'.format( raw.raw_msg_id)) else: log.exception(exc) else: log.info('Created message {}'.format(obj_message.message_id))
def import_email(email, import_path, format, contact_probability, **kwargs): """Import emails for an user.""" from caliopen_main.user.core import User, UserIdentity from caliopen_main.contact.core import Contact, ContactLookup from caliopen_main.message.parsers.mail import MailMessage from caliopen_main.contact.parameters import NewContact, NewEmail from caliopen_nats.delivery import UserMailDelivery from caliopen_main.message.core import RawMessage from caliopen_storage.config import Configuration max_size = int(Configuration("global").get("object_store.db_size_limit")) if 'to' in kwargs and kwargs['to']: dest_email = kwargs['to'] else: dest_email = email if format == 'maildir': if dest_email != email: raise Exception('Cannot change To email using maildir format') emails = Maildir(import_path, factory=message_from_file) mode = 'maildir' else: if os.path.isdir(import_path): mode = 'mbox_directory' emails = {} files = [f for f in os.listdir(import_path) if os.path.isfile(os.path.join(import_path, f))] for f in files: try: log.debug('Importing mail from file {}'.format(f)) with open('%s/%s' % (import_path, f)) as fh: data = fh.read() data = re.sub('^To: (.*)', 'To: %s' % dest_email, data, flags=re.MULTILINE) emails[f] = message_from_string(data) except Exception as exc: log.error('Error importing email {}'.format(exc)) else: mode = 'mbox' emails = mbox(import_path) user = User.by_local_identifier(dest_email, 'email') log.info("Processing mode %s" % mode) for key, data in emails.iteritems(): # Prevent creating message too large to fit in db. # (should use inject cmd for large messages) size = len(data.as_string()) if size > max_size: log.warn("Message too large to fit into db. \ Please, use 'inject' cmd for importing large emails.") continue raw = RawMessage.create(data.as_string()) log.debug('Created raw message {}'.format(raw.raw_msg_id)) message = MailMessage(data.as_string()) dice = random() if dice <= contact_probability: for participant in message.participants: try: ContactLookup.get(user, participant.address) except NotFound: log.info('Creating contact %s' % participant.address) name, domain = participant.address.split('@') contact_param = NewContact() contact_param.family_name = name if participant.address: e_mail = NewEmail() e_mail.address = participant.address contact_param.emails = [e_mail] Contact.create(user, contact_param) log.info('No contact associated to raw {} '.format(raw.raw_msg_id)) processor = UserMailDelivery(user, user.local_identities[0]) # assume one local identity try: obj_message = processor.process_raw(raw.raw_msg_id) except Exception as exc: log.exception(exc) else: log.info('Created message {}'.format(obj_message.message_id))