def process_key_discovery(self, msg, payload):
"""Discover public keys related to a new contact identifier."""
if 'user_id' not in payload or 'contact_id' not in payload:
raise Exception('Invalid contact_update structure')
user = User.get(payload['user_id'])
contact = Contact(user.user_id, contact_id=payload['contact_id'])
contact.get_db()
contact.unmarshall_db()
manager = ContactPublicKeyManager()
founds = []
for ident in payload.get('emails', []):
log.info('Process email identity {0}'.format(ident['address']))
discovery = manager.process_identity(user, contact,
ident['address'], 'email')
if discovery:
founds.append(discovery)
for ident in payload.get('identities', []):
log.info('Process identity {0}:{1}'.format(ident['type'],
ident['name']))
discovery = manager.process_identity(user, contact, ident['name'],
ident['type'])
if discovery:
founds.append(discovery)
if founds:
log.info('Found %d results' % len(founds))
self._process_results(user, contact, founds)
def process_raw(self, msg, payload):
"""Process an inbound raw message."""
nats_error = {
'error': '',
'message': 'inbound email message process failed'
}
nats_success = {
'message': 'OK : inbound email message proceeded'
}
try:
user = User.get(payload['user_id'])
identity = UserIdentity.get(user, payload['identity_id'])
deliver = UserMailDelivery(user, identity)
new_message = deliver.process_raw(payload['message_id'])
nats_success['message_id'] = str(new_message.message_id)
self.natsConn.publish(msg.reply, json.dumps(nats_success))
except DuplicateObject:
log.info("Message already imported : {}".format(payload))
nats_success['message_id'] = str(payload['message_id'])
nats_success['message'] = 'raw message already imported'
self.natsConn.publish(msg.reply, json.dumps(nats_success))
except Exception as exc:
# TODO: handle abort exception and report it as special case
log.error("deliver process failed for raw {}: {}".
format(payload, exc))
nats_error['error'] = str(exc.message)
self.natsConn.publish(msg.reply, json.dumps(nats_error))
return exc
def process_raw(self, msg, payload):
"""Process an inbound raw message."""
nats_error = {
'error': '',
'message': 'inbound twitter message process failed'
}
nats_success = {'message': 'OK : inbound twitter message proceeded'}
try:
user = User.get(payload['user_id'])
identity = UserIdentity.get(user, payload['identity_id'])
deliver = UserTwitterDelivery(user, identity)
new_message = deliver.process_raw(payload['message_id'])
nats_success['message_id'] = str(new_message.message_id)
nats_success['discussion_id'] = str(new_message.discussion_id)
self.natsConn.publish(msg.reply, json.dumps(nats_success))
except DuplicateObject:
log.info("Message already imported : {}".format(payload))
nats_success['message_id'] = str(payload['message_id'])
nats_success['discussion_id'] = "" # message has not been parsed
nats_success['message'] = 'raw message already imported'
self.natsConn.publish(msg.reply, json.dumps(nats_success))
except Exception as exc:
# TODO: handle abort exception and report it as special case
exc_info = sys.exc_info()
log.error("deliver process failed for raw {}: {}".format(
payload, traceback.print_exception(*exc_info)))
nats_error['error'] = str(exc.message)
self.natsConn.publish(msg.reply, json.dumps(nats_error))
return exc
def process_key_discovery(self, msg, payload):
"""Discover public keys related to a new contact identifier."""
if 'user_id' not in payload or 'contact_id' not in payload:
raise Exception('Invalid contact_update structure')
user = User.get(payload['user_id'])
contact = Contact(user.user_id, contact_id=payload['contact_id'])
contact.get_db()
contact.unmarshall_db()
manager = ContactPublicKeyManager()
founds = []
for ident in payload.get('emails', []):
log.info('Process email identity {0}'.format(ident['address']))
discovery = manager.process_identity(user, contact,
ident['address'], 'email')
if discovery:
founds.append(discovery)
for ident in payload.get('identities', []):
log.info('Process identity {0}:{1}'.
format(ident['type'], ident['name']))
discovery = manager.process_identity(user, contact,
ident['name'], ident['type'])
if discovery:
founds.append(discovery)
if founds:
log.info('Found %d results' % len(founds))
self._process_results(user, contact, founds)
def get(self):
"""Get information about logged user."""
user_id = self.request.authenticated_userid.user_id
user = User.get(user_id)
identifier = self.request.swagger_data['identifier']
identity = user.get_remote_identity(identifier)
return ReturnRemoteIdentity.build(identity).serialize()
def user(self):
"""Return user related to this object."""
from caliopen_main.user.core import User
if not self._user:
self._user = User.get(self.user_id)
return self._user
def resync_index(**kwargs):
"""Resync an index for an user."""
from caliopen_main.user.core import User
from caliopen_main.user.core.setups import setup_index
from caliopen_main.contact.store import Contact
from caliopen_main.contact.objects import Contact as ContactObject
from caliopen_main.message.store import Message
from caliopen_main.message.objects import Message as MessageObject
if 'user_name' in kwargs and kwargs['user_name']:
user = User.by_name(kwargs['user_name'])
elif 'user_id' in kwargs and kwargs['user_id']:
user = User.get(kwargs['user_id'])
else:
print('Need user_name or user_id parameter')
sys.exit(1)
mapping_key = 'elasticsearch.mappings_version'
current_version = Configuration('global').get(mapping_key)
new_index = '{}_{}'.format(user.user_id, current_version)
if 'version' in kwargs and kwargs['version']:
old_version = kwargs['version']
old_index = '{}_{}'.format(user.user_id, old_version)
else:
old_index = new_index
es_url = Configuration('global').get('elasticsearch.url')
es_client = Elasticsearch(es_url)
# Delete current index
if not es_client.indices.exists([old_index]):
log.warn('Index %r not found for user %s' % (old_index, user.name))
sys.exit(1)
es_client.indices.delete([old_index])
log.info('Index %r deleted for user %s' % (old_index, user.name))
# Recreate index and mappings
setup_index(user)
contacts = Contact.filter(user_id=user.user_id)
for contact in contacts:
log.debug('Reindex contact %r' % contact.contact_id)
obj = ContactObject(user.user_id, contact_id=contact.contact_id)
obj.create_index()
messages = Message.filter(user_id=user.user_id).allow_filtering()
for message in messages:
log.debug('Reindex message %r' % message.message_id)
obj = MessageObject(user.user_id, message_id=message.message_id)
obj.create_index()
log.info('Create index alias %r' % user.user_id)
es_client.indices.put_alias(index=new_index, name=user.user_id)
def process_update(self, msg, payload):
"""Process a contact update message."""
# XXX validate payload structure
if 'user_id' not in payload or 'contact_id' not in payload:
raise Exception('Invalid contact_update structure')
user = User.get(payload['user_id'])
contact = Contact(user.user_id, contact_id=payload['contact_id'])
contact.get_db()
contact.unmarshall_db()
qualifier = ContactMessageQualifier(user)
log.info('Will process update for contact {0} of user {1}'.format(
contact.contact_id, user.user_id))
qualifier.process(contact)
def process_update(self, msg, payload):
"""Process a contact update message."""
# XXX validate payload structure
if 'user_id' not in payload or 'contact_id' not in payload:
raise Exception('Invalid contact_update structure')
user = User.get(payload['user_id'])
contact = Contact(user, contact_id=payload['contact_id'])
contact.get_db()
contact.unmarshall_db()
qualifier = ContactMessageQualifier(user)
log.info('Will process update for contact {0} of user {1}'.
format(contact.contact_id, user.user_id))
# TODO: (re)discover GPG keys
qualifier.process(contact)
def collection_post(self):
"""Get information about logged user."""
user_id = self.request.authenticated_userid.user_id
user = User.get(user_id)
data = self.request.swagger_data['identity']
param = NewRemoteIdentity({'display_name': data['display_name'],
'identifier': data['identifier'],
'type': data['type'],
'status': data.get('status', 'active'),
'infos': data['infos']
})
param.validate()
identity = user.add_remote_identity(param)
identity_url = self.request.route_path('RemoteIdentities',
identifier=identity.identity_id)
self.request.response.location = identity_url.encode('utf-8')
return {'location': identity_url}
def process_raw(self, msg, payload):
"""Process an inbound raw message."""
nats_error = {
'error': '',
'message': 'inbound email message process failed'
}
nats_success = {'message': 'OK : inbound email message proceeded'}
user = User.get(payload['user_id'])
deliver = UserMessageDelivery(user)
try:
deliver.process_raw(payload['message_id'])
self.natsConn.publish(msg.reply, json.dumps(nats_success))
except Exception as exc:
log.error("deliver process failed : {}".format(exc))
nats_error['error'] = str(exc.message)
self.natsConn.publish(msg.reply, json.dumps(nats_error))
return exc
def process_raw(self, msg, payload):
"""Process an inbound raw message."""
nats_error = {
'error': '',
'message': 'inbound twitter message process failed'
}
nats_success = {
'message': 'OK : inbound twitter message proceeded'
}
try:
user = User.get(payload['user_id'])
identity = UserIdentity.get(user, payload['identity_id'])
deliver = UserTwitterDMDelivery(user, identity)
new_message = deliver.process_raw(payload['message_id'])
nats_success['message_id'] = str(new_message.message_id)
self.natsConn.publish(msg.reply, json.dumps(nats_success))
except Exception as exc:
# TODO: handle abort exception and report it as special case
log.error("deliver process failed : {}".format(exc))
nats_error['error'] = str(exc.message)
self.natsConn.publish(msg.reply, json.dumps(nats_error))
return exc
def _add_from_participant(self, user_id):
if 'identities' not in self:
raise err.PatchUnprocessable
if len(self['identities']) != 1:
raise err.PatchUnprocessable
provided_identity = self['identities'][0]
local_identity = LocalIdentity(
identifier=provided_identity['identifier'])
try:
local_identity.get_db()
local_identity.unmarshall_db()
except NotFound:
raise NotFound
if str(local_identity.user_id) != user_id:
raise err.ForbiddenAction(message="Action forbidden for this user")
# add 'from' participant with local identity's identifier
user = User.get(user_id)
if not hasattr(self, 'participants'):
self.participants = []
else:
if len(self.participants) > 0:
for i, participant in enumerate(self.participants):
if re.match("from", participant['type'], re.IGNORECASE):
self.participants.pop(i)
from_participant = Participant()
from_participant.address = local_identity.identifier
from_participant.label = local_identity.display_name
from_participant.protocol = "email"
from_participant.type = "From"
from_participant.contact_ids = [user.contact.contact_id]
self.participants.append(from_participant)
return from_participant
def user(self):
from caliopen_main.user.core import User
return User.get(self.user_id)
def get(self):
"""Get information about logged user."""
user_id = self.request.authenticated_userid.user_id
user = User.get(user_id)
return ReturnUser.build(user).serialize()
def _load_user(self):
if self._user:
return
self._user = User.get(self.user_id)
def resync_index(**kwargs):
"""Resync an index for an user."""
from caliopen_main.user.core import User, allocate_user_shard
from caliopen_main.user.core.setups import setup_index
from caliopen_main.contact.store import Contact
from caliopen_main.contact.objects import Contact as ContactObject
from caliopen_main.message.store import Message
from caliopen_main.message.objects import Message as MessageObject
if 'user_name' in kwargs and kwargs['user_name']:
user = User.by_name(kwargs['user_name'])
elif 'user_id' in kwargs and kwargs['user_id']:
user = User.get(kwargs['user_id'])
else:
print('Need user_name or user_id parameter')
sys.exit(1)
es_url = Configuration('global').get('elasticsearch.url')
es_client = Elasticsearch(es_url)
if 'delete' in kwargs and kwargs['delete']:
del_msg, del_con = clean_index_user(es_client, user)
log.info('Delete of {0} old contacts and {1} old messages'.format(
del_con, del_msg))
user_id = uuid.UUID(user.user_id)
shard_id = allocate_user_shard(user_id)
if user.shard_id != shard_id:
log.warn('Reallocate user index shard from {} to {}'.format(
user.shard_id, shard_id))
# XXX fixme. attribute should be set without using model
user.model.shard_id = shard_id
user.save()
setup_index(user)
cpt_contact = 0
contacts = Contact.filter(user_id=user.user_id)
for contact in contacts:
log.debug('Reindex contact %r' % contact.contact_id)
obj = ContactObject(user, contact_id=contact.contact_id)
obj.get_db()
obj.unmarshall_db()
obj.create_index()
cpt_contact += 1
cpt_message = 0
messages = Message.filter(user_id=user.user_id).timeout(None). \
allow_filtering()
for message in messages:
log.debug('Reindex message %r' % message.message_id)
obj = MessageObject(user, message_id=message.message_id)
obj.get_db()
obj.unmarshall_db()
obj.create_index()
cpt_message += 1
log.info('Sync of {0} contacts, {1} messages'.format(
cpt_contact, cpt_message))
log.info('Create index alias %r' % user.user_id)
try:
es_client.indices.put_alias(index=shard_id, name=user.user_id)
except Exception as exc:
log.exception('Error during alias creation : {}'.format(exc))
raise exc
def get(self):
"""Get information about logged user."""
user_id = self.request.authenticated_userid.user_id
user = User.get(user_id)
return ReturnUser.build(user).serialize()
def _check_discussion_consistency(self, user_id):
from caliopen_main.message.objects.message import Message
new_discussion = False
if not hasattr(self, 'discussion_id') or self.discussion_id == "" \
or self.discussion_id is None:
# no discussion_id provided. Try to find one with draft's parent_id
# or create new discussion
if hasattr(self, 'parent_id') \
and self.parent_id is not None \
and self.parent_id != "":
parent_msg = Message(user_id, message_id=self.parent_id)
try:
parent_msg.get_db()
parent_msg.unmarshall_db()
except NotFound:
raise err.PatchError(message="parent message not found")
self.discussion_id = parent_msg.discussion_id
else:
user = User.get(user_id)
discussion = Discussion.create_from_message(user, self)
self.discussion_id = discussion.discussion_id
new_discussion = True
if not new_discussion:
dim = DIM(user_id)
d_id = self.discussion_id
last_message = dim.get_last_message(d_id, -10, 10, True)
if last_message == {}:
raise err.PatchError(
message='No such discussion {}'.format(d_id))
is_a_reply = (str(last_message.message_id) != str(self.message_id))
if is_a_reply:
# check participants consistency
if hasattr(self,
"participants") and len(self.participants) > 0:
participants = [p['address'] for p in self.participants]
last_msg_participants = [
p['address'] for p in last_message.participants
]
if len(participants) != len(last_msg_participants):
raise err.PatchError(
message="list of participants "
"is not consistent for this discussion")
participants.sort()
last_msg_participants.sort()
for i, participant in enumerate(participants):
if participant != last_msg_participants[i]:
raise err.PatchConflict(
message="list of participants "
"is not consistent for this discussion")
else:
self.build_participants_for_reply(user_id)
# check parent_id consistency
if 'parent_id' in self and self.parent_id != "" \
and self.parent_id is not None:
if not dim.message_belongs_to(
discussion_id=self.discussion_id,
message_id=self.parent_id):
raise err.PatchConflict(message="provided message "
"parent_id does not belong "
"to this discussion")
else:
self.parent_id = last_message.parent_id
self.update_external_references(user_id)
else:
last_message = None
else:
last_message = None
return last_message
def resync_index(**kwargs):
"""Resync an index for an user."""
from caliopen_main.user.core import User, allocate_user_shard
from caliopen_main.user.core.setups import setup_index
from caliopen_main.contact.store import Contact
from caliopen_main.contact.objects import Contact as ContactObject
from caliopen_main.message.store import Message
from caliopen_main.message.objects import Message as MessageObject
if 'user_name' in kwargs and kwargs['user_name']:
user = User.by_name(kwargs['user_name'])
elif 'user_id' in kwargs and kwargs['user_id']:
user = User.get(kwargs['user_id'])
else:
print('Need user_name or user_id parameter')
sys.exit(1)
es_url = Configuration('global').get('elasticsearch.url')
es_client = Elasticsearch(es_url)
if 'delete' in kwargs and kwargs['delete']:
del_msg, del_con = clean_index_user(es_client, user)
log.info('Delete of {0} old contacts and {1} old messages'.
format(del_con, del_msg))
user_id = uuid.UUID(user.user_id)
shard_id = allocate_user_shard(user_id)
if user.shard_id != shard_id:
log.warn('Reallocate user index shard from {} to {}'.
format(user.shard_id, shard_id))
# XXX fixme. attribute should be set without using model
user.model.shard_id = shard_id
user.save()
setup_index(user)
cpt_contact = 0
contacts = Contact.filter(user_id=user.user_id)
for contact in contacts:
log.debug('Reindex contact %r' % contact.contact_id)
obj = ContactObject(user, contact_id=contact.contact_id)
obj.get_db()
obj.unmarshall_db()
obj.create_index()
cpt_contact += 1
cpt_message = 0
messages = Message.filter(user_id=user.user_id).timeout(None). \
allow_filtering()
for message in messages:
log.debug('Reindex message %r' % message.message_id)
obj = MessageObject(user, message_id=message.message_id)
obj.get_db()
obj.unmarshall_db()
obj.create_index()
cpt_message += 1
log.info('Sync of {0} contacts, {1} messages'.
format(cpt_contact, cpt_message))
log.info('Create index alias %r' % user.user_id)
try:
es_client.indices.put_alias(index=shard_id, name=user.user_id)
except Exception as exc:
log.exception('Error during alias creation : {}'.format(exc))
raise exc