def process_key_discovery(self, msg, payload):
"""Discover public keys related to a new contact identifier."""
if 'user_id' not in payload or 'contact_id' not in payload:
raise Exception('Invalid contact_update structure')
user = User.get(payload['user_id'])
contact = Contact(user.user_id, contact_id=payload['contact_id'])
contact.get_db()
contact.unmarshall_db()
manager = ContactPublicKeyManager()
founds = []
for ident in payload.get('emails', []):
log.info('Process email identity {0}'.format(ident['address']))
discovery = manager.process_identity(user, contact,
ident['address'], 'email')
if discovery:
founds.append(discovery)
for ident in payload.get('identities', []):
log.info('Process identity {0}:{1}'.
format(ident['type'], ident['name']))
discovery = manager.process_identity(user, contact,
ident['name'], ident['type'])
if discovery:
founds.append(discovery)
if founds:
log.info('Found %d results' % len(founds))
self._process_results(user, contact, founds)
def process_update(self, msg, payload):
"""Process a contact update message."""
# XXX validate payload structure
if 'user_id' not in payload or 'contact_id' not in payload:
raise Exception('Invalid contact_update structure')
user = User.get(payload['user_id'])
contact = Contact(user, contact_id=payload['contact_id'])
contact.get_db()
contact.unmarshall_db()
qualifier = ContactMessageQualifier(user)
log.info('Will process update for contact {0} of user {1}'.
format(contact.contact_id, user.user_id))
# TODO: (re)discover GPG keys
qualifier.process(contact)
def process_key_discovery(self, msg, payload):
"""Discover public keys related to a new contact identifier."""
if 'user_id' not in payload or 'contact_id' not in payload:
raise Exception('Invalid contact_update structure')
user = User.get(payload['user_id'])
contact = Contact(user.user_id, contact_id=payload['contact_id'])
contact.get_db()
contact.unmarshall_db()
manager = ContactPublicKeyManager()
founds = []
for ident in payload.get('emails', []):
log.info('Process email identity {0}'.format(ident['address']))
discovery = manager.process_identity(user, contact,
ident['address'], 'email')
if discovery:
founds.append(discovery)
for ident in payload.get('identities', []):
log.info('Process identity {0}:{1}'.format(ident['type'],
ident['name']))
discovery = manager.process_identity(user, contact, ident['name'],
ident['type'])
if discovery:
founds.append(discovery)
if founds:
log.info('Found %d results' % len(founds))
self._process_results(user, contact, founds)
def basic_compute(username, job, **kwargs):
"""Import emails for an user."""
from caliopen_main.user.core import User
from caliopen_main.contact.objects import Contact
from caliopen_pi.qualifiers import ContactMessageQualifier
user = User.by_name(username)
qualifier = ContactMessageQualifier(user)
contacts = Contact.list_db(user.user_id)
if job == 'contact_privacy':
for contact in contacts:
log.info('Processing contact {0}'.format(contact.contact_id))
qualifier.process(contact)
def basic_compute(username, job, ** kwargs):
"""Import emails for an user."""
from caliopen_main.user.core import User
from caliopen_main.contact.objects import Contact
from caliopen_pi.qualifiers import ContactMessageQualifier
user = User.by_name(username)
qualifier = ContactMessageQualifier(user)
contacts = Contact.list_db(user.user_id)
if job == 'contact_privacy':
for contact in contacts:
log.info('Processing contact {0}'.format(contact.contact_id))
qualifier.process(contact)
def resync_index(**kwargs):
"""Resync an index for an user."""
from caliopen_main.user.core import User
from caliopen_main.user.core.setups import setup_index
from caliopen_main.contact.store import Contact
from caliopen_main.contact.objects import Contact as ContactObject
from caliopen_main.message.store import Message
from caliopen_main.message.objects import Message as MessageObject
if 'user_name' in kwargs and kwargs['user_name']:
user = User.by_name(kwargs['user_name'])
elif 'user_id' in kwargs and kwargs['user_id']:
user = User.get(kwargs['user_id'])
else:
print('Need user_name or user_id parameter')
sys.exit(1)
mapping_key = 'elasticsearch.mappings_version'
current_version = Configuration('global').get(mapping_key)
new_index = '{}_{}'.format(user.user_id, current_version)
if 'version' in kwargs and kwargs['version']:
old_version = kwargs['version']
old_index = '{}_{}'.format(user.user_id, old_version)
else:
old_index = new_index
es_url = Configuration('global').get('elasticsearch.url')
es_client = Elasticsearch(es_url)
# Delete current index
if not es_client.indices.exists([old_index]):
log.warn('Index %r not found for user %s' % (old_index, user.name))
sys.exit(1)
es_client.indices.delete([old_index])
log.info('Index %r deleted for user %s' % (old_index, user.name))
# Recreate index and mappings
setup_index(user)
contacts = Contact.filter(user_id=user.user_id)
for contact in contacts:
log.debug('Reindex contact %r' % contact.contact_id)
obj = ContactObject(user.user_id, contact_id=contact.contact_id)
obj.create_index()
messages = Message.filter(user_id=user.user_id).allow_filtering()
for message in messages:
log.debug('Reindex message %r' % message.message_id)
obj = MessageObject(user.user_id, message_id=message.message_id)
obj.create_index()
log.info('Create index alias %r' % user.user_id)
es_client.indices.put_alias(index=new_index, name=user.user_id)
def process_update(self, msg, payload):
"""Process a contact update message."""
# XXX validate payload structure
if 'user_id' not in payload or 'contact_id' not in payload:
raise Exception('Invalid contact_update structure')
user = User.get(payload['user_id'])
contact = Contact(user.user_id, contact_id=payload['contact_id'])
contact.get_db()
contact.unmarshall_db()
qualifier = ContactMessageQualifier(user)
log.info('Will process update for contact {0} of user {1}'.format(
contact.contact_id, user.user_id))
qualifier.process(contact)
def resync_index(**kwargs):
"""Resync an index for an user."""
from caliopen_main.user.core import User, allocate_user_shard
from caliopen_main.user.core.setups import setup_index
from caliopen_main.contact.store import Contact
from caliopen_main.contact.objects import Contact as ContactObject
from caliopen_main.message.store import Message
from caliopen_main.message.objects import Message as MessageObject
if 'user_name' in kwargs and kwargs['user_name']:
user = User.by_name(kwargs['user_name'])
elif 'user_id' in kwargs and kwargs['user_id']:
user = User.get(kwargs['user_id'])
else:
print('Need user_name or user_id parameter')
sys.exit(1)
es_url = Configuration('global').get('elasticsearch.url')
es_client = Elasticsearch(es_url)
if 'delete' in kwargs and kwargs['delete']:
del_msg, del_con = clean_index_user(es_client, user)
log.info('Delete of {0} old contacts and {1} old messages'.format(
del_con, del_msg))
user_id = uuid.UUID(user.user_id)
shard_id = allocate_user_shard(user_id)
if user.shard_id != shard_id:
log.warn('Reallocate user index shard from {} to {}'.format(
user.shard_id, shard_id))
# XXX fixme. attribute should be set without using model
user.model.shard_id = shard_id
user.save()
setup_index(user)
cpt_contact = 0
contacts = Contact.filter(user_id=user.user_id)
for contact in contacts:
log.debug('Reindex contact %r' % contact.contact_id)
obj = ContactObject(user, contact_id=contact.contact_id)
obj.get_db()
obj.unmarshall_db()
obj.create_index()
cpt_contact += 1
cpt_message = 0
messages = Message.filter(user_id=user.user_id).timeout(None). \
allow_filtering()
for message in messages:
log.debug('Reindex message %r' % message.message_id)
obj = MessageObject(user, message_id=message.message_id)
obj.get_db()
obj.unmarshall_db()
obj.create_index()
cpt_message += 1
log.info('Sync of {0} contacts, {1} messages'.format(
cpt_contact, cpt_message))
log.info('Create index alias %r' % user.user_id)
try:
es_client.indices.put_alias(index=shard_id, name=user.user_id)
except Exception as exc:
log.exception('Error during alias creation : {}'.format(exc))
raise exc
def resync_user(user):
"""Resync data for an user into its index shard."""
from caliopen_main.contact.store import Contact
from caliopen_main.contact.objects import Contact as ContactObject
from caliopen_main.message.store import Message
from caliopen_main.message.objects import Message as MessageObject
log.info('Sync user {0} into shard {1}'.format(user.user_id,
user.shard_id))
client = Elasticsearch(Configuration('global').get('elasticsearch.url'))
body = {'filter': {'term': {'user_id': user.user_id}}}
# if not client.indices.exists_alias(user.user_id):
# log.info('Creating alias {} for index {}'.format(user.user_id, user.shard_id))
client.indices.put_alias(user.shard_id, user.user_id, body=body)
contacts = Contact.filter(user_id=user.user_id).timeout(None)
for contact in contacts:
log.debug('Reindex contact %r' % contact.contact_id)
obj = ContactObject(user, contact_id=contact.contact_id)
obj.get_db()
obj.unmarshall_db()
obj.create_index()
messages = Message.filter(user_id=user.user_id). \
allow_filtering().timeout(None)
for message in messages:
log.debug('Reindex message %r' % message.message_id)
obj = MessageObject(user, message_id=message.message_id)
obj.get_db()
obj.unmarshall_db()
obj.create_index()
def resync_index(**kwargs):
"""Resync an index for an user."""
from caliopen_main.user.core import User, allocate_user_shard
from caliopen_main.user.core.setups import setup_index
from caliopen_main.contact.store import Contact
from caliopen_main.contact.objects import Contact as ContactObject
from caliopen_main.message.store import Message
from caliopen_main.message.objects import Message as MessageObject
if 'user_name' in kwargs and kwargs['user_name']:
user = User.by_name(kwargs['user_name'])
elif 'user_id' in kwargs and kwargs['user_id']:
user = User.get(kwargs['user_id'])
else:
print('Need user_name or user_id parameter')
sys.exit(1)
es_url = Configuration('global').get('elasticsearch.url')
es_client = Elasticsearch(es_url)
if 'delete' in kwargs and kwargs['delete']:
del_msg, del_con = clean_index_user(es_client, user)
log.info('Delete of {0} old contacts and {1} old messages'.
format(del_con, del_msg))
user_id = uuid.UUID(user.user_id)
shard_id = allocate_user_shard(user_id)
if user.shard_id != shard_id:
log.warn('Reallocate user index shard from {} to {}'.
format(user.shard_id, shard_id))
# XXX fixme. attribute should be set without using model
user.model.shard_id = shard_id
user.save()
setup_index(user)
cpt_contact = 0
contacts = Contact.filter(user_id=user.user_id)
for contact in contacts:
log.debug('Reindex contact %r' % contact.contact_id)
obj = ContactObject(user, contact_id=contact.contact_id)
obj.get_db()
obj.unmarshall_db()
obj.create_index()
cpt_contact += 1
cpt_message = 0
messages = Message.filter(user_id=user.user_id).timeout(None). \
allow_filtering()
for message in messages:
log.debug('Reindex message %r' % message.message_id)
obj = MessageObject(user, message_id=message.message_id)
obj.get_db()
obj.unmarshall_db()
obj.create_index()
cpt_message += 1
log.info('Sync of {0} contacts, {1} messages'.
format(cpt_contact, cpt_message))
log.info('Create index alias %r' % user.user_id)
try:
es_client.indices.put_alias(index=shard_id, name=user.user_id)
except Exception as exc:
log.exception('Error during alias creation : {}'.format(exc))
raise exc
def resync_user(user):
"""Resync data for an user into its index shard."""
from caliopen_main.contact.store import Contact
from caliopen_main.contact.objects import Contact as ContactObject
from caliopen_main.message.store import Message
from caliopen_main.message.objects import Message as MessageObject
log.info('Sync user {0} into shard {1}'.format(user.user_id,
user.shard_id))
client = Elasticsearch(Configuration('global').get('elasticsearch.url'))
body = {'filter': {'term': {'user_id': user.user_id}}}
# if not client.indices.exists_alias(user.user_id):
# log.info('Creating alias {} for index {}'.format(user.user_id, user.shard_id))
client.indices.put_alias(user.shard_id, user.user_id, body=body)
contacts = Contact.filter(user_id=user.user_id).timeout(None)
for contact in contacts:
log.debug('Reindex contact %r' % contact.contact_id)
obj = ContactObject(user, contact_id=contact.contact_id)
obj.get_db()
obj.unmarshall_db()
obj.create_index()
messages = Message.filter(user_id=user.user_id). \
allow_filtering().timeout(None)
for message in messages:
log.debug('Reindex message %r' % message.message_id)
obj = MessageObject(user, message_id=message.message_id)
obj.get_db()
obj.unmarshall_db()
obj.create_index()