def setup_index(user):
"""Creates user index and setups mappings."""
url = Configuration('global').get('elasticsearch.url')
client = Elasticsearch(url)
shard_id = user.shard_id
# does shard exist ?
if not client.indices.exists(shard_id):
setup_shard_index(shard_id)
# Creates a versioned index with our custom analyzers, tokenizers, etc.
m_version = Configuration('global').get('elasticsearch.mappings_version')
if m_version == "":
raise Exception(
'Invalid mapping version for shard {0}'.format(shard_id))
index_name = shard_id
alias_name = user.user_id
# Points an alias to the underlying user's index
try:
client.indices.put_alias(index=index_name, name=alias_name)
except Exception as exc:
log.exception("failed to create alias : {0}".format(exc))
raise exc
def _check_max_users(cls):
"""Check if maximum number of users reached."""
conf = Configuration('global').get('system', {})
max_users = conf.get('max_users', 0)
if max_users:
nb_users = User._model_class.objects.count()
if nb_users >= max_users:
raise ValueError('Max number of users reached')
def _check_max_users(cls):
"""Check if maximum number of users reached."""
conf = Configuration('global').get('system', {})
max_users = conf.get('max_users', 0)
if max_users:
nb_users = User._model_class.objects.count()
if nb_users >= max_users:
raise ValueError('Max number of users reached')
def main(args=sys.argv):
parser = argparse.ArgumentParser()
parser.add_argument('-f', dest='conffile', default='caliopen.yaml')
kwargs = parser.parse_args(args[1:])
kwargs = vars(kwargs)
config_uri = kwargs.pop('conffile')
Configuration.load(config_uri, 'global')
connect_storage()
shell(**kwargs)
def _check_whitelistes(cls, user):
"""Check if user is in a white list if apply."""
whitelistes = Configuration('global').get('whitelistes', {})
emails_file = whitelistes.get('user_emails')
if emails_file and os.path.isfile(emails_file):
with open(emails_file) as f:
emails = [x for x in f.read().split('\n') if x]
if user.recovery_email in emails:
return True
else:
raise ValueError('user email not in whitelist')
def _check_whitelistes(cls, user):
"""Check if user is in a white list if apply."""
whitelistes = Configuration('global').get('whitelistes', {})
emails_file = whitelistes.get('user_emails')
if emails_file and os.path.isfile(emails_file):
with open(emails_file) as f:
emails = [x for x in f.read().split('\n') if x]
if user.recovery_email in emails:
return True
else:
raise ValueError('user email not in whitelist')
def migrate_index(**kwargs):
Migrator = load_from_file(kwargs["input_script"])
if Migrator:
url = Configuration('global').get('elasticsearch.url')
mappings_version = Configuration('global').get(
'elasticsearch.mappings_version')
if url and mappings_version:
client = Elasticsearch(url)
migration = Migrator(client=client,
mappings_version=mappings_version)
migration.run()
def resync_index(**kwargs):
"""Resync an index for an user."""
from caliopen_main.user.core import User
from caliopen_main.user.core.setups import setup_index
from caliopen_main.contact.store import Contact
from caliopen_main.contact.objects import Contact as ContactObject
from caliopen_main.message.store import Message
from caliopen_main.message.objects import Message as MessageObject
if 'user_name' in kwargs and kwargs['user_name']:
user = User.by_name(kwargs['user_name'])
elif 'user_id' in kwargs and kwargs['user_id']:
user = User.get(kwargs['user_id'])
else:
print('Need user_name or user_id parameter')
sys.exit(1)
mapping_key = 'elasticsearch.mappings_version'
current_version = Configuration('global').get(mapping_key)
new_index = '{}_{}'.format(user.user_id, current_version)
if 'version' in kwargs and kwargs['version']:
old_version = kwargs['version']
old_index = '{}_{}'.format(user.user_id, old_version)
else:
old_index = new_index
es_url = Configuration('global').get('elasticsearch.url')
es_client = Elasticsearch(es_url)
# Delete current index
if not es_client.indices.exists([old_index]):
log.warn('Index %r not found for user %s' % (old_index, user.name))
sys.exit(1)
es_client.indices.delete([old_index])
log.info('Index %r deleted for user %s' % (old_index, user.name))
# Recreate index and mappings
setup_index(user)
contacts = Contact.filter(user_id=user.user_id)
for contact in contacts:
log.debug('Reindex contact %r' % contact.contact_id)
obj = ContactObject(user.user_id, contact_id=contact.contact_id)
obj.create_index()
messages = Message.filter(user_id=user.user_id).allow_filtering()
for message in messages:
log.debug('Reindex message %r' % message.message_id)
obj = MessageObject(user.user_id, message_id=message.message_id)
obj.create_index()
log.info('Create index alias %r' % user.user_id)
es_client.indices.put_alias(index=new_index, name=user.user_id)
def recreate_user_alias(client, user, dry_run=True):
"""Create an index alias mapping user_id -> shard_id."""
if not user.shard_id:
log.error('No shard for user {}'.format(user.user_id))
return False
shards = Configuration('global').get('elasticsearch.shards')
alias_exists = False
if client.indices.exists_alias(name=user.user_id):
alias = client.indices.get_alias(name=user.user_id)
for index, alias_infos in alias.items():
if index not in shards:
if not dry_run:
client.indices.delete_alias(index=index, name=user.user_id)
else:
log.info('Alias exist {} with index {}, should delete'.
format(user.user_id, index))
else:
log.info('Alias on shard exist, skipping')
alias_exists = True
if alias_exists:
return True
if not dry_run:
body = {'filter': {'term': {'user_id': user.user_id}}}
try:
client.indices.put_alias(index=user.shard_id,
name=user.user_id,
body=body)
except Exception as exc:
log.exception('Error during alias creation for user {} : {}'.
format(user.user_id, exc))
return False
else:
log.info('Should create alias {}'.format(user.user_id))
return True
def allocate_user_shard(user_id):
"""Find allocation to a shard for an user."""
shards = Configuration('global').get('elasticsearch.shards')
if not shards:
raise Exception('No shards configured for index')
shard_idx = int(user_id.hex, 16) % len(shards)
return shards[shard_idx]
def setup_storage(settings=None):
"""Create cassandra models."""
from caliopen_storage.core import core_registry
# Make discovery happen
from caliopen_main.user.core import User
from caliopen_main.user.core import (UserIdentity, IdentityLookup,
IdentityTypeLookup)
from caliopen_main.contact.objects.contact import Contact
from caliopen_main.message.objects.message import Message
from caliopen_main.common.objects.tag import ResourceTag
from caliopen_main.device.core import Device
from caliopen_main.notification.core import Notification, NotificationTtl
from cassandra.cqlengine.management import sync_table, \
create_keyspace_simple
keyspace = Configuration('global').get('cassandra.keyspace')
if not keyspace:
raise Exception('Configuration missing for cassandra keyspace')
# XXX tofix : define strategy and replication_factor in configuration
create_keyspace_simple(keyspace, 1)
for name, kls in core_registry.items():
log.info('Creating cassandra model %s' % name)
if hasattr(kls._model_class, 'pk'):
# XXX find a better way to detect model from udt
sync_table(kls._model_class)
def resync_shard_index(**kwargs):
"""Resync all index of a shard."""
from caliopen_main.user.core import User
from caliopen_main.user.core.setups import setup_shard_index
shard_id = kwargs['shard_id']
old_shard_id = kwargs.get('old_shard_id')
if not old_shard_id:
old_shard_id = shard_id
shards = Configuration('global').get('elasticsearch.shards')
if shard_id not in shards:
log.error('Invalid shard {0}'.format(shard_id))
sys.exit(1)
# Recreate index and mappings
setup_shard_index(shard_id)
users = User._model_class.all()
cpt = 0
for user in users:
if user.shard_id not in (old_shard_id, shard_id):
continue
if user.shard_id != shard_id:
user.shard_id = shard_id
user.save()
resync_user(user)
cpt += 1
log.info('Sync {0} users into shards'.format(cpt, shard_id))
def copy_model(**kwargs):
conf = Configuration('global').configuration
cluster_source = Cluster(conf['old_cassandra']['hosts'])
source = cluster_source.connect(conf['old_cassandra']['keyspace'])
source.row_factory = dict_factory
cluster_dest = Cluster(conf['cassandra']['hosts'])
dest = cluster_dest.connect(conf['cassandra']['keyspace'])
table = kwargs['model'].lower()
fetch_size = kwargs.get('fetch_size', 100)
query = "SELECT * FROM {0}".format(table)
if 'where' in kwargs and kwargs['where']:
query = "{0} WHERE {1} ALLOW FILTERING".format(query, kwargs['where'])
statement = SimpleStatement(query, fetch_size=fetch_size)
insert_query = "INSERT INTO {0} ({1}) VALUES ({2})"
cpt = 0
insert = None
for row in source.execute(statement):
if cpt == 0:
columns = ['"{}"'.format(x) for x in row.keys()]
binds = ['?' for x in range(0, len(columns))]
insert_str = insert_query.format(table, ','.join(columns),
','.join(binds))
insert = dest.prepare(insert_str)
bound = insert.bind(row.values())
dest.execute(bound)
cpt += 1
print('Copy of {} records from {}'.format(cpt, table))
return cpt
def get(cls, raw_msg_id):
"""
Get raw message from db or ObjectStorage service
:param raw_msg_id:
:return: a RawMessage or NotFound exception
"""
try:
raw_msg = super(RawMessage, cls).get(raw_msg_id)
except NotFound:
return NotFound
if raw_msg.raw_data == "" and raw_msg.uri != "":
# means raw message data have been stored in object store
# need to retrieve raw_data from it
minioConf = Configuration("global").get("object_store")
minioClient = Minio(minioConf["endpoint"],
access_key=minioConf["access_key"],
secret_key=minioConf["secret_key"],
secure=False,
region=minioConf["location"])
try:
resp = minioClient.get_object(
minioConf["buckets"]["raw_messages"], raw_msg_id)
except ResponseError as exc:
log.warn(exc)
return NotFound
# resp is a urllib3.response.HTTPResponse class
try:
raw_msg.raw_data = resp.data
except Exception as exc:
log.warn(exc)
return NotFound
return raw_msg
def resync_user(user):
"""Resync data for an user into its index shard."""
from caliopen_main.contact.store import Contact
from caliopen_main.contact.objects import Contact as ContactObject
from caliopen_main.message.store import Message
from caliopen_main.message.objects import Message as MessageObject
log.info('Sync user {0} into shard {1}'.format(user.user_id,
user.shard_id))
client = Elasticsearch(Configuration('global').get('elasticsearch.url'))
body = {'filter': {'term': {'user_id': user.user_id}}}
# if not client.indices.exists_alias(user.user_id):
# log.info('Creating alias {} for index {}'.format(user.user_id, user.shard_id))
client.indices.put_alias(user.shard_id, user.user_id, body=body)
contacts = Contact.filter(user_id=user.user_id).timeout(None)
for contact in contacts:
log.debug('Reindex contact %r' % contact.contact_id)
obj = ContactObject(user, contact_id=contact.contact_id)
obj.get_db()
obj.unmarshall_db()
obj.create_index()
messages = Message.filter(user_id=user.user_id). \
allow_filtering().timeout(None)
for message in messages:
log.debug('Reindex message %r' % message.message_id)
obj = MessageObject(user, message_id=message.message_id)
obj.get_db()
obj.unmarshall_db()
obj.create_index()
def validate_recovery_email(cls, email):
"""
provided email has to pass the validations below,
otherwise this func will raise an error
@:arg email: string
"""
# 1. is email well-formed ?
if not validate_email(email):
raise ValueError("recovery email malformed")
# 2. is email already in our db ? (recovery_email must be unique)
try:
UserRecoveryEmail.get(email)
except NotFound:
pass
else:
raise ValueError("recovery email already used in this instance")
# 3. is email belonging to one of our domains ?
# (recovery_email must be external)
domain = email.split("@")[1]
if domain in Configuration("global").get("default_domain"):
raise ValueError(
"recovery email must be outside of this domain instance")
def create_user(**kwargs):
"""Create user in Caliopen instance."""
from caliopen_main.user.core import User
from caliopen_main.user.parameters import NewUser
from caliopen_main.contact.parameters import NewContact
from caliopen_main.contact.parameters import NewEmail
# Fill core registry
from caliopen_main.message.objects.message import Message
param = NewUser()
param.name = kwargs['email']
if '@' in param.name:
username, domain = param.name.split('@')
param.name = username
# Monkey patch configuration local_domain with provided value
conf = Configuration('global').configuration
conf['default_domain'] = domain
param.password = kwargs['password']
param.recovery_email = u'{}@caliopen.local'.format(param.name)
contact = NewContact()
contact.given_name = kwargs.get('given_name')
contact.family_name = kwargs.get('family_name')
email = NewEmail()
email.address = param.recovery_email
contact.emails = [email]
param.contact = contact
user = User.create(param)
log.info('User %s (%s) created' % (user.user_id, user.name))
def setup_system_tags(self):
"""Create system tags."""
# TODO: translate tags'name to user's preferred language
default_tags = Configuration('global').get('system.default_tags')
for tag in default_tags:
tag['type'] = 'system'
tag['date_insert'] = datetime.datetime.now(tz=pytz.utc)
Tag.create(self, **tag)
def create_draft(cls, user_id=None, **params):
"""
Create and save a new message (draft) for an user.
:params: a NewMessage dict
"""
# silently remove unexpected props within patch if not in strict mode
strict_patch = Configuration('global').get('apiV1.strict_patch', False)
if not strict_patch:
allowed_properties = [
"body",
"identities",
"message_id",
"parent_id",
"participants",
"subject",
]
for key, value in params.items():
if key not in allowed_properties:
del (params[key])
if user_id is None or user_id is "":
raise ValueError
try:
draft_param = Draft(params, strict=strict_patch)
if draft_param.message_id:
draft_param.validate_uuid(user_id)
else:
draft_param.message_id = uuid.uuid4()
draft_param.validate_consistency(user_id, True)
except Exception as exc:
log.warn("draft_param error")
log.warn(exc)
raise exc
message = Message()
message.unmarshall_json_dict(draft_param.to_primitive())
message.user_id = UUID(user_id)
message.is_draft = True
message.is_received = False
message.type = "email" # TODO: type handling inferred from participants
message.date = message.date_sort = message.date_insert = \
datetime.datetime.now(tz=pytz.utc)
try:
message.marshall_db()
message.save_db()
except Exception as exc:
log.warn(exc)
raise exc
try:
message.marshall_index()
message.save_index(wait_for=True)
except Exception as exc:
log.warn(exc)
raise exc
return message
def setup_shard_index(shard):
"""Setup a shard index."""
url = Configuration('global').get('elasticsearch.url')
client = Elasticsearch(url)
try:
log.info('Creating index {0}'.format(shard))
client.indices.create(
index=shard,
body={
"settings": {
"analysis": {
"analyzer": {
"text_analyzer": {
"type": "custom",
"tokenizer": "lowercase",
"filter": [
"ascii_folding"
]
},
"email_analyzer": {
"type": "custom",
"tokenizer": "email_tokenizer",
"filter": [
"ascii_folding"
]
}
},
"filter": {
"ascii_folding": {
"type": "asciifolding",
"preserve_original": True
}
},
"tokenizer": {
"email_tokenizer": {
"type": "ngram",
"min_gram": 3,
"max_gram": 25
}
}
}
}
})
except Exception as exc:
log.warn("failed to create index {} : {}".format(shard, exc))
return
# PUT mappings for each type, if any
for name, kls in core_registry.items():
if hasattr(kls, "_index_class") and \
hasattr(kls._model_class, 'user_id'):
idx_kls = kls._index_class()
if hasattr(idx_kls, "build_mapping"):
log.debug('Init index mapping for {}'.format(idx_kls))
idx_kls.create_mapping(shard)
def setup_system_tags(user):
"""Create system tags."""
# TODO: translate tags'name to user's preferred language
default_tags = Configuration('global').get('system.default_tags')
for tag in default_tags:
tag['type'] = 'system'
tag['date_insert'] = datetime.datetime.now(tz=pytz.utc)
tag['label'] = tag.get('label', tag['name'])
from .user import Tag
Tag.create(user, **tag)
def create_all_shards(dry_run=True):
"""Create all needed index shards."""
client = get_index_connection()
shards = Configuration('global').get('elasticsearch.shards')
for shard_id in shards:
if not client.indices.exists(shard_id):
log.info('Creating shard {}'.format(shard_id))
if not dry_run:
setup_shard_index(shard_id)
def setup_shard_index(shard):
"""Setup a shard index."""
url = Configuration('global').get('elasticsearch.url')
client = Elasticsearch(url)
try:
log.info('Creating index {0}'.format(shard))
client.indices.create(index=shard,
body={
"settings": {
"analysis": {
"analyzer": {
"text_analyzer": {
"type": "custom",
"tokenizer": "lowercase",
"filter": ["ascii_folding"]
},
"email_analyzer": {
"type": "custom",
"tokenizer":
"email_tokenizer",
"filter": ["ascii_folding"]
}
},
"filter": {
"ascii_folding": {
"type": "asciifolding",
"preserve_original": True
}
},
"tokenizer": {
"email_tokenizer": {
"type": "ngram",
"min_gram": 3,
"max_gram": 25
}
}
}
}
})
except Exception as exc:
log.warn("failed to create index {} : {}".format(shard, exc))
return
# TOFIX
# core Message is no more in core_registry, use hard coded build mapping
from caliopen_main.message.store.message_index import IndexedMessage
from caliopen_main.contact.store.contact_index import IndexedContact
log.info(
'Creating index mapping for message and contact in shard {}'.format(
shard))
message_mapping = IndexedMessage.build_mapping()
message_mapping.save(shard, using=client)
contact_mapping = IndexedContact.build_mapping()
contact_mapping.save(shard, using=client)
def main(global_config, **settings):
"""Caliopen entry point for WSGI application.
Load Caliopen configuration and setup a WSGI application
with loaded API services.
"""
# XXX ugly way to init caliopen configuration before pyramid
caliopen_config = settings['caliopen.config'].split(':')[1]
Configuration.load(caliopen_config, 'global')
settings['pyramid_swagger.exclude_paths'] = [r'^/api-ui/?', r'^/doc/api/?', r'^/defs/?']
settings['pyramid_swagger.enable_response_validation'] = True
config = Configurator(settings=settings)
services = config.registry.settings. \
get('caliopen_api.services', []). \
split('\n')
route_prefix = settings.get('caliopen_api.route_prefix')
for service in services:
log.info('Loading %s service' % service)
config.include(service, route_prefix=route_prefix)
config.end()
return config.make_wsgi_app()
def main(global_config, **settings):
"""Caliopen entry point for WSGI application.
Load Caliopen configuration and setup a WSGI application
with loaded API services.
"""
# XXX ugly way to init caliopen configuration before pyramid
caliopen_config = settings['caliopen.config'].split(':')[1]
Configuration.load(caliopen_config, 'global')
settings['pyramid_swagger.exclude_paths'] = [
r'^/api-ui/?', r'^/doc/api/?', r'^/defs/?'
]
settings['pyramid_swagger.enable_response_validation'] = True
config = Configurator(settings=settings)
services = config.registry.settings. \
get('caliopen_api.services', []). \
split('\n')
route_prefix = settings.get('caliopen_api.route_prefix')
for service in services:
log.info('Loading %s service' % service)
config.include(service, route_prefix=route_prefix)
config.end()
return config.make_wsgi_app()
def main(user):
nc = Client()
try:
server = Configuration('global').get('message_queue')
opts = {"servers": ['nats://{}:{}'.format(server['host'],
server['port'])]}
print('Connecting to {}'.format(server))
yield nc.connect(**opts)
data = make_message(user)
yield nc.publish('userAction', data)
yield nc.flush()
print("Published to '{0}'".format(data))
except Exception as exc:
print(exc)
raise exc
def delete_all_shards(dry_run=True):
"""Delete all index shards."""
client = get_index_connection()
shards = Configuration('global').get('elasticsearch.shards')
for shard in shards:
log.info('Processing shard {}'.format(shard))
if not shard.startswith('caliopen-'):
log.warn('Invalid shard name, pass')
continue
if not client.indices.exists(shard):
log.warn('Shard does not exist')
continue
if dry_run:
log.info('Delete shard but dry run do not touch')
else:
client.indices.delete(shard)
log.info('Index {} deleted'.format(shard))
def _setup_user_index(self):
"""Create user index and setup mappings."""
url = Configuration('global').get('elasticsearch.url')
client = Elasticsearch(url)
log.debug('Creating index for user {}'.format(self.user_id))
if not client.indices.exists(self.user_id):
client.indices.create(self.user_id)
else:
log.warn('Index already exist {}'.format(self.user_id))
for name, kls in core_registry.items():
if hasattr(kls, "_index_class") and \
hasattr(kls._model_class, 'user_id'):
idx_kls = kls._index_class()
log.debug('Init index for {}'.format(idx_kls))
if hasattr(idx_kls, 'create_mapping'):
log.info('Create index {} mapping for doc_type {}'.format(
self.user_id, idx_kls.doc_type))
idx_kls.create_mapping(self.user_id)
def get(cls, raw_msg_id):
"""
Get raw message from db or ObjectStorage service
:param raw_msg_id:
:return: a RawMessage or NotFound exception
"""
try:
raw_msg = super(RawMessage, cls).get(raw_msg_id)
except Exception as exc:
log.warn(exc)
raise NotFound
if raw_msg.raw_data == "" and raw_msg.uri != "":
# means raw message data have been stored in object store
# need to retrieve raw_data from it
url = urlparse.urlsplit(raw_msg.uri)
path = url.path.strip("/")
if url.scheme == 's3':
minioConf = Configuration("global").get("object_store")
minioClient = Minio(minioConf["endpoint"],
access_key=minioConf["access_key"],
secret_key=minioConf["secret_key"],
secure=False,
region=minioConf["location"])
try:
resp = minioClient.get_object(url.netloc, path)
except Exception as exc:
log.warn(exc)
raise NotFound
# resp is a urllib3.response.HTTPResponse class
try:
raw_msg.raw_data = resp.data
except Exception as exc:
log.warn(exc)
raise NotFound
else:
log.warn("raw message uri scheme not implemented")
raise NotFound
return raw_msg
class IndexUser(object):
"""User index management class."""
__url__ = Configuration('global').get('elasticsearch.url')
@classmethod
def create(cls, user, **kwargs):
"""Create user index."""
# Create index for user
client = Elasticsearch(cls.__url__)
indice = IndicesClient(client)
if indice.exists(index=user.user_id):
if 'delete_existing' in kwargs and kwargs['delete_existing']:
log.warn('Deleting existing index for user %s' % user.user_id)
indice.delete(index=user.user_id)
else:
log.warn('Index already exists for user %s' % user.user_id)
return False
log.info('Creating index for user %s' % user.user_id)
indice.create(index=user.user_id)
return True
def import_email(email, import_path, format, contact_probability, **kwargs):
"""Import emails for an user."""
from caliopen_main.user.core import User
from caliopen_main.contact.core import Contact, ContactLookup
from caliopen_main.message.parsers.mail import MailMessage
from caliopen_main.contact.parameters import NewContact, NewEmail
from caliopen_nats.delivery import UserMessageDelivery
from caliopen_main.message.core import RawMessage
from caliopen_storage.config import Configuration
max_size = int(Configuration("global").get("object_store.db_size_limit"))
if format == 'maildir':
emails = Maildir(import_path, factory=message_from_file)
mode = 'maildir'
else:
if os.path.isdir(import_path):
mode = 'mbox_directory'
emails = {}
files = [
f for f in os.listdir(import_path)
if os.path.isfile(os.path.join(import_path, f))
]
for f in files:
try:
log.debug('Importing mail from file {}'.format(f))
with open('%s/%s' % (import_path, f)) as fh:
emails[f] = message_from_file(fh)
except Exception as exc:
log.error('Error importing email {}'.format(exc))
else:
mode = 'mbox'
emails = mbox(import_path)
user = User.by_local_identity(email)
log.info("Processing mode %s" % mode)
for key, data in emails.iteritems():
# Prevent creating message too large to fit in db.
# (should use inject cmd for large messages)
size = len(data.as_string())
if size > max_size:
log.warn("Message too large to fit into db. \
Please, use 'inject' cmd for importing large emails.")
continue
raw = RawMessage.create(data.as_string())
log.debug('Created raw message {}'.format(raw.raw_msg_id))
message = MailMessage(data.as_string())
dice = random()
if dice <= contact_probability:
for participant in message.participants:
try:
ContactLookup.get(user, participant.address)
except NotFound:
log.info('Creating contact %s' % participant.address)
name, domain = participant.address.split('@')
contact_param = NewContact()
contact_param.family_name = name
if participant.address:
e_mail = NewEmail()
e_mail.address = participant.address
contact_param.emails = [e_mail]
Contact.create(user, contact_param)
log.info('No contact associated to raw {} '.format(raw.raw_msg_id))
processor = UserMessageDelivery(user)
obj_message = processor.process_raw(raw.raw_msg_id)
log.info('Created message {}'.format(obj_message.message_id))
def __init__(self, filename):
from caliopen_storage.config import Configuration
from caliopen_storage.helpers.connection import connect_storage
self.conf = Configuration.load(filename, 'global').configuration
connect_storage()
def __init__(self, user):
"""Create a new instance of an user device qualifier."""
self.user = user
self.conf = Configuration('global').configuration
def patch_draft(self, user, patch, **options):
"""Operation specific to draft, before applying generic patch."""
try:
params = dict(patch)
except Exception as exc:
log.info(exc)
raise err.PatchError(message=exc.message)
# silently remove unexpected props within patch if not in strict mode
strict_patch = Configuration('global').get('apiV1.strict_patch', False)
if not strict_patch:
allowed_properties = [
"body",
"current_state",
"user_identities",
"message_id",
"parent_id",
"participants",
"subject",
"privacy_features",
]
for key, value in params.items():
if key not in allowed_properties:
del (params[key])
for key, value in params["current_state"].items():
if key not in allowed_properties:
del (params["current_state"][key])
try:
self.get_db()
self.unmarshall_db()
except Exception as exc:
log.info("patch_draft() failed to get msg from db: {}".format(exc))
raise exc
if not self.is_draft:
raise err.PatchUnprocessable(message="this message is not a draft")
try:
current_state = params.pop("current_state")
draft_param = Draft(params, strict=strict_patch)
except Exception as exc:
log.info(exc)
raise err.PatchError(message=exc.message)
# add missing params to be able to check consistency
self_dict = self.marshall_dict()
if "message_id" not in params and self.message_id:
draft_param.message_id = UUIDType().to_native(self.message_id)
if "parent_id" not in params and self.parent_id:
draft_param.parent_id = UUIDType().to_native(self.parent_id)
if "subject" not in params:
draft_param.subject = self.subject
if "participants" not in params and self.participants:
for participant in self_dict['participants']:
indexed = IndexedParticipant(participant)
draft_param.participants.append(indexed)
if "user_identities" not in params and self.user_identities:
draft_param.user_identities = self_dict["user_identities"]
# make sure the <from> participant is present
# and is consistent with selected user's identity
try:
new_discussion_id = draft_param.validate_consistency(user, False)
except Exception as exc:
log.info("consistency validation failed with err : {}".format(exc))
raise err.PatchError(message=exc.message)
validated_draft = draft_param.serialize()
validated_params = copy.deepcopy(params)
if "participants" in params:
validated_params["participants"] = validated_draft["participants"]
if new_discussion_id != self.discussion_id:
# discussion_id has changed, update draft's discussion_id
current_state["discussion_id"] = self.discussion_id
validated_params["discussion_id"] = new_discussion_id
# remove empty ids from current state if any
if "parent_id" in current_state and current_state["parent_id"] == "":
del (current_state["parent_id"])
# handle body key mapping to body_plain or body_html
# TODO: handle plain/html flag to map to right field
if "body" in validated_params:
validated_params["body_plain"] = validated_params["body"]
del (validated_params["body"])
if "body" in current_state:
current_state["body_plain"] = current_state["body"]
del (current_state["body"])
# date should reflect last edit time
current_state["date"] = self.date
current_state["date_sort"] = self.date_sort
validated_params["date"] = validated_params["date_sort"] = \
datetime.datetime.now(tz=pytz.utc)
validated_params["current_state"] = current_state
if "participants" in current_state and self.participants:
# replace participants' label and contact_ids
# because frontend has up-to-date data for these properties
# which are probably not the one stored in db
db_parts = {}
for p in self_dict['participants']:
db_parts[p['protocol'] + p['type'] + p['address']] = p
for i, p in enumerate(current_state['participants']):
current_state['participants'][i] = db_parts[p['protocol'] +
p['type'] +
p['address']]
try:
self.apply_patch(validated_params, **options)
except Exception as exc:
log.info("apply_patch() failed with error : {}".format(exc))
raise exc
from caliopen_storage.config import Configuration
from caliopen_storage.helpers.connection import connect_storage
from caliopen_main.common.errors import PatchConflict
from caliopen_storage.exception import NotFound
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('-f', dest='conffile')
parser.add_argument('-t', dest='test', action='store_true', default=False)
args = parser.parse_args()
Configuration.load(args.conffile, 'global')
connect_storage()
from caliopen_main.user.core.user import LocalIdentity
from caliopen_main.message.store.message import Message
from caliopen_main.message.objects.message import Message as ObjMessage
identities = [x.identifier for x in LocalIdentity._model_class.all()]
cpt = {'received': 0, 'sent': 0}
for message in Message.all():
if message.is_received is None:
from_participant = [x for x in message.participants
if x['type'] == 'From']
if not from_participant:
log.error('No from participant found for message {} : {}'.
format(message.message_id, message.participants))
else:
def main(args=sys.argv):
parser = argparse.ArgumentParser()
parser.add_argument('-f', dest='conffile', default='development.ini')
subparsers = parser.add_subparsers(title="action")
sp_import = subparsers.add_parser('import', help='import existing mailbox')
sp_import.set_defaults(func=import_email)
sp_import.add_argument('-f', dest='format', choices=['mbox', 'maildir'],
default='mbox')
sp_import.add_argument('-p', dest='import_path')
sp_import.add_argument('-e', dest='email')
sp_import.add_argument('--contact-probability', dest='contact_probability',
default=1.0)
sp_import.add_argument('-t', dest='to')
sp_import_vcard = subparsers.add_parser('import_vcard',
help='import vcard')
sp_import_vcard.set_defaults(func=import_vcard)
sp_import_vcard.add_argument('-u', dest='username', help='username')
sp_import_vcard.add_argument('-d', dest='directory', help='directory')
sp_import_vcard.add_argument('-f', dest='file_vcard', help='file')
sp_setup = subparsers.add_parser('setup',
help='initialize the storage engine')
sp_setup.set_defaults(func=setup)
sp_create_user = subparsers.add_parser('create_user',
help='Create a new user')
sp_create_user.set_defaults(func=create_user)
sp_create_user.add_argument('-e', dest='email', help='user email')
sp_create_user.add_argument('-p', dest='password', help='password')
sp_create_user.add_argument('-g', dest='given_name',
help='user given name')
sp_create_user.add_argument('-f', dest='family_name',
help='user family name')
sp_shell = subparsers.add_parser('shell')
sp_shell.set_defaults(func=shell)
sp_dump = subparsers.add_parser('dump')
sp_dump.set_defaults(func=dump_model)
sp_dump.add_argument('-m', dest='model', help='model to dump')
sp_dump.add_argument('-o', dest='output_path', help='output path')
sp_copy = subparsers.add_parser('copy')
sp_copy.set_defaults(func=copy_model)
sp_copy.add_argument('-m', dest='model', help='model to dump')
sp_copy.add_argument('--where', dest='where', help='where condition')
sp_copy.add_argument('--fetch-size', dest='fetch_size', default=100)
sp_dump_index = subparsers.add_parser('dump_index')
sp_dump_index.set_defaults(func=dump_indexes)
sp_dump_index.add_argument('-o', dest='output_path', help='output path')
sp_migrate_index = subparsers.add_parser('migrate_index')
sp_migrate_index.set_defaults(func=migrate_index)
sp_migrate_index.add_argument('-s', dest='input_script',
help='python script to execute on index')
sp_inject = subparsers.add_parser('inject')
sp_inject.set_defaults(func=inject_email)
sp_inject.add_argument('-e', dest='email')
sp_inject.add_argument('-r', dest='recipient')
sp_compute = subparsers.add_parser('compute', help='Launch basic compute')
sp_compute.set_defaults(func=basic_compute)
sp_compute.add_argument('-u', dest='username', help='username')
sp_compute.add_argument('-j', dest='job', help='job name')
sp_reserved = subparsers.add_parser('reserved_names',
help='Import reserved names list')
sp_reserved.set_defaults(func=import_reserved_names)
sp_reserved.add_argument('-i', dest='input_file', help='csv file')
sp_resync = subparsers.add_parser('resync_index',
help='Resync index for an user')
sp_resync.set_defaults(func=resync_index)
sp_resync.add_argument('-u', dest='user_name', help='User name')
sp_resync.add_argument('-i', dest='user_id', help='User uuid')
sp_resync.add_argument('--delete', dest='delete', action='store_true')
sp_resync = subparsers.add_parser('resync_shard',
help='Resync shard index')
sp_resync.set_defaults(func=resync_shard_index)
sp_resync.add_argument('-s', dest='shard_id', help='Shard id')
sp_resync.add_argument('-o', dest='old_shard_id', help='Old shard id')
kwargs = parser.parse_args(args[1:])
kwargs = vars(kwargs)
config_uri = kwargs.pop('conffile')
func = kwargs.pop('func')
Configuration.load(config_uri, 'global')
connect_storage()
func(**kwargs)
"""Test importance level compute."""
import unittest
import os
from caliopen_storage.config import Configuration
if 'CALIOPEN_BASEDIR' in os.environ:
conf_file = '{}/src/backend/configs/caliopen.yaml.template'. \
format(os.environ['CALIOPEN_BASEDIR'])
else:
conf_file = '../../../../../configs/caliopen.yaml.template'
Configuration.load(conf_file, 'global')
from caliopen_pi.features.helpers.importance_level import compute_importance
class MockPI(object):
def __init__(self, technic, context, comportment):
self.technic = technic
self.context = context
self.comportment = comportment
class MockMessage(object):
def __init__(self, pi, tags=None, refs=None):
self.pi = pi
self.tags = tags if tags else []