def setup_index(user): """Creates user index and setups mappings.""" url = Configuration('global').get('elasticsearch.url') client = Elasticsearch(url) shard_id = user.shard_id # does shard exist ? if not client.indices.exists(shard_id): setup_shard_index(shard_id) # Creates a versioned index with our custom analyzers, tokenizers, etc. m_version = Configuration('global').get('elasticsearch.mappings_version') if m_version == "": raise Exception( 'Invalid mapping version for shard {0}'.format(shard_id)) index_name = shard_id alias_name = user.user_id # Points an alias to the underlying user's index try: client.indices.put_alias(index=index_name, name=alias_name) except Exception as exc: log.exception("failed to create alias : {0}".format(exc)) raise exc
def _check_max_users(cls): """Check if maximum number of users reached.""" conf = Configuration('global').get('system', {}) max_users = conf.get('max_users', 0) if max_users: nb_users = User._model_class.objects.count() if nb_users >= max_users: raise ValueError('Max number of users reached')
def _check_max_users(cls): """Check if maximum number of users reached.""" conf = Configuration('global').get('system', {}) max_users = conf.get('max_users', 0) if max_users: nb_users = User._model_class.objects.count() if nb_users >= max_users: raise ValueError('Max number of users reached')
def main(args=sys.argv): parser = argparse.ArgumentParser() parser.add_argument('-f', dest='conffile', default='caliopen.yaml') kwargs = parser.parse_args(args[1:]) kwargs = vars(kwargs) config_uri = kwargs.pop('conffile') Configuration.load(config_uri, 'global') connect_storage() shell(**kwargs)
def _check_whitelistes(cls, user): """Check if user is in a white list if apply.""" whitelistes = Configuration('global').get('whitelistes', {}) emails_file = whitelistes.get('user_emails') if emails_file and os.path.isfile(emails_file): with open(emails_file) as f: emails = [x for x in f.read().split('\n') if x] if user.recovery_email in emails: return True else: raise ValueError('user email not in whitelist')
def _check_whitelistes(cls, user): """Check if user is in a white list if apply.""" whitelistes = Configuration('global').get('whitelistes', {}) emails_file = whitelistes.get('user_emails') if emails_file and os.path.isfile(emails_file): with open(emails_file) as f: emails = [x for x in f.read().split('\n') if x] if user.recovery_email in emails: return True else: raise ValueError('user email not in whitelist')
def migrate_index(**kwargs): Migrator = load_from_file(kwargs["input_script"]) if Migrator: url = Configuration('global').get('elasticsearch.url') mappings_version = Configuration('global').get( 'elasticsearch.mappings_version') if url and mappings_version: client = Elasticsearch(url) migration = Migrator(client=client, mappings_version=mappings_version) migration.run()
def resync_index(**kwargs): """Resync an index for an user.""" from caliopen_main.user.core import User from caliopen_main.user.core.setups import setup_index from caliopen_main.contact.store import Contact from caliopen_main.contact.objects import Contact as ContactObject from caliopen_main.message.store import Message from caliopen_main.message.objects import Message as MessageObject if 'user_name' in kwargs and kwargs['user_name']: user = User.by_name(kwargs['user_name']) elif 'user_id' in kwargs and kwargs['user_id']: user = User.get(kwargs['user_id']) else: print('Need user_name or user_id parameter') sys.exit(1) mapping_key = 'elasticsearch.mappings_version' current_version = Configuration('global').get(mapping_key) new_index = '{}_{}'.format(user.user_id, current_version) if 'version' in kwargs and kwargs['version']: old_version = kwargs['version'] old_index = '{}_{}'.format(user.user_id, old_version) else: old_index = new_index es_url = Configuration('global').get('elasticsearch.url') es_client = Elasticsearch(es_url) # Delete current index if not es_client.indices.exists([old_index]): log.warn('Index %r not found for user %s' % (old_index, user.name)) sys.exit(1) es_client.indices.delete([old_index]) log.info('Index %r deleted for user %s' % (old_index, user.name)) # Recreate index and mappings setup_index(user) contacts = Contact.filter(user_id=user.user_id) for contact in contacts: log.debug('Reindex contact %r' % contact.contact_id) obj = ContactObject(user.user_id, contact_id=contact.contact_id) obj.create_index() messages = Message.filter(user_id=user.user_id).allow_filtering() for message in messages: log.debug('Reindex message %r' % message.message_id) obj = MessageObject(user.user_id, message_id=message.message_id) obj.create_index() log.info('Create index alias %r' % user.user_id) es_client.indices.put_alias(index=new_index, name=user.user_id)
def recreate_user_alias(client, user, dry_run=True): """Create an index alias mapping user_id -> shard_id.""" if not user.shard_id: log.error('No shard for user {}'.format(user.user_id)) return False shards = Configuration('global').get('elasticsearch.shards') alias_exists = False if client.indices.exists_alias(name=user.user_id): alias = client.indices.get_alias(name=user.user_id) for index, alias_infos in alias.items(): if index not in shards: if not dry_run: client.indices.delete_alias(index=index, name=user.user_id) else: log.info('Alias exist {} with index {}, should delete'. format(user.user_id, index)) else: log.info('Alias on shard exist, skipping') alias_exists = True if alias_exists: return True if not dry_run: body = {'filter': {'term': {'user_id': user.user_id}}} try: client.indices.put_alias(index=user.shard_id, name=user.user_id, body=body) except Exception as exc: log.exception('Error during alias creation for user {} : {}'. format(user.user_id, exc)) return False else: log.info('Should create alias {}'.format(user.user_id)) return True
def allocate_user_shard(user_id): """Find allocation to a shard for an user.""" shards = Configuration('global').get('elasticsearch.shards') if not shards: raise Exception('No shards configured for index') shard_idx = int(user_id.hex, 16) % len(shards) return shards[shard_idx]
def setup_storage(settings=None): """Create cassandra models.""" from caliopen_storage.core import core_registry # Make discovery happen from caliopen_main.user.core import User from caliopen_main.user.core import (UserIdentity, IdentityLookup, IdentityTypeLookup) from caliopen_main.contact.objects.contact import Contact from caliopen_main.message.objects.message import Message from caliopen_main.common.objects.tag import ResourceTag from caliopen_main.device.core import Device from caliopen_main.notification.core import Notification, NotificationTtl from cassandra.cqlengine.management import sync_table, \ create_keyspace_simple keyspace = Configuration('global').get('cassandra.keyspace') if not keyspace: raise Exception('Configuration missing for cassandra keyspace') # XXX tofix : define strategy and replication_factor in configuration create_keyspace_simple(keyspace, 1) for name, kls in core_registry.items(): log.info('Creating cassandra model %s' % name) if hasattr(kls._model_class, 'pk'): # XXX find a better way to detect model from udt sync_table(kls._model_class)
def resync_shard_index(**kwargs): """Resync all index of a shard.""" from caliopen_main.user.core import User from caliopen_main.user.core.setups import setup_shard_index shard_id = kwargs['shard_id'] old_shard_id = kwargs.get('old_shard_id') if not old_shard_id: old_shard_id = shard_id shards = Configuration('global').get('elasticsearch.shards') if shard_id not in shards: log.error('Invalid shard {0}'.format(shard_id)) sys.exit(1) # Recreate index and mappings setup_shard_index(shard_id) users = User._model_class.all() cpt = 0 for user in users: if user.shard_id not in (old_shard_id, shard_id): continue if user.shard_id != shard_id: user.shard_id = shard_id user.save() resync_user(user) cpt += 1 log.info('Sync {0} users into shards'.format(cpt, shard_id))
def copy_model(**kwargs): conf = Configuration('global').configuration cluster_source = Cluster(conf['old_cassandra']['hosts']) source = cluster_source.connect(conf['old_cassandra']['keyspace']) source.row_factory = dict_factory cluster_dest = Cluster(conf['cassandra']['hosts']) dest = cluster_dest.connect(conf['cassandra']['keyspace']) table = kwargs['model'].lower() fetch_size = kwargs.get('fetch_size', 100) query = "SELECT * FROM {0}".format(table) if 'where' in kwargs and kwargs['where']: query = "{0} WHERE {1} ALLOW FILTERING".format(query, kwargs['where']) statement = SimpleStatement(query, fetch_size=fetch_size) insert_query = "INSERT INTO {0} ({1}) VALUES ({2})" cpt = 0 insert = None for row in source.execute(statement): if cpt == 0: columns = ['"{}"'.format(x) for x in row.keys()] binds = ['?' for x in range(0, len(columns))] insert_str = insert_query.format(table, ','.join(columns), ','.join(binds)) insert = dest.prepare(insert_str) bound = insert.bind(row.values()) dest.execute(bound) cpt += 1 print('Copy of {} records from {}'.format(cpt, table)) return cpt
def get(cls, raw_msg_id): """ Get raw message from db or ObjectStorage service :param raw_msg_id: :return: a RawMessage or NotFound exception """ try: raw_msg = super(RawMessage, cls).get(raw_msg_id) except NotFound: return NotFound if raw_msg.raw_data == "" and raw_msg.uri != "": # means raw message data have been stored in object store # need to retrieve raw_data from it minioConf = Configuration("global").get("object_store") minioClient = Minio(minioConf["endpoint"], access_key=minioConf["access_key"], secret_key=minioConf["secret_key"], secure=False, region=minioConf["location"]) try: resp = minioClient.get_object( minioConf["buckets"]["raw_messages"], raw_msg_id) except ResponseError as exc: log.warn(exc) return NotFound # resp is a urllib3.response.HTTPResponse class try: raw_msg.raw_data = resp.data except Exception as exc: log.warn(exc) return NotFound return raw_msg
def resync_user(user): """Resync data for an user into its index shard.""" from caliopen_main.contact.store import Contact from caliopen_main.contact.objects import Contact as ContactObject from caliopen_main.message.store import Message from caliopen_main.message.objects import Message as MessageObject log.info('Sync user {0} into shard {1}'.format(user.user_id, user.shard_id)) client = Elasticsearch(Configuration('global').get('elasticsearch.url')) body = {'filter': {'term': {'user_id': user.user_id}}} # if not client.indices.exists_alias(user.user_id): # log.info('Creating alias {} for index {}'.format(user.user_id, user.shard_id)) client.indices.put_alias(user.shard_id, user.user_id, body=body) contacts = Contact.filter(user_id=user.user_id).timeout(None) for contact in contacts: log.debug('Reindex contact %r' % contact.contact_id) obj = ContactObject(user, contact_id=contact.contact_id) obj.get_db() obj.unmarshall_db() obj.create_index() messages = Message.filter(user_id=user.user_id). \ allow_filtering().timeout(None) for message in messages: log.debug('Reindex message %r' % message.message_id) obj = MessageObject(user, message_id=message.message_id) obj.get_db() obj.unmarshall_db() obj.create_index()
def validate_recovery_email(cls, email): """ provided email has to pass the validations below, otherwise this func will raise an error @:arg email: string """ # 1. is email well-formed ? if not validate_email(email): raise ValueError("recovery email malformed") # 2. is email already in our db ? (recovery_email must be unique) try: UserRecoveryEmail.get(email) except NotFound: pass else: raise ValueError("recovery email already used in this instance") # 3. is email belonging to one of our domains ? # (recovery_email must be external) domain = email.split("@")[1] if domain in Configuration("global").get("default_domain"): raise ValueError( "recovery email must be outside of this domain instance")
def create_user(**kwargs): """Create user in Caliopen instance.""" from caliopen_main.user.core import User from caliopen_main.user.parameters import NewUser from caliopen_main.contact.parameters import NewContact from caliopen_main.contact.parameters import NewEmail # Fill core registry from caliopen_main.message.objects.message import Message param = NewUser() param.name = kwargs['email'] if '@' in param.name: username, domain = param.name.split('@') param.name = username # Monkey patch configuration local_domain with provided value conf = Configuration('global').configuration conf['default_domain'] = domain param.password = kwargs['password'] param.recovery_email = u'{}@caliopen.local'.format(param.name) contact = NewContact() contact.given_name = kwargs.get('given_name') contact.family_name = kwargs.get('family_name') email = NewEmail() email.address = param.recovery_email contact.emails = [email] param.contact = contact user = User.create(param) log.info('User %s (%s) created' % (user.user_id, user.name))
def setup_system_tags(self): """Create system tags.""" # TODO: translate tags'name to user's preferred language default_tags = Configuration('global').get('system.default_tags') for tag in default_tags: tag['type'] = 'system' tag['date_insert'] = datetime.datetime.now(tz=pytz.utc) Tag.create(self, **tag)
def create_draft(cls, user_id=None, **params): """ Create and save a new message (draft) for an user. :params: a NewMessage dict """ # silently remove unexpected props within patch if not in strict mode strict_patch = Configuration('global').get('apiV1.strict_patch', False) if not strict_patch: allowed_properties = [ "body", "identities", "message_id", "parent_id", "participants", "subject", ] for key, value in params.items(): if key not in allowed_properties: del (params[key]) if user_id is None or user_id is "": raise ValueError try: draft_param = Draft(params, strict=strict_patch) if draft_param.message_id: draft_param.validate_uuid(user_id) else: draft_param.message_id = uuid.uuid4() draft_param.validate_consistency(user_id, True) except Exception as exc: log.warn("draft_param error") log.warn(exc) raise exc message = Message() message.unmarshall_json_dict(draft_param.to_primitive()) message.user_id = UUID(user_id) message.is_draft = True message.is_received = False message.type = "email" # TODO: type handling inferred from participants message.date = message.date_sort = message.date_insert = \ datetime.datetime.now(tz=pytz.utc) try: message.marshall_db() message.save_db() except Exception as exc: log.warn(exc) raise exc try: message.marshall_index() message.save_index(wait_for=True) except Exception as exc: log.warn(exc) raise exc return message
def setup_shard_index(shard): """Setup a shard index.""" url = Configuration('global').get('elasticsearch.url') client = Elasticsearch(url) try: log.info('Creating index {0}'.format(shard)) client.indices.create( index=shard, body={ "settings": { "analysis": { "analyzer": { "text_analyzer": { "type": "custom", "tokenizer": "lowercase", "filter": [ "ascii_folding" ] }, "email_analyzer": { "type": "custom", "tokenizer": "email_tokenizer", "filter": [ "ascii_folding" ] } }, "filter": { "ascii_folding": { "type": "asciifolding", "preserve_original": True } }, "tokenizer": { "email_tokenizer": { "type": "ngram", "min_gram": 3, "max_gram": 25 } } } } }) except Exception as exc: log.warn("failed to create index {} : {}".format(shard, exc)) return # PUT mappings for each type, if any for name, kls in core_registry.items(): if hasattr(kls, "_index_class") and \ hasattr(kls._model_class, 'user_id'): idx_kls = kls._index_class() if hasattr(idx_kls, "build_mapping"): log.debug('Init index mapping for {}'.format(idx_kls)) idx_kls.create_mapping(shard)
def setup_system_tags(user): """Create system tags.""" # TODO: translate tags'name to user's preferred language default_tags = Configuration('global').get('system.default_tags') for tag in default_tags: tag['type'] = 'system' tag['date_insert'] = datetime.datetime.now(tz=pytz.utc) tag['label'] = tag.get('label', tag['name']) from .user import Tag Tag.create(user, **tag)
def create_all_shards(dry_run=True): """Create all needed index shards.""" client = get_index_connection() shards = Configuration('global').get('elasticsearch.shards') for shard_id in shards: if not client.indices.exists(shard_id): log.info('Creating shard {}'.format(shard_id)) if not dry_run: setup_shard_index(shard_id)
def setup_shard_index(shard): """Setup a shard index.""" url = Configuration('global').get('elasticsearch.url') client = Elasticsearch(url) try: log.info('Creating index {0}'.format(shard)) client.indices.create(index=shard, body={ "settings": { "analysis": { "analyzer": { "text_analyzer": { "type": "custom", "tokenizer": "lowercase", "filter": ["ascii_folding"] }, "email_analyzer": { "type": "custom", "tokenizer": "email_tokenizer", "filter": ["ascii_folding"] } }, "filter": { "ascii_folding": { "type": "asciifolding", "preserve_original": True } }, "tokenizer": { "email_tokenizer": { "type": "ngram", "min_gram": 3, "max_gram": 25 } } } } }) except Exception as exc: log.warn("failed to create index {} : {}".format(shard, exc)) return # TOFIX # core Message is no more in core_registry, use hard coded build mapping from caliopen_main.message.store.message_index import IndexedMessage from caliopen_main.contact.store.contact_index import IndexedContact log.info( 'Creating index mapping for message and contact in shard {}'.format( shard)) message_mapping = IndexedMessage.build_mapping() message_mapping.save(shard, using=client) contact_mapping = IndexedContact.build_mapping() contact_mapping.save(shard, using=client)
def main(global_config, **settings): """Caliopen entry point for WSGI application. Load Caliopen configuration and setup a WSGI application with loaded API services. """ # XXX ugly way to init caliopen configuration before pyramid caliopen_config = settings['caliopen.config'].split(':')[1] Configuration.load(caliopen_config, 'global') settings['pyramid_swagger.exclude_paths'] = [r'^/api-ui/?', r'^/doc/api/?', r'^/defs/?'] settings['pyramid_swagger.enable_response_validation'] = True config = Configurator(settings=settings) services = config.registry.settings. \ get('caliopen_api.services', []). \ split('\n') route_prefix = settings.get('caliopen_api.route_prefix') for service in services: log.info('Loading %s service' % service) config.include(service, route_prefix=route_prefix) config.end() return config.make_wsgi_app()
def main(global_config, **settings): """Caliopen entry point for WSGI application. Load Caliopen configuration and setup a WSGI application with loaded API services. """ # XXX ugly way to init caliopen configuration before pyramid caliopen_config = settings['caliopen.config'].split(':')[1] Configuration.load(caliopen_config, 'global') settings['pyramid_swagger.exclude_paths'] = [ r'^/api-ui/?', r'^/doc/api/?', r'^/defs/?' ] settings['pyramid_swagger.enable_response_validation'] = True config = Configurator(settings=settings) services = config.registry.settings. \ get('caliopen_api.services', []). \ split('\n') route_prefix = settings.get('caliopen_api.route_prefix') for service in services: log.info('Loading %s service' % service) config.include(service, route_prefix=route_prefix) config.end() return config.make_wsgi_app()
def main(user): nc = Client() try: server = Configuration('global').get('message_queue') opts = {"servers": ['nats://{}:{}'.format(server['host'], server['port'])]} print('Connecting to {}'.format(server)) yield nc.connect(**opts) data = make_message(user) yield nc.publish('userAction', data) yield nc.flush() print("Published to '{0}'".format(data)) except Exception as exc: print(exc) raise exc
def delete_all_shards(dry_run=True): """Delete all index shards.""" client = get_index_connection() shards = Configuration('global').get('elasticsearch.shards') for shard in shards: log.info('Processing shard {}'.format(shard)) if not shard.startswith('caliopen-'): log.warn('Invalid shard name, pass') continue if not client.indices.exists(shard): log.warn('Shard does not exist') continue if dry_run: log.info('Delete shard but dry run do not touch') else: client.indices.delete(shard) log.info('Index {} deleted'.format(shard))
def _setup_user_index(self): """Create user index and setup mappings.""" url = Configuration('global').get('elasticsearch.url') client = Elasticsearch(url) log.debug('Creating index for user {}'.format(self.user_id)) if not client.indices.exists(self.user_id): client.indices.create(self.user_id) else: log.warn('Index already exist {}'.format(self.user_id)) for name, kls in core_registry.items(): if hasattr(kls, "_index_class") and \ hasattr(kls._model_class, 'user_id'): idx_kls = kls._index_class() log.debug('Init index for {}'.format(idx_kls)) if hasattr(idx_kls, 'create_mapping'): log.info('Create index {} mapping for doc_type {}'.format( self.user_id, idx_kls.doc_type)) idx_kls.create_mapping(self.user_id)
def get(cls, raw_msg_id): """ Get raw message from db or ObjectStorage service :param raw_msg_id: :return: a RawMessage or NotFound exception """ try: raw_msg = super(RawMessage, cls).get(raw_msg_id) except Exception as exc: log.warn(exc) raise NotFound if raw_msg.raw_data == "" and raw_msg.uri != "": # means raw message data have been stored in object store # need to retrieve raw_data from it url = urlparse.urlsplit(raw_msg.uri) path = url.path.strip("/") if url.scheme == 's3': minioConf = Configuration("global").get("object_store") minioClient = Minio(minioConf["endpoint"], access_key=minioConf["access_key"], secret_key=minioConf["secret_key"], secure=False, region=minioConf["location"]) try: resp = minioClient.get_object(url.netloc, path) except Exception as exc: log.warn(exc) raise NotFound # resp is a urllib3.response.HTTPResponse class try: raw_msg.raw_data = resp.data except Exception as exc: log.warn(exc) raise NotFound else: log.warn("raw message uri scheme not implemented") raise NotFound return raw_msg
class IndexUser(object): """User index management class.""" __url__ = Configuration('global').get('elasticsearch.url') @classmethod def create(cls, user, **kwargs): """Create user index.""" # Create index for user client = Elasticsearch(cls.__url__) indice = IndicesClient(client) if indice.exists(index=user.user_id): if 'delete_existing' in kwargs and kwargs['delete_existing']: log.warn('Deleting existing index for user %s' % user.user_id) indice.delete(index=user.user_id) else: log.warn('Index already exists for user %s' % user.user_id) return False log.info('Creating index for user %s' % user.user_id) indice.create(index=user.user_id) return True
def import_email(email, import_path, format, contact_probability, **kwargs): """Import emails for an user.""" from caliopen_main.user.core import User from caliopen_main.contact.core import Contact, ContactLookup from caliopen_main.message.parsers.mail import MailMessage from caliopen_main.contact.parameters import NewContact, NewEmail from caliopen_nats.delivery import UserMessageDelivery from caliopen_main.message.core import RawMessage from caliopen_storage.config import Configuration max_size = int(Configuration("global").get("object_store.db_size_limit")) if format == 'maildir': emails = Maildir(import_path, factory=message_from_file) mode = 'maildir' else: if os.path.isdir(import_path): mode = 'mbox_directory' emails = {} files = [ f for f in os.listdir(import_path) if os.path.isfile(os.path.join(import_path, f)) ] for f in files: try: log.debug('Importing mail from file {}'.format(f)) with open('%s/%s' % (import_path, f)) as fh: emails[f] = message_from_file(fh) except Exception as exc: log.error('Error importing email {}'.format(exc)) else: mode = 'mbox' emails = mbox(import_path) user = User.by_local_identity(email) log.info("Processing mode %s" % mode) for key, data in emails.iteritems(): # Prevent creating message too large to fit in db. # (should use inject cmd for large messages) size = len(data.as_string()) if size > max_size: log.warn("Message too large to fit into db. \ Please, use 'inject' cmd for importing large emails.") continue raw = RawMessage.create(data.as_string()) log.debug('Created raw message {}'.format(raw.raw_msg_id)) message = MailMessage(data.as_string()) dice = random() if dice <= contact_probability: for participant in message.participants: try: ContactLookup.get(user, participant.address) except NotFound: log.info('Creating contact %s' % participant.address) name, domain = participant.address.split('@') contact_param = NewContact() contact_param.family_name = name if participant.address: e_mail = NewEmail() e_mail.address = participant.address contact_param.emails = [e_mail] Contact.create(user, contact_param) log.info('No contact associated to raw {} '.format(raw.raw_msg_id)) processor = UserMessageDelivery(user) obj_message = processor.process_raw(raw.raw_msg_id) log.info('Created message {}'.format(obj_message.message_id))
def __init__(self, filename): from caliopen_storage.config import Configuration from caliopen_storage.helpers.connection import connect_storage self.conf = Configuration.load(filename, 'global').configuration connect_storage()
def __init__(self, user): """Create a new instance of an user device qualifier.""" self.user = user self.conf = Configuration('global').configuration
def patch_draft(self, user, patch, **options): """Operation specific to draft, before applying generic patch.""" try: params = dict(patch) except Exception as exc: log.info(exc) raise err.PatchError(message=exc.message) # silently remove unexpected props within patch if not in strict mode strict_patch = Configuration('global').get('apiV1.strict_patch', False) if not strict_patch: allowed_properties = [ "body", "current_state", "user_identities", "message_id", "parent_id", "participants", "subject", "privacy_features", ] for key, value in params.items(): if key not in allowed_properties: del (params[key]) for key, value in params["current_state"].items(): if key not in allowed_properties: del (params["current_state"][key]) try: self.get_db() self.unmarshall_db() except Exception as exc: log.info("patch_draft() failed to get msg from db: {}".format(exc)) raise exc if not self.is_draft: raise err.PatchUnprocessable(message="this message is not a draft") try: current_state = params.pop("current_state") draft_param = Draft(params, strict=strict_patch) except Exception as exc: log.info(exc) raise err.PatchError(message=exc.message) # add missing params to be able to check consistency self_dict = self.marshall_dict() if "message_id" not in params and self.message_id: draft_param.message_id = UUIDType().to_native(self.message_id) if "parent_id" not in params and self.parent_id: draft_param.parent_id = UUIDType().to_native(self.parent_id) if "subject" not in params: draft_param.subject = self.subject if "participants" not in params and self.participants: for participant in self_dict['participants']: indexed = IndexedParticipant(participant) draft_param.participants.append(indexed) if "user_identities" not in params and self.user_identities: draft_param.user_identities = self_dict["user_identities"] # make sure the <from> participant is present # and is consistent with selected user's identity try: new_discussion_id = draft_param.validate_consistency(user, False) except Exception as exc: log.info("consistency validation failed with err : {}".format(exc)) raise err.PatchError(message=exc.message) validated_draft = draft_param.serialize() validated_params = copy.deepcopy(params) if "participants" in params: validated_params["participants"] = validated_draft["participants"] if new_discussion_id != self.discussion_id: # discussion_id has changed, update draft's discussion_id current_state["discussion_id"] = self.discussion_id validated_params["discussion_id"] = new_discussion_id # remove empty ids from current state if any if "parent_id" in current_state and current_state["parent_id"] == "": del (current_state["parent_id"]) # handle body key mapping to body_plain or body_html # TODO: handle plain/html flag to map to right field if "body" in validated_params: validated_params["body_plain"] = validated_params["body"] del (validated_params["body"]) if "body" in current_state: current_state["body_plain"] = current_state["body"] del (current_state["body"]) # date should reflect last edit time current_state["date"] = self.date current_state["date_sort"] = self.date_sort validated_params["date"] = validated_params["date_sort"] = \ datetime.datetime.now(tz=pytz.utc) validated_params["current_state"] = current_state if "participants" in current_state and self.participants: # replace participants' label and contact_ids # because frontend has up-to-date data for these properties # which are probably not the one stored in db db_parts = {} for p in self_dict['participants']: db_parts[p['protocol'] + p['type'] + p['address']] = p for i, p in enumerate(current_state['participants']): current_state['participants'][i] = db_parts[p['protocol'] + p['type'] + p['address']] try: self.apply_patch(validated_params, **options) except Exception as exc: log.info("apply_patch() failed with error : {}".format(exc)) raise exc
from caliopen_storage.config import Configuration from caliopen_storage.helpers.connection import connect_storage from caliopen_main.common.errors import PatchConflict from caliopen_storage.exception import NotFound log = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('-f', dest='conffile') parser.add_argument('-t', dest='test', action='store_true', default=False) args = parser.parse_args() Configuration.load(args.conffile, 'global') connect_storage() from caliopen_main.user.core.user import LocalIdentity from caliopen_main.message.store.message import Message from caliopen_main.message.objects.message import Message as ObjMessage identities = [x.identifier for x in LocalIdentity._model_class.all()] cpt = {'received': 0, 'sent': 0} for message in Message.all(): if message.is_received is None: from_participant = [x for x in message.participants if x['type'] == 'From'] if not from_participant: log.error('No from participant found for message {} : {}'. format(message.message_id, message.participants)) else:
def main(args=sys.argv): parser = argparse.ArgumentParser() parser.add_argument('-f', dest='conffile', default='development.ini') subparsers = parser.add_subparsers(title="action") sp_import = subparsers.add_parser('import', help='import existing mailbox') sp_import.set_defaults(func=import_email) sp_import.add_argument('-f', dest='format', choices=['mbox', 'maildir'], default='mbox') sp_import.add_argument('-p', dest='import_path') sp_import.add_argument('-e', dest='email') sp_import.add_argument('--contact-probability', dest='contact_probability', default=1.0) sp_import.add_argument('-t', dest='to') sp_import_vcard = subparsers.add_parser('import_vcard', help='import vcard') sp_import_vcard.set_defaults(func=import_vcard) sp_import_vcard.add_argument('-u', dest='username', help='username') sp_import_vcard.add_argument('-d', dest='directory', help='directory') sp_import_vcard.add_argument('-f', dest='file_vcard', help='file') sp_setup = subparsers.add_parser('setup', help='initialize the storage engine') sp_setup.set_defaults(func=setup) sp_create_user = subparsers.add_parser('create_user', help='Create a new user') sp_create_user.set_defaults(func=create_user) sp_create_user.add_argument('-e', dest='email', help='user email') sp_create_user.add_argument('-p', dest='password', help='password') sp_create_user.add_argument('-g', dest='given_name', help='user given name') sp_create_user.add_argument('-f', dest='family_name', help='user family name') sp_shell = subparsers.add_parser('shell') sp_shell.set_defaults(func=shell) sp_dump = subparsers.add_parser('dump') sp_dump.set_defaults(func=dump_model) sp_dump.add_argument('-m', dest='model', help='model to dump') sp_dump.add_argument('-o', dest='output_path', help='output path') sp_copy = subparsers.add_parser('copy') sp_copy.set_defaults(func=copy_model) sp_copy.add_argument('-m', dest='model', help='model to dump') sp_copy.add_argument('--where', dest='where', help='where condition') sp_copy.add_argument('--fetch-size', dest='fetch_size', default=100) sp_dump_index = subparsers.add_parser('dump_index') sp_dump_index.set_defaults(func=dump_indexes) sp_dump_index.add_argument('-o', dest='output_path', help='output path') sp_migrate_index = subparsers.add_parser('migrate_index') sp_migrate_index.set_defaults(func=migrate_index) sp_migrate_index.add_argument('-s', dest='input_script', help='python script to execute on index') sp_inject = subparsers.add_parser('inject') sp_inject.set_defaults(func=inject_email) sp_inject.add_argument('-e', dest='email') sp_inject.add_argument('-r', dest='recipient') sp_compute = subparsers.add_parser('compute', help='Launch basic compute') sp_compute.set_defaults(func=basic_compute) sp_compute.add_argument('-u', dest='username', help='username') sp_compute.add_argument('-j', dest='job', help='job name') sp_reserved = subparsers.add_parser('reserved_names', help='Import reserved names list') sp_reserved.set_defaults(func=import_reserved_names) sp_reserved.add_argument('-i', dest='input_file', help='csv file') sp_resync = subparsers.add_parser('resync_index', help='Resync index for an user') sp_resync.set_defaults(func=resync_index) sp_resync.add_argument('-u', dest='user_name', help='User name') sp_resync.add_argument('-i', dest='user_id', help='User uuid') sp_resync.add_argument('--delete', dest='delete', action='store_true') sp_resync = subparsers.add_parser('resync_shard', help='Resync shard index') sp_resync.set_defaults(func=resync_shard_index) sp_resync.add_argument('-s', dest='shard_id', help='Shard id') sp_resync.add_argument('-o', dest='old_shard_id', help='Old shard id') kwargs = parser.parse_args(args[1:]) kwargs = vars(kwargs) config_uri = kwargs.pop('conffile') func = kwargs.pop('func') Configuration.load(config_uri, 'global') connect_storage() func(**kwargs)
"""Test importance level compute.""" import unittest import os from caliopen_storage.config import Configuration if 'CALIOPEN_BASEDIR' in os.environ: conf_file = '{}/src/backend/configs/caliopen.yaml.template'. \ format(os.environ['CALIOPEN_BASEDIR']) else: conf_file = '../../../../../configs/caliopen.yaml.template' Configuration.load(conf_file, 'global') from caliopen_pi.features.helpers.importance_level import compute_importance class MockPI(object): def __init__(self, technic, context, comportment): self.technic = technic self.context = context self.comportment = comportment class MockMessage(object): def __init__(self, pi, tags=None, refs=None): self.pi = pi self.tags = tags if tags else []