def setup_storage(settings=None): """Create cassandra models.""" from caliopen_storage.core import core_registry # Make discovery happen from caliopen_main.user.core import User from caliopen_main.user.core import (UserIdentity, IdentityLookup, IdentityTypeLookup) from caliopen_main.contact.objects.contact import Contact from caliopen_main.message.objects.message import Message from caliopen_main.common.objects.tag import ResourceTag from caliopen_main.device.core import Device from caliopen_main.notification.core import Notification, NotificationTtl from cassandra.cqlengine.management import sync_table, \ create_keyspace_simple keyspace = Configuration('global').get('cassandra.keyspace') if not keyspace: raise Exception('Configuration missing for cassandra keyspace') # XXX tofix : define strategy and replication_factor in configuration create_keyspace_simple(keyspace, 1) for name, kls in core_registry.items(): log.info('Creating cassandra model %s' % name) if hasattr(kls._model_class, 'pk'): # XXX find a better way to detect model from udt sync_table(kls._model_class)
def setup_shard_index(shard): """Setup a shard index.""" url = Configuration('global').get('elasticsearch.url') client = Elasticsearch(url) try: log.info('Creating index {0}'.format(shard)) client.indices.create( index=shard, body={ "settings": { "analysis": { "analyzer": { "text_analyzer": { "type": "custom", "tokenizer": "lowercase", "filter": [ "ascii_folding" ] }, "email_analyzer": { "type": "custom", "tokenizer": "email_tokenizer", "filter": [ "ascii_folding" ] } }, "filter": { "ascii_folding": { "type": "asciifolding", "preserve_original": True } }, "tokenizer": { "email_tokenizer": { "type": "ngram", "min_gram": 3, "max_gram": 25 } } } } }) except Exception as exc: log.warn("failed to create index {} : {}".format(shard, exc)) return # PUT mappings for each type, if any for name, kls in core_registry.items(): if hasattr(kls, "_index_class") and \ hasattr(kls._model_class, 'user_id'): idx_kls = kls._index_class() if hasattr(idx_kls, "build_mapping"): log.debug('Init index mapping for {}'.format(idx_kls)) idx_kls.create_mapping(shard)
def setup_shard_index(shard): """Setup a shard index.""" url = Configuration('global').get('elasticsearch.url') client = Elasticsearch(url) try: log.info('Creating index {0}'.format(shard)) client.indices.create( index=shard, body={ "settings": { "analysis": { "analyzer": { "text_analyzer": { "type": "custom", "tokenizer": "lowercase", "filter": [ "ascii_folding" ] }, "email_analyzer": { "type": "custom", "tokenizer": "email_tokenizer", "filter": [ "ascii_folding" ] } }, "filter": { "ascii_folding": { "type": "asciifolding", "preserve_original": True } }, "tokenizer": { "email_tokenizer": { "type": "ngram", "min_gram": 3, "max_gram": 25 } } } } }) except Exception as exc: log.warn("failed to create index {} : {}".format(shard, exc)) return # PUT mappings for each type, if any for name, kls in core_registry.items(): if getattr(kls, '_index_class', None) and \ hasattr(kls._model_class, 'user_id'): idx_kls = kls._index_class() if hasattr(idx_kls, "build_mapping"): log.debug('Init index mapping for {}'.format(idx_kls)) idx_kls.create_mapping(shard)
def _setup_user_index(self): """Create user index and setup mappings.""" url = Configuration('global').get('elasticsearch.url') client = Elasticsearch(url) log.debug('Creating index for user {}'.format(self.user_id)) if not client.indices.exists(self.user_id): client.indices.create(self.user_id) else: log.warn('Index already exist {}'.format(self.user_id)) for name, kls in core_registry.items(): if hasattr(kls, "_index_class") and \ hasattr(kls._model_class, 'user_id'): idx_kls = kls._index_class() log.debug('Init index for {}'.format(idx_kls)) if hasattr(idx_kls, 'create_mapping'): log.info('Create index {} mapping for doc_type {}'.format( self.user_id, idx_kls.doc_type)) idx_kls.create_mapping(self.user_id)