def setup_storage(settings=None):
"""Create cassandra models."""
from caliopen_storage.core import core_registry
# Make discovery happen
from caliopen_main.user.core import User
from caliopen_main.user.core import (UserIdentity, IdentityLookup,
IdentityTypeLookup)
from caliopen_main.contact.objects.contact import Contact
from caliopen_main.message.objects.message import Message
from caliopen_main.common.objects.tag import ResourceTag
from caliopen_main.device.core import Device
from caliopen_main.notification.core import Notification, NotificationTtl
from cassandra.cqlengine.management import sync_table, \
create_keyspace_simple
keyspace = Configuration('global').get('cassandra.keyspace')
if not keyspace:
raise Exception('Configuration missing for cassandra keyspace')
# XXX tofix : define strategy and replication_factor in configuration
create_keyspace_simple(keyspace, 1)
for name, kls in core_registry.items():
log.info('Creating cassandra model %s' % name)
if hasattr(kls._model_class, 'pk'):
# XXX find a better way to detect model from udt
sync_table(kls._model_class)
def setup_shard_index(shard):
"""Setup a shard index."""
url = Configuration('global').get('elasticsearch.url')
client = Elasticsearch(url)
try:
log.info('Creating index {0}'.format(shard))
client.indices.create(
index=shard,
body={
"settings": {
"analysis": {
"analyzer": {
"text_analyzer": {
"type": "custom",
"tokenizer": "lowercase",
"filter": [
"ascii_folding"
]
},
"email_analyzer": {
"type": "custom",
"tokenizer": "email_tokenizer",
"filter": [
"ascii_folding"
]
}
},
"filter": {
"ascii_folding": {
"type": "asciifolding",
"preserve_original": True
}
},
"tokenizer": {
"email_tokenizer": {
"type": "ngram",
"min_gram": 3,
"max_gram": 25
}
}
}
}
})
except Exception as exc:
log.warn("failed to create index {} : {}".format(shard, exc))
return
# PUT mappings for each type, if any
for name, kls in core_registry.items():
if hasattr(kls, "_index_class") and \
hasattr(kls._model_class, 'user_id'):
idx_kls = kls._index_class()
if hasattr(idx_kls, "build_mapping"):
log.debug('Init index mapping for {}'.format(idx_kls))
idx_kls.create_mapping(shard)
def setup_shard_index(shard):
"""Setup a shard index."""
url = Configuration('global').get('elasticsearch.url')
client = Elasticsearch(url)
try:
log.info('Creating index {0}'.format(shard))
client.indices.create(
index=shard,
body={
"settings": {
"analysis": {
"analyzer": {
"text_analyzer": {
"type": "custom",
"tokenizer": "lowercase",
"filter": [
"ascii_folding"
]
},
"email_analyzer": {
"type": "custom",
"tokenizer": "email_tokenizer",
"filter": [
"ascii_folding"
]
}
},
"filter": {
"ascii_folding": {
"type": "asciifolding",
"preserve_original": True
}
},
"tokenizer": {
"email_tokenizer": {
"type": "ngram",
"min_gram": 3,
"max_gram": 25
}
}
}
}
})
except Exception as exc:
log.warn("failed to create index {} : {}".format(shard, exc))
return
# PUT mappings for each type, if any
for name, kls in core_registry.items():
if getattr(kls, '_index_class', None) and \
hasattr(kls._model_class, 'user_id'):
idx_kls = kls._index_class()
if hasattr(idx_kls, "build_mapping"):
log.debug('Init index mapping for {}'.format(idx_kls))
idx_kls.create_mapping(shard)
def _setup_user_index(self):
"""Create user index and setup mappings."""
url = Configuration('global').get('elasticsearch.url')
client = Elasticsearch(url)
log.debug('Creating index for user {}'.format(self.user_id))
if not client.indices.exists(self.user_id):
client.indices.create(self.user_id)
else:
log.warn('Index already exist {}'.format(self.user_id))
for name, kls in core_registry.items():
if hasattr(kls, "_index_class") and \
hasattr(kls._model_class, 'user_id'):
idx_kls = kls._index_class()
log.debug('Init index for {}'.format(idx_kls))
if hasattr(idx_kls, 'create_mapping'):
log.info('Create index {} mapping for doc_type {}'.format(
self.user_id, idx_kls.doc_type))
idx_kls.create_mapping(self.user_id)