def recreate_index(self):
self.log.info('Deleting index')
ES.delete_index()
self.log.info('Creating index')
create_index_with_settings(self.settings)
self.log.info('Creating mappings')
ES.setup_mappings()
def index_models(self, model_names):
self.log.info('Indexing models documents')
params = self.options.params or ''
params = dict([[k, v[0]] for k, v in urllib.parse.parse_qs(params).items()])
for model_name in model_names:
self.log.info('Processing model `{}`'.format(model_name))
model = engine.get_document_cls(model_name)
local_params = dict()
local_params.update(params)
if '_limit' not in local_params:
limit = model.get_collection().count()
local_params['_limit'] = limit
chunk_size = int(self.options.chunk or local_params['_limit'])
es = ES(source=model_name, index_name=self.options.index,
chunk_size=chunk_size)
query_set = model.get_collection(**local_params)
documents = to_indexable_dicts(query_set)
self.log.info('Indexing missing `{}` documents'.format(
model_name))
es.index_missing_documents(documents)
def on_after_delete(mapper, connection, target):
from nefertari.elasticsearch import ES
model_cls = target.__class__
es = ES(model_cls.__name__)
obj_id = getattr(target, model_cls.pk_field())
es.delete(obj_id)
es.index_refs(target)
def includeme(config):
Settings = dictset(config.registry.settings)
config.include("nefertari.engine")
config.include("nefertari")
config.include("nefertari.view")
config.include("nefertari.elasticsearch")
# Process nefertari settings
if Settings.asbool("debug"):
log.warning("*** DEBUG DEBUG DEBUG mode ***")
config.add_tween("nefertari.tweens.get_tunneling")
if Settings.asbool("cors.enable"):
config.add_tween("nefertari.tweens.cors")
if Settings.asbool("ssl_middleware.enable"):
config.add_tween("nefertari.tweens.ssl")
if Settings.asbool("request_timing.enable"):
config.add_tween("nefertari.tweens.request_timing")
# Set root factory
config.root_factory = NefertariRootACL
# Process auth settings
root = config.get_root_resource()
ramses_auth = Settings.asbool("ramses.auth", False)
root.auth = ramses_auth
log.info("Parsing RAML")
parsed_raml = pyraml.parser.load(Settings["ramses.raml_schema"])
log.info("Starting models generation")
generate_models(config, raml_resources=parsed_raml.resources)
if ramses_auth:
if getattr(config.registry, "auth_model", None) is None:
from nefertari.authentication.models import AuthUser
config.registry.auth_model = AuthUser
from .auth import setup_auth_policies
setup_auth_policies(config, parsed_raml)
log.info("Starting server generation")
generate_server(parsed_raml, config)
log.info("Running nefertari.engine.setup_database")
from nefertari.engine import setup_database
setup_database(config)
from nefertari.elasticsearch import ES
ES.setup_mappings()
if ramses_auth:
config.include("ramses.auth")
log.info("Server succesfully generated\n")
def reindextask(model, boxsize=5000):
"""Index model by small chunks (ie: a box, with a reasonable size)
"""
global log
mcls = engine.get_document_cls(model)
# proceed by chunks of 'boxsize'
count = mcls.get_collection(_count=True)
if count < 1: # Hu ? nothing in DB
return
# Let us be aware of some numbers
boxes = count // boxsize
rest = count % boxsize
es = ES(source=model) # humm quick & dirty: get a connector
log.info('Processing model `{}` with {} documents in {} boxes'.format(
model, count, boxes))
# dump by 'boxes' ; add one for the rest (NB: if rest=0 the last box will be
# empty anyway )
for n in range(boxes + 1):
log.info('Indexing missing `{}` documents (box: {}/{})'.format(
model, n, boxes + 1))
query_set = mcls.get_collection(
_limit=boxsize, _page=n,
_sort=mcls.pk_field()) ## don't forget the sort
documents = to_dicts(query_set)
log.debug('---> from db {} documents ; send to ES'.format(
len(documents)))
## TODO: add a control ? The last box size should be equal to 'rest'
es.index(documents)
def getitem_es(self, key):
es = ES(self.item_model.__name__)
obj = es.get_item(id=key)
obj.__acl__ = self.item_acl(obj)
obj.__parent__ = self
obj.__name__ = key
return obj
def getitem_es(self, key):
es = ES(self.item_model.__name__)
obj = es.get_item(id=key)
obj.__acl__ = self.item_acl(obj)
obj.__parent__ = self
obj.__name__ = key
return obj
def on_post_bulk_insert(sender,documents,**kw):
if not documents:
return
from nefertari.elasticsearch import ES
es = ES(source=documents[0].__class__.__name__)
docs = to_dicts(documents)
es.index(docs)
def on_after_delete(mapper, connection, target):
from nefertari.elasticsearch import ES
request = getattr(target, '_request', None)
model_cls = target.__class__
es = ES(model_cls.__name__)
obj_id = getattr(target, model_cls.pk_field())
es.delete(obj_id, request=request)
es.index_relations(target, request=request)
def includeme(config):
Settings = dictset(config.registry.settings)
config.include('nefertari.engine')
config.include('nefertari')
config.include('nefertari.view')
# Process nefertari settings
if Settings.asbool('debug'):
log.warning('*** DEBUG DEBUG DEBUG mode ***')
config.add_tween('nefertari.tweens.get_tunneling')
if Settings.asbool('cors.enable'):
config.add_tween('nefertari.tweens.cors')
if Settings.asbool('ssl_middleware.enable'):
config.add_tween('nefertari.tweens.ssl')
if Settings.asbool('request_timing.enable'):
config.add_tween('nefertari.tweens.request_timing')
# Set root factory
config.root_factory = NefertariRootACL
# Process auth settings
root = config.get_root_resource()
ramses_auth = Settings.asbool('ramses.auth', False)
root.auth = ramses_auth
log.info('Parsing RAML')
parsed_raml = pyraml.parser.load(Settings['ramses.raml_schema'])
log.info('Starting models generation')
generate_models(config, raml_resources=parsed_raml.resources)
if ramses_auth:
if getattr(config.registry, 'auth_model', None) is None:
from nefertari.authentication.models import get_authuser_model
config.registry.auth_model = get_authuser_model()
from .auth import setup_auth_policies
setup_auth_policies(config, parsed_raml)
config.include('nefertari.elasticsearch')
log.info('Starting server generation')
generate_server(parsed_raml, config)
log.info('Running nefertari.engine.setup_database')
from nefertari.engine import setup_database
setup_database(config)
from nefertari.elasticsearch import ES
ES.setup_mappings()
if ramses_auth:
config.include('ramses.auth')
log.info('Server succesfully generated\n')
def recreate_mapping(model):
""" Delete and Put the mapping of a model.
Effect: delete all document linked to this mapping in the working index.
"""
global log
mcls = engine.get_document_cls(model)
es = ES(model)
# delete: work with elasticsearch=1.7.0
es.api.indices.delete_mapping(es.index_name, doc_type=model)
# put good old mapping.
es.put_mapping(body=mcls.get_es_mapping())
def run(self):
ES.setup(self.settings)
if self.options.recreate:
self.recreate_index()
models = engine.get_document_classes()
model_names = [
name for name, model in models.items()
if getattr(model, '_index_enabled', False)]
else:
model_names = split_strip(self.options.models)
self.index_models(model_names)
def run(self):
ES.setup(self.settings)
if self.options.recreate:
self.recreate_index()
models = engine.get_document_classes()
model_names = [
name for name, model in models.items()
if getattr(model, '_index_enabled', False)
]
else:
model_names = split_strip(self.options.models)
self.index_models(model_names)
def on_bulk_delete(model_cls, objects, request):
if not getattr(model_cls, '_index_enabled', False):
return
pk_field = model_cls.pk_field()
ids = [getattr(obj, pk_field) for obj in objects]
from nefertari.elasticsearch import ES
es = ES(source=model_cls.__name__)
es.delete(ids, request=request)
# Reindex relationships
es.bulk_index_relations(objects, request=request)
def on_bulk_update(model_cls, objects, request):
if not getattr(model_cls, '_index_enabled', False):
return
if not objects:
return
from nefertari.elasticsearch import ES
es = ES(source=model_cls.__name__)
documents = to_dicts(objects)
es.index(documents, request=request)
# Reindex relationships
es.bulk_index_relations(objects, request=request, nested_only=True)
def run(self, quiet=False):
from nefertari.elasticsearch import ES
ES.setup(self.settings)
models_paths = split_strip(self.options.models)
for path in models_paths:
model = resolve(path)
model_name = path.split('.')[-1]
params = self.options.params or ''
params = dict([[k, v[0]]
for k, v in urlparse.parse_qs(params).items()])
params.setdefault('_limit', params.get('_limit', 10000))
chunk_size = self.options.chunk or params['_limit']
es = ES(source=model_name, index_name=self.options.index)
query_set = model.get_collection(**params)
documents = to_dicts(query_set)
if self.options.force:
es.index(documents, chunk_size=chunk_size)
else:
es.index_missing(documents, chunk_size=chunk_size)
return 0
def getitem_es(self, key):
""" Get item with ID of :key: from elasticsearch """
from nefertari.elasticsearch import ES
es = ES(self.__context_class__.__name__)
pk_field = self.__context_class__.pk_field()
kwargs = {
pk_field: key,
'_limit': 1,
'__raise_on_empty': True,
}
obj = es.get_collection(**kwargs)[0]
obj.__acl__ = self.context_acl(obj)
obj.__parent__ = self
obj.__name__ = key
return obj
def on_after_update(mapper, connection, target):
from .documents import BaseDocument
# Reindex old one-to-one related object
committed_state = attributes.instance_state(target).committed_state
columns = set()
for field, value in committed_state.items():
if isinstance(value, BaseDocument):
obj_session = object_session(value)
# Make sure object is not updated yet
if not obj_session.is_modified(value):
obj_session.expire(value)
index_object(value, with_refs=False)
else:
id_pos = field.rfind('_id')
if id_pos >= 0:
rel_name = field[:id_pos]
rel = mapper.relationships.get(rel_name, False)
if rel and any(c.name == field for c in rel.local_columns):
columns.add(rel_name)
# Reload `target` to get access to processed fields values
columns = columns.union(
[c.name for c in class_mapper(target.__class__).columns])
object_session(target).expire(target, attribute_names=columns)
index_object(target, with_refs=False, nested_only=True)
# Reindex the item's parents. This must be done after the child has been processes
for parent, children_field in target.get_parent_documents(
nested_only=True):
columns = [c.name for c in class_mapper(parent.__class__).columns]
object_session(parent).expire(parent, attribute_names=columns)
ES(parent.__class__.__name__).index_nested_document(
parent, children_field, target)
def find_by_ace(ace, models, count=False):
""" Find documents of models that include ace.
Look into ACLEncoderMixin.stringify_acl for details on ace format.
:param ace: Stringified ACL entry (ACE) to match agains.
:param models: List of document classes objects of which should
be found.
:param count: Boolean. When True objects count is returned.
:returns: Number of matching documents when count=True or documents
otherwise.
:raises ValueError: If no es-based models passed.
"""
es_types = _get_es_types(models)
if not es_types:
raise ValueError('No es-based models passed')
params = {'body': _get_es_body(ace)}
if count:
params['_count'] = True
documents = ES(es_types).get_collection(**params)
docs_count = (documents if isinstance(documents, int) else len(documents))
log.info('Found {} documents that match ACE {}.'.format(
docs_count, str(ace)))
return documents
def get_es_mapping(cls, _depth=None, types_map=None):
""" Generate ES mapping from model schema. """
from nefertari.elasticsearch import ES
if types_map is None:
types_map = TYPES_MAP
if _depth is None:
_depth = cls._nesting_depth
depth_reached = _depth <= 0
nested_substitutions = []
properties = {}
mapping = {
ES.src2type(cls.__name__): {
'properties': properties
}
}
mapper = class_mapper(cls)
columns = {c.name: c for c in mapper.columns}
relationships = {r.key: r for r in mapper.relationships}
for name, column in columns.items():
column_type = column.type
if isinstance(column_type, types.ChoiceArray):
column_type = column_type.impl.item_type
column_type = type(column_type)
if column_type not in types_map:
continue
properties[name] = types_map[column_type]
if hasattr(column, "_es_multi_field") and getattr(column, "_es_multi_field"):
multi_fields = getattr(column, "_es_multi_field")
properties[name] = properties[name].copy()
properties[name]["fields"] = {}
for multi_field_name in multi_fields:
properties[name]["fields"][multi_field_name] = multi_fields[multi_field_name].copy()
properties[name]["fields"][multi_field_name].update(types_map[column_type])
for name, column in relationships.items():
if name in cls._nested_relationships and not depth_reached:
column_type = {'type': 'nested', 'include_in_parent': True}
nested_substitutions.append(name)
submapping, sub_substitutions = column.mapper.class_.get_es_mapping(
_depth=_depth - 1)
column_type.update(list(submapping.values())[0])
properties[name + "_nested"] = column_type
rel_pk_field = column.mapper.class_.pk_field_type()
column_type = types_map[rel_pk_field]
properties[name] = column_type
properties['_pk'] = {'type': 'string'}
return mapping, nested_substitutions
def on_bulk_update(update_context):
request = getattr(
update_context.query, '_request', None)
model_cls = update_context.mapper.entity
if not getattr(model_cls, '_index_enabled', False):
return
objects = update_context.query.all()
if not objects:
return
from nefertari.elasticsearch import ES
es = ES(source=model_cls.__name__)
es.index(objects, request=request)
# Reindex relationships
es.bulk_index_relations(objects, request=request, nested_only=True)
def main(argv=sys.argv):
global log
args = parse_args()
options = parse_vars(args.options)
settings = get_appsettings(args.ini, name="hathor", options=options)
setup_logging(args.ini)
log = logging.getLogger()
## Init underlaying FW -- Code inspire from nefertari
# see: https://github.com/ramses-tech/nefertari/blob/master/nefertari/scripts/es.py
mappings_setup = getattr(ES, '_mappings_setup', False)
try:
ES._mappings_setup = True
env = bootstrap("%s#hathor" % args.ini)
## Not sure if I need this, bootstrap should take care of this
config = Configurator(settings=settings)
config.include('nefertari.engine')
finally:
ES._mappings_setup = mappings_setup
registry = env['registry']
ES.setup(dictset(registry.settings))
# Recreate: drop index, get all available model names -- ignore -m from
# arguments list
if args.recreate and not args.list:
recreate_index()
args.models = available_models()
if args.list:
print("Available models:\n {}".format(", ".join(available_models())))
elif args.models:
model_names = args.models
if model_names:
av_models = available_models()
for elem in model_names:
if not elem in av_models:
raise (ValueError(
"model '{}' not available."
"Use '-l' to list available models.".format(elem)))
## still here we gonna re-index things
for model in model_names:
if args.delete_mapping:
recreate_mapping(model)
reindextask(model, boxsize=args.boxsize)
def run(self, quiet=False):
from nefertari.elasticsearch import ES
ES.setup(self.settings)
models_paths = split_strip(self.options.models)
for path in models_paths:
model = resolve(path)
model_name = path.split('.')[-1]
params = self.options.params or ''
params = dict([
[k, v[0]] for k, v in urlparse.parse_qs(params).items()
])
params.setdefault('_limit', params.get('_limit', 10000))
chunk_size = self.options.chunk or params['_limit']
es = ES(source=model_name, index_name=self.options.index)
query_set = model.get_collection(**params)
documents = to_dicts(query_set)
if self.options.force:
es.index(documents, chunk_size=chunk_size)
else:
es.index_missing(documents, chunk_size=chunk_size)
return 0
def get_collection_es(self):
""" Query ES collection and return results.
This is default implementation of querying ES collection with
`self._query_params`. It must return found ES collection
results for default response renderers to work properly.
"""
from nefertari.elasticsearch import ES
return ES(self.Model.__name__).get_collection(**self._query_params)
def index_models(self, model_names):
self.log.info('Indexing models documents')
params = self.options.params or ''
params = dict([[k, v[0]]
for k, v in urllib.parse.parse_qs(params).items()])
params.setdefault('_limit', params.get('_limit', 10000))
chunk_size = self.options.chunk or params['_limit']
for model_name in model_names:
self.log.info('Processing model `{}`'.format(model_name))
model = engine.get_document_cls(model_name)
es = ES(source=model_name,
index_name=self.options.index,
chunk_size=chunk_size)
query_set = model.get_collection(**params)
documents = to_dicts(query_set)
self.log.info('Indexing missing `{}` documents'.format(model_name))
es.index_missing_documents(documents)
def _get_es_types(models):
""" Get ES types from document model classes.
:param models: List of document classes.
:returns: String with ES type names joing by comma.
"""
type_names = [t.__name__ for t in models
if getattr(t, '_index_enabled', False)]
es_types = [ES.src2type(name) for name in type_names]
return ','.join(es_types)
def index_models(self, model_names):
self.log.info('Indexing models documents')
params = self.options.params or ''
params = dict([
[k, v[0]] for k, v in urllib.parse.parse_qs(params).items()
])
params.setdefault('_limit', params.get('_limit', 10000))
chunk_size = self.options.chunk or params['_limit']
for model_name in model_names:
self.log.info('Processing model `{}`'.format(model_name))
model = engine.get_document_cls(model_name)
es = ES(source=model_name, index_name=self.options.index,
chunk_size=chunk_size)
query_set = model.get_collection(**params)
documents = to_dicts(query_set)
self.log.info('Indexing missing `{}` documents'.format(
model_name))
es.index_missing_documents(documents)
def on_post_save(sender, document, **kw):
""" Add new document to index or update existing. """
from nefertari.elasticsearch import ES
common_kw = {'request': getattr(document, '_request', None)}
created = kw.get('created', False)
if created:
es = ES(document.__class__.__name__)
es.index(document.to_dict(), **common_kw)
elif not created and document._get_changed_fields():
es = ES(document.__class__.__name__)
es.index(document.to_dict(), **common_kw)
es.index_relations(document, nested_only=True, **common_kw)
def _get_es_types(models):
""" Get ES types from document model classes.
:param models: List of document classes.
:returns: String with ES type names joing by comma.
"""
type_names = [
t.__name__ for t in models if getattr(t, '_index_enabled', False)
]
es_types = [ES.src2type(name) for name in type_names]
return ','.join(es_types)
def aggregate(self):
""" Perform aggregation and return response. """
from nefertari.elasticsearch import ES
aggregations_params = self.pop_aggregations_params()
if self.view._auth_enabled:
self.check_aggregations_privacy(aggregations_params)
self.stub_wrappers()
return ES(self.view.Model.__name__).aggregate(
_aggregations_params=aggregations_params,
**self._query_params)
def recreate_index(self):
self.log.info('Deleting index')
ES.delete_index()
self.log.info('Creating index')
ES.create_index()
self.log.info('Creating mappings')
ES.setup_mappings()
def get_collection_es(self, **kwargs):
""" Get ES objects collection taking into account the generated
queryset of parent view.
This method allows working with nested resources properly. Thus a
queryset returned by this method will be a subset of its parent view's
queryset, thus filtering out objects that don't belong to the parent
object.
"""
from nefertari.elasticsearch import ES
es = ES(self.Model.__name__)
objects_ids = self._parent_queryset_es()
if objects_ids is not None:
objects_ids = self.get_es_object_ids(objects_ids)
if not objects_ids:
return []
self._query_params['id'] = objects_ids
return es.get_collection(
_raw_terms=self._get_raw_terms(),
**self._query_params)
def on_after_delete(mapper, connection, target):
from nefertari.elasticsearch import ES
request = getattr(target, '_request', None)
model_cls = target.__class__
es = ES(model_cls.__name__)
obj_id = getattr(target, model_cls.pk_field())
es.delete(obj_id, request=request)
es.index_relations(target, request=request)
def on_after_delete(mapper, connection, target):
from nefertari.elasticsearch import ES
model_cls = target.__class__
es = ES(model_cls.__name__)
obj_id = getattr(target, model_cls.pk_field())
es.delete(obj_id)
target.expire_parents()
es.index_relations(target)
def recreate_index():
global log
log.info('Deleting index')
ES.delete_index()
log.info('Creating index')
ES.create_index()
log.info('Creating mappings')
ES.setup_mappings()
def _set_object_self(self, obj):
""" Add '_self' key value to :obj: dict. """
from nefertari.elasticsearch import ES
location = self.request.path_url
try:
type_, obj_pk = obj['_type'], obj['_pk']
except KeyError:
return
resource = (self.model_collections.get(type_) or
self.model_collections.get(ES.src2type(type_)))
if resource is not None:
location = self.request.route_url(
resource.uid, **{resource.id_name: obj_pk})
obj.setdefault('_self', location)
def determine_types(self):
""" Determine ES type names from request data.
In particular `request.matchdict['collections']` is used to
determine types names. Its value is comma-separated sequence
of collection names under which views have been registered.
"""
from nefertari.elasticsearch import ES
collections = self.get_collections()
resources = self.get_resources(collections)
models = set([res.view.Model for res in resources])
es_models = [mdl for mdl in models if mdl
and getattr(mdl, '_index_enabled', False)]
types = [ES.src2type(mdl.__name__) for mdl in es_models]
return types
def determine_types(self):
""" Determine ES type names from request data.
In particular `request.matchdict['collections']` is used to
determine types names. Its value is comma-separated sequence
of collection names under which views have been registered.
"""
from nefertari.elasticsearch import ES
collections = self.get_collections()
resources = self.get_resources(collections)
models = set([res.view.Model for res in resources])
es_models = [
mdl for mdl in models
if mdl and getattr(mdl, '_index_enabled', False)
]
types = [ES.src2type(mdl.__name__) for mdl in es_models]
return types
def _set_object_self(self, obj):
""" Add '_self' key value to :obj: dict. """
from nefertari.elasticsearch import ES
location = self.request.path_url
route_kwargs = {}
""" Check for parents """
if self.request.matchdict:
route_kwargs.update(self.request.matchdict)
try:
type_, obj_pk = obj['_type'], obj['_pk']
except KeyError:
return
resource = (self.model_collections.get(type_)
or self.model_collections.get(ES.src2type(type_)))
if resource is not None:
route_kwargs.update({resource.id_name: obj_pk})
location = self.request.route_url(resource.uid, **route_kwargs)
obj.setdefault('_self', location)
def on_bulk_delete(model_cls, objects, request):
if not getattr(model_cls, '_index_enabled', False):
return
pk_field = model_cls.pk_field()
ids = [getattr(obj, pk_field) for obj in objects]
from nefertari.elasticsearch import ES
es = ES(source=model_cls.__name__)
es.delete(ids, request=request)
# Reindex relationships
es.bulk_index_relations(objects, request=request)
def get_es_mapping(cls, _depth=None, types_map=None):
""" Generate ES mapping from model schema. """
from nefertari.elasticsearch import ES
if types_map is None:
types_map = TYPES_MAP
if _depth is None:
_depth = cls._nesting_depth
depth_reached = _depth <= 0
properties = {}
mapping = {
ES.src2type(cls.__name__): {
'properties': properties
}
}
fields = cls._fields.copy()
for name, field in fields.items():
if isinstance(field, RelationshipField):
field = field.field
if isinstance(field, (ReferenceField, RelationshipField)):
if name in cls._nested_relationships and not depth_reached:
field_mapping = {'type': 'nested'}
submapping = field.document_type.get_es_mapping(
_depth=_depth-1)
field_mapping.update(list(submapping.values())[0])
else:
field_mapping = types_map[
field.document_type.pk_field_type()]
properties[name] = field_mapping
continue
if isinstance(field, ChoiceField):
field = field._real_field
field_type = type(field)
if field_type is ListField:
field_type = field.item_type
if field_type not in types_map:
continue
properties[name] = types_map[field_type]
properties['_pk'] = {'type': 'string'}
return mapping
def _set_object_self(self, obj):
""" Add '_self' key value to :obj: dict. """
from nefertari.elasticsearch import ES
location = self.request.path_url
route_kwargs = {}
""" Check for parents """
if self.request.matchdict:
route_kwargs.update(self.request.matchdict)
try:
type_, obj_pk = obj['_type'], obj['_pk']
except KeyError:
return
resource = (self.model_collections.get(type_) or
self.model_collections.get(ES.src2type(type_)))
if resource is not None:
route_kwargs.update({resource.id_name: obj_pk})
location = self.request.route_url(
resource.uid, **route_kwargs)
obj.setdefault('_self', location)
def on_bulk_update(update_context):
model_cls = update_context.mapper.entity
if not getattr(model_cls, '_index_enabled', False):
return
objects = update_context.query.all()
if not objects:
return
from nefertari.elasticsearch import ES
es = ES(source=model_cls.__name__)
es.index(objects)
# Reindex relationships
es.bulk_index_relations(objects, nested_only=True)
def get_es_mapping(cls, _depth=None, types_map=None):
""" Generate ES mapping from model schema. """
from nefertari.elasticsearch import ES
if types_map is None:
types_map = TYPES_MAP
if _depth is None:
_depth = cls._nesting_depth
depth_reached = _depth <= 0
properties = {}
mapping = {
ES.src2type(cls.__name__): {
'properties': properties
}
}
mapper = class_mapper(cls)
columns = {c.name: c for c in mapper.columns}
relationships = {r.key: r for r in mapper.relationships}
for name, column in columns.items():
column_type = column.type
if isinstance(column_type, types.ChoiceArray):
column_type = column_type.impl.item_type
column_type = type(column_type)
if column_type not in types_map:
continue
properties[name] = types_map[column_type]
for name, column in relationships.items():
if name in cls._nested_relationships and not depth_reached:
column_type = {'type': 'nested'}
submapping = column.mapper.class_.get_es_mapping(
_depth=_depth-1)
column_type.update(list(submapping.values())[0])
else:
rel_pk_field = column.mapper.class_.pk_field_type()
column_type = types_map[rel_pk_field]
properties[name] = column_type
properties['_pk'] = {'type': 'string'}
return mapping
def get_es_mapping(cls, _depth=None, types_map=None):
""" Generate ES mapping from model schema. """
from nefertari.elasticsearch import ES
if types_map is None:
types_map = TYPES_MAP
if _depth is None:
_depth = cls._nesting_depth
depth_reached = _depth <= 0
properties = {}
mapping = {ES.src2type(cls.__name__): {'properties': properties}}
mapper = class_mapper(cls)
columns = {c.name: c for c in mapper.columns}
relationships = {r.key: r for r in mapper.relationships}
for name, column in columns.items():
column_type = column.type
if isinstance(column_type, types.ChoiceArray):
column_type = column_type.impl.item_type
column_type = type(column_type)
if column_type not in types_map:
continue
properties[name] = types_map[column_type]
for name, column in relationships.items():
if name in cls._nested_relationships and not depth_reached:
column_type = {'type': 'nested'}
submapping = column.mapper.class_.get_es_mapping(
_depth=_depth - 1)
column_type.update(list(submapping.values())[0])
else:
rel_pk_field = column.mapper.class_.pk_field_type()
column_type = types_map[rel_pk_field]
properties[name] = column_type
properties['_pk'] = {'type': 'string'}
return mapping
def index_object(obj, with_refs=True, **kwargs):
es = ES(obj.__class__.__name__)
es.index(obj, **kwargs)
if with_refs:
es.index_relations(obj, **kwargs)
def index_object(obj, with_refs=True, **kwargs):
es = ES(obj.__class__.__name__)
es.index(obj, **kwargs)
if with_refs:
es.index_relations(obj, **kwargs)
def includeme(config):
from .generators import generate_server, generate_models
Settings = dictset(config.registry.settings)
config.include('nefertari.engine')
config.registry.database_acls = Settings.asbool('database_acls')
if config.registry.database_acls:
config.include('nefertari_guards')
config.include('nefertari')
config.include('nefertari.view')
config.include('nefertari.json_httpexceptions')
# Process nefertari settings
if Settings.asbool('enable_get_tunneling'):
config.add_tween('nefertari.tweens.get_tunneling')
if Settings.asbool('cors.enable'):
config.add_tween('nefertari.tweens.cors')
if Settings.asbool('ssl_middleware.enable'):
config.add_tween('nefertari.tweens.ssl')
if Settings.asbool('request_timing.enable'):
config.add_tween('nefertari.tweens.request_timing')
# Set root factory
config.root_factory = NefertariRootACL
# Process auth settings
root = config.get_root_resource()
root_auth = getattr(root, 'auth', False)
log.info('Parsing RAML')
raml_root = ramlfications.parse(Settings['ramses.raml_schema'])
log.info('Starting models generation')
generate_models(config, raml_resources=raml_root.resources)
if root_auth:
from .auth import setup_auth_policies, get_authuser_model
if getattr(config.registry, 'auth_model', None) is None:
config.registry.auth_model = get_authuser_model()
setup_auth_policies(config, raml_root)
config.include('nefertari.elasticsearch')
log.info('Starting server generation')
generate_server(raml_root, config)
log.info('Running nefertari.engine.setup_database')
from nefertari.engine import setup_database
setup_database(config)
from nefertari.elasticsearch import ES
ES.setup_mappings()
if root_auth:
config.include('ramses.auth')
log.info('Server succesfully generated\n')
def run(self):
ES.setup(self.settings)
model_names = split_strip(self.options.models)
for model_name in model_names:
self.log.info('Processing model `{}`'.format(model_name))
model = engine.get_document_cls(model_name)
params = self.options.params or ''
params = dict([
[k, v[0]] for k, v in urllib.parse.parse_qs(params).items()
])
params.setdefault('_limit', params.get('_limit', 10000))
chunk_size = self.options.chunk or params['_limit']
es = ES(source=model_name, index_name=self.options.index,
chunk_size=chunk_size)
query_set = model.get_collection(**params)
documents = to_dicts(query_set)
if self.options.force:
self.log.info('Recreating `{}` ES mapping'.format(model_name))
es.delete_mapping()
es.put_mapping(body=model.get_es_mapping())
self.log.info('Indexing all `{}` documents'.format(
model_name))
es.index(documents)
else:
self.log.info('Indexing missing `{}` documents'.format(
model_name))
es.index_missing_documents(documents)
return 0
def index_object(obj, with_refs=True, **kwargs):
from nefertari.elasticsearch import ES
es = ES(obj.__class__.__name__)
es.index(obj.to_dict(), **kwargs)
if with_refs:
es.index_relations(obj, **kwargs)
def run(self):
ES.setup(self.settings)
model_names = split_strip(self.options.models)
for model_name in model_names:
self.log.info('Processing model `{}`'.format(model_name))
model = engine.get_document_cls(model_name)
params = self.options.params or ''
params = dict([[k, v[0]]
for k, v in urllib.parse.parse_qs(params).items()])
params.setdefault('_limit', params.get('_limit', 10000))
chunk_size = self.options.chunk or params['_limit']
es = ES(source=model_name,
index_name=self.options.index,
chunk_size=chunk_size)
query_set = model.get_collection(**params)
documents = to_dicts(query_set)
if self.options.force:
self.log.info('Recreating `{}` ES mapping'.format(model_name))
es.delete_mapping()
es.put_mapping(body=model.get_es_mapping())
self.log.info('Indexing all `{}` documents'.format(model_name))
es.index(documents)
else:
self.log.info(
'Indexing missing `{}` documents'.format(model_name))
es.index_missing_documents(documents)
return 0
def on_pre_save(sender, document, **kw):
from nefertari.elasticsearch import ES
if not kw.get('created', False) and document._get_changed_fields():
ES(document.__class__.__name__).index(document.to_dict())
def on_delete(sender, document, **kw):
from nefertari.elasticsearch import ES
ES(document.__class__.__name__).delete(document.id)
def includeme(config):
from .generators import generate_server, generate_models
Settings = dictset(config.registry.settings)
config.include('nefertari.engine')
config.registry.database_acls = Settings.asbool('database_acls')
if config.registry.database_acls:
config.include('nefertari_guards')
config.include('nefertari')
config.include('nefertari.view')
config.include('nefertari.json_httpexceptions')
# Process nefertari settings
if Settings.asbool('enable_get_tunneling'):
config.add_tween('nefertari.tweens.get_tunneling')
if Settings.asbool('cors.enable'):
config.add_tween('nefertari.tweens.cors')
if Settings.asbool('ssl_middleware.enable'):
config.add_tween('nefertari.tweens.ssl')
if Settings.asbool('request_timing.enable'):
config.add_tween('nefertari.tweens.request_timing')
# Set root factory
config.root_factory = NefertariRootACL
# Process auth settings
root = config.get_root_resource()
root_auth = getattr(root, 'auth', False)
log.info('Parsing RAML')
raml_root = ramlfications.parse(Settings['ramses.raml_schema'])
log.info('Starting models generation')
generate_models(config, raml_resources=raml_root.resources)
if root_auth:
from .auth import setup_auth_policies, get_authuser_model
if getattr(config.registry, 'auth_model', None) is None:
config.registry.auth_model = get_authuser_model()
setup_auth_policies(config, raml_root)
config.include('nefertari.elasticsearch')
log.info('Starting server generation')
generate_server(raml_root, config)
log.info('Running nefertari.engine.setup_database')
from nefertari.engine import setup_database
setup_database(config)
from nefertari.elasticsearch import ES
ES.setup_mappings()
if root_auth:
config.include('ramses.auth')
log.info('Server succesfully generated\n')
def get_es_mapping(cls, _depth=None, types_map=None):
""" Generate ES mapping from model schema. """
from nefertari.elasticsearch import ES
if types_map is None:
types_map = TYPES_MAP
if _depth is None:
_depth = cls._nesting_depth
depth_reached = _depth <= 0
nested_substitutions = []
properties = {}
mapping = {ES.src2type(cls.__name__): {'properties': properties}}
mapper = class_mapper(cls)
columns = {c.name: c for c in mapper.columns}
relationships = {r.key: r for r in mapper.relationships}
for name, column in columns.items():
column_type = column.type
if isinstance(column_type, types.ChoiceArray):
column_type = column_type.impl.item_type
column_type = type(column_type)
if column_type not in types_map:
continue
if getattr(column, '_custom_analyzer', False):
properties[name] = {'analyzer': column._custom_analyzer}
properties[name].update(types_map[column_type])
continue
properties[name] = types_map[column_type]
if hasattr(column, "_es_multi_field") and getattr(
column, "_es_multi_field"):
multi_fields = getattr(column, "_es_multi_field")
properties[name] = properties[name].copy()
properties[name]["fields"] = {}
for multi_field_name in multi_fields:
properties[name]["fields"][
multi_field_name] = multi_fields[
multi_field_name].copy()
properties[name]["fields"][multi_field_name].update(
types_map[column_type])
for name, column in relationships.items():
if name in cls._nested_relationships and not depth_reached:
column_type = {'type': 'nested', 'include_in_parent': True}
nested_substitutions.append(name)
submapping, sub_substitutions = column.mapper.class_.get_es_mapping(
_depth=_depth - 1)
column_type.update(list(submapping.values())[0])
properties[name + "_nested"] = column_type
rel_pk_field = column.mapper.class_.pk_field_type()
column_type = types_map[rel_pk_field]
properties[name] = column_type
properties['_pk'] = {'type': 'string'}
return mapping, nested_substitutions
def on_post_save(sender, document, **kw):
from nefertari.elasticsearch import ES
if kw.get('created', False):
ES(document.__class__.__name__).index(document.to_dict())
def index_object(obj, with_refs=True, **kwargs):
from nefertari.elasticsearch import ES
es = ES(obj.__class__.__name__)
es.index(obj.to_dict(), **kwargs)
if with_refs:
es.index_relations(obj, **kwargs)