def setup_fields_processors(config, model_cls, schema):
""" Set up model fields' processors.
:param config: Pyramid Configurator instance.
:param model_cls: Model class for field of which processors should be
set up.
:param schema: Dict of model JSON schema.
"""
properties = schema.get("properties", {})
for field_name, props in properties.items():
if not props:
continue
processors = props.get("_processors")
backref_processors = props.get("_backref_processors")
if processors:
processors = [resolve_to_callable(val) for val in processors]
setup_kwargs = {"model": model_cls, "field": field_name}
config.add_field_processors(processors, **setup_kwargs)
if backref_processors:
db_settings = props.get("_db_settings", {})
is_relationship = db_settings.get("type") == "relationship"
document = db_settings.get("document")
backref_name = db_settings.get("backref_name")
if not (is_relationship and document and backref_name):
continue
backref_processors = [resolve_to_callable(val) for val in backref_processors]
setup_kwargs = {"model": engine.get_document_cls(document), "field": backref_name}
config.add_field_processors(backref_processors, **setup_kwargs)
def run(self):
from nefertari.elasticsearch import ES
ES.setup(self.settings)
model_names = split_strip(self.options.models)
for model_name in model_names:
model = engine.get_document_cls(model_name)
params = self.options.params or ''
params = dict([
[k, v[0]] for k, v in urlparse.parse_qs(params).items()
])
params.setdefault('_limit', params.get('_limit', 10000))
chunk_size = self.options.chunk or params['_limit']
es = ES(source=model_name, index_name=self.options.index)
query_set = model.get_collection(**params)
documents = to_dicts(query_set)
if self.options.force:
es.index(documents, chunk_size=chunk_size)
else:
es.index_missing_documents(documents, chunk_size=chunk_size)
return 0
def run(self):
if self.options.models:
model_names = split_strip(self.options.models)
models = [engine.get_document_cls(name) for name in model_names]
else:
models = None
try:
from_ace = json.loads(self.options.from_ace)
except ValueError as ex:
raise ValueError('--from_ace: {}'.format(ex))
try:
to_ace = json.loads(self.options.to_ace)
except ValueError as ex:
raise ValueError('--to_ace: {}'.format(ex))
six.print_('Updating documents ACE')
update_ace(from_ace=from_ace, to_ace=to_ace, models=models)
try:
import transaction
transaction.commit()
except:
pass
six.print_('Done')
def run(self):
ES.setup(self.settings)
model_names = split_strip(self.options.models)
for model_name in model_names:
self.log.info('Processing model `{}`'.format(model_name))
model = engine.get_document_cls(model_name)
params = self.options.params or ''
params = dict([[k, v[0]]
for k, v in urllib.parse.parse_qs(params).items()])
params.setdefault('_limit', params.get('_limit', 10000))
chunk_size = self.options.chunk or params['_limit']
es = ES(source=model_name,
index_name=self.options.index,
chunk_size=chunk_size)
query_set = model.get_collection(**params)
documents = to_dicts(query_set)
if self.options.force:
self.log.info('Recreating `{}` ES mapping'.format(model_name))
es.delete_mapping()
es.put_mapping(body=model.get_es_mapping())
self.log.info('Indexing all `{}` documents'.format(model_name))
es.index(documents)
else:
self.log.info(
'Indexing missing `{}` documents'.format(model_name))
es.index_missing_documents(documents)
return 0
def index_models(self, model_names):
self.log.info('Indexing models documents')
params = self.options.params or ''
params = dict([[k, v[0]] for k, v in urllib.parse.parse_qs(params).items()])
for model_name in model_names:
self.log.info('Processing model `{}`'.format(model_name))
model = engine.get_document_cls(model_name)
local_params = dict()
local_params.update(params)
if '_limit' not in local_params:
limit = model.get_collection().count()
local_params['_limit'] = limit
chunk_size = int(self.options.chunk or local_params['_limit'])
es = ES(source=model_name, index_name=self.options.index,
chunk_size=chunk_size)
query_set = model.get_collection(**local_params)
documents = to_indexable_dicts(query_set)
self.log.info('Indexing missing `{}` documents'.format(
model_name))
es.index_missing_documents(documents)
def run(self):
ES.setup(self.settings)
model_names = split_strip(self.options.models)
for model_name in model_names:
self.log.info('Processing model `{}`'.format(model_name))
model = engine.get_document_cls(model_name)
params = self.options.params or ''
params = dict([
[k, v[0]] for k, v in urllib.parse.parse_qs(params).items()
])
params.setdefault('_limit', params.get('_limit', 10000))
chunk_size = self.options.chunk or params['_limit']
es = ES(source=model_name, index_name=self.options.index,
chunk_size=chunk_size)
query_set = model.get_collection(**params)
documents = to_dicts(query_set)
if self.options.force:
self.log.info('Recreating `{}` ES mapping'.format(model_name))
es.delete_mapping()
es.put_mapping(body=model.get_es_mapping())
self.log.info('Indexing all `{}` documents'.format(
model_name))
es.index(documents)
else:
self.log.info('Indexing missing `{}` documents'.format(
model_name))
es.index_missing_documents(documents)
return 0
def run(self):
if self.options.models:
model_names = split_strip(self.options.models)
models = [engine.get_document_cls(name)
for name in model_names]
else:
models = None
try:
from_ace = json.loads(self.options.from_ace)
except ValueError as ex:
raise ValueError('--from_ace: {}'.format(ex))
try:
to_ace = json.loads(self.options.to_ace)
except ValueError as ex:
raise ValueError('--to_ace: {}'.format(ex))
six.print_('Updating documents ACE')
update_ace(from_ace=from_ace, to_ace=to_ace, models=models)
try:
import transaction
transaction.commit()
except:
pass
six.print_('Done')
def reindextask(model, boxsize=5000):
"""Index model by small chunks (ie: a box, with a reasonable size)
"""
global log
mcls = engine.get_document_cls(model)
# proceed by chunks of 'boxsize'
count = mcls.get_collection(_count=True)
if count < 1: # Hu ? nothing in DB
return
# Let us be aware of some numbers
boxes = count // boxsize
rest = count % boxsize
es = ES(source=model) # humm quick & dirty: get a connector
log.info('Processing model `{}` with {} documents in {} boxes'.format(
model, count, boxes))
# dump by 'boxes' ; add one for the rest (NB: if rest=0 the last box will be
# empty anyway )
for n in range(boxes + 1):
log.info('Indexing missing `{}` documents (box: {}/{})'.format(
model, n, boxes + 1))
query_set = mcls.get_collection(
_limit=boxsize, _page=n,
_sort=mcls.pk_field()) ## don't forget the sort
documents = to_dicts(query_set)
log.debug('---> from db {} documents ; send to ES'.format(
len(documents)))
## TODO: add a control ? The last box size should be equal to 'rest'
es.index(documents)
def __init__(self, user_model, check=None, credentials_callback=None):
""" Init the policy.
Arguments:
:user_model: String name or class of a User model for which ApiKey
model is to be generated
:check: A callback passed the username, api_key and the request,
expected to return None if user doesn't exist or a sequence of
principal identifiers (possibly empty) if the user does exist.
If callback is None, the username will be assumed to exist with
no principals. Optional.
:credentials_callback: A callback passed the username and current
request, expected to return and user's api key.
Is used to generate 'WWW-Authenticate' header with a value of
valid 'Authorization' request header that should be used to
perform requests.
"""
self.user_model = user_model
if isinstance(self.user_model, six.string_types):
self.user_model = engine.get_document_cls(self.user_model)
create_apikey_model(self.user_model)
self.check = check
self.credentials_callback = credentials_callback
super(ApiKeyAuthenticationPolicy, self).__init__()
def _filter_fields(self, data):
if '_type' not in data:
return data
try:
model_cls = engine.get_document_cls(data['_type'])
except ValueError as ex:
log.error(str(ex))
return data
public_fields = set(getattr(model_cls, '_public_fields', None) or [])
auth_fields = set(getattr(model_cls, '_auth_fields', None) or [])
fields = set(data.keys())
user = getattr(self.request, 'user', None)
if self.request:
# User authenticated
if user:
# User not admin
if not self.is_admin:
fields &= auth_fields
# User not authenticated
else:
fields &= public_fields
fields.add('_type')
fields.add('self')
return data.subset(fields)
def recreate_mapping(model):
""" Delete and Put the mapping of a model.
Effect: delete all document linked to this mapping in the working index.
"""
global log
mcls = engine.get_document_cls(model)
es = ES(model)
# delete: work with elasticsearch=1.7.0
es.api.indices.delete_mapping(es.index_name, doc_type=model)
# put good old mapping.
es.put_mapping(body=mcls.get_es_mapping())
def get_existing_model(model_name):
""" Try to find existing model class named `model_name`.
:param model_name: String name of the model class.
"""
try:
model_cls = engine.get_document_cls(model_name)
log.debug("Model `{}` already exists. Using existing one".format(model_name))
return model_cls
except ValueError:
log.debug("Model `{}` does not exist".format(model_name))
def get_existing_model(model_name):
""" Try to find existing model class named `model_name`.
:param model_name: String name of the model class.
"""
try:
model_cls = engine.get_document_cls(model_name)
log.debug('Model `{}` already exists. Using existing one'.format(
model_name))
return model_cls
except ValueError:
log.debug('Model `{}` does not exist'.format(model_name))
def create_apikey_model(user_model):
""" Generate ApiKey model class and connect it with :user_model:.
ApiKey is generated with relationship to user model class :user_model:
as a One-to-One relationship with a backreference.
ApiKey is set up to be auto-generated when a new :user_model: is created.
Returns ApiKey document class. If ApiKey is already defined, it is not
generated.
Arguments:
:user_model: Class that represents user model for which api keys will
be generated and with which ApiKey will have relationship.
"""
try:
return engine.get_document_cls('ApiKey')
except ValueError:
pass
fk_kwargs = {
'ref_column': None,
}
if hasattr(user_model, '__tablename__'):
fk_kwargs['ref_column'] = '.'.join([
user_model.__tablename__, user_model.pk_field()])
fk_kwargs['ref_column_type'] = user_model.pk_field_type()
class ApiKey(engine.BaseDocument):
__tablename__ = 'nefertari_apikey'
id = engine.IdField(primary_key=True)
token = engine.StringField(default=create_apikey_token)
user = engine.Relationship(
document=user_model.__name__,
uselist=False,
backref_name='api_key',
backref_uselist=False)
user_id = engine.ForeignKeyField(
ref_document=user_model.__name__,
**fk_kwargs)
def reset_token(self):
self.update({'token': create_apikey_token()})
return self.token
# Setup ApiKey autogeneration on :user_model: creation
ApiKey.autogenerate_for(user_model, 'user')
return ApiKey
def setup(req, examples):
""" Setup database state for tests.
NOTE: For objects to be created, when using SQLA transaction
needs to be commited as follows:
import transaction
transaction.commit()
"""
from nefertari import engine
Item = engine.get_document_cls('Item')
if req.match(exclude='POST /items'):
if Item.get_collection(_count=True) == 0:
example = examples.build('item')
Item(**example).save()
def create_apikey_model(user_model):
""" Generate ApiKey model class and connect it with :user_model:.
ApiKey is generated with relationship to user model class :user_model:
as a One-to-One relationship with a backreference.
ApiKey is set up to be auto-generated when a new :user_model: is created.
Returns ApiKey document class. If ApiKey is already defined, it is not
generated.
Arguments:
:user_model: Class that represents user model for which api keys will
be generated and with which ApiKey will have relationship.
"""
try:
return engine.get_document_cls('ApiKey')
except ValueError:
pass
fk_kwargs = {
'ref_column': None,
}
if hasattr(user_model, '__tablename__'):
fk_kwargs['ref_column'] = '.'.join(
[user_model.__tablename__,
user_model.pk_field()])
fk_kwargs['ref_column_type'] = user_model.pk_field_type()
class ApiKey(engine.BaseDocument):
__tablename__ = 'nefertari_apikey'
id = engine.IdField(primary_key=True)
token = engine.StringField(default=create_apikey_token)
user = engine.Relationship(document=user_model.__name__,
uselist=False,
backref_name='api_key',
backref_uselist=False)
user_id = engine.ForeignKeyField(ref_document=user_model.__name__,
**fk_kwargs)
def reset_token(self):
self.update({'token': create_apikey_token()})
return self.token
# Setup ApiKey autogeneration on :user_model: creation
ApiKey.autogenerate_for(user_model, 'user')
return ApiKey
def run(self):
if self.options.models:
model_names = split_strip(self.options.models)
models = [engine.get_document_cls(name) for name in model_names]
else:
models = None
try:
ace = json.loads(self.options.ace)
except ValueError as ex:
raise ValueError('--ace: {}'.format(ex))
counts = count_ace(ace=ace, models=models)
six.print_('Model,Count')
for model, count in counts.items():
if count is None:
count = 'Not es-based'
six.print_('{},{}'.format(model.__name__, count))
def index_models(self, model_names):
self.log.info('Indexing models documents')
params = self.options.params or ''
params = dict([[k, v[0]]
for k, v in urllib.parse.parse_qs(params).items()])
params.setdefault('_limit', params.get('_limit', 10000))
chunk_size = self.options.chunk or params['_limit']
for model_name in model_names:
self.log.info('Processing model `{}`'.format(model_name))
model = engine.get_document_cls(model_name)
es = ES(source=model_name,
index_name=self.options.index,
chunk_size=chunk_size)
query_set = model.get_collection(**params)
documents = to_dicts(query_set)
self.log.info('Indexing missing `{}` documents'.format(model_name))
es.index_missing_documents(documents)
def index_models(self, model_names):
self.log.info('Indexing models documents')
params = self.options.params or ''
params = dict([
[k, v[0]] for k, v in urllib.parse.parse_qs(params).items()
])
params.setdefault('_limit', params.get('_limit', 10000))
chunk_size = self.options.chunk or params['_limit']
for model_name in model_names:
self.log.info('Processing model `{}`'.format(model_name))
model = engine.get_document_cls(model_name)
es = ES(source=model_name, index_name=self.options.index,
chunk_size=chunk_size)
query_set = model.get_collection(**params)
documents = to_dicts(query_set)
self.log.info('Indexing missing `{}` documents'.format(
model_name))
es.index_missing_documents(documents)
def _group_by_type(documents, models=None):
""" Group documents by document class.
:param documents: List of documents to group.
:param models: List models classes of documents.
:returns: Dict of grouped documents of format
{Model: [doc1, doc2, ...]}.
"""
doc_classes = {}
if models is not None:
doc_classes.update({model.__name__: model for model in models})
grouped = defaultdict(list)
for doc in documents:
if doc._type not in doc_classes:
doc_classes[doc._type] = engine.get_document_cls(doc._type)
doc_cls = doc_classes[doc._type]
grouped[doc_cls].append(doc)
return grouped
def _group_by_type(documents, models=None):
""" Group documents by document class.
:param documents: List of documents to group.
:param models: List models classes of documents.
:returns: Dict of grouped documents of format
{Model: [doc1, doc2, ...]}.
"""
doc_classes = {}
if models is not None:
doc_classes.update({model.__name__: model for model in models})
grouped = defaultdict(list)
for doc in documents:
if doc._type not in doc_classes:
doc_classes[doc._type] = engine.get_document_cls(doc._type)
doc_cls = doc_classes[doc._type]
grouped[doc_cls].append(doc)
return grouped
def _filter_fields(self, data):
if '_type' not in data:
return data
try:
model_cls = engine.get_document_cls(data['_type'])
except ValueError as ex:
log.error(str(ex))
return data
public_fields = set(getattr(model_cls, '_public_fields', None) or [])
auth_fields = set(getattr(model_cls, '_auth_fields', None) or [])
hidden_fields = set(getattr(model_cls, '_hidden_fields', None) or [])
fields = set(data.keys())
user = getattr(self.request, 'user', None)
if self.request:
# User authenticated
if user:
# User not admin
if not self.is_admin:
fields &= auth_fields
# User not authenticated
else:
fields &= public_fields
if self.drop_hidden:
if not self.is_admin:
fields -= hidden_fields
else:
fields.update(hidden_fields)
fields.update(['_type', '_pk', '_self'])
if not isinstance(data, dictset):
data = dictset(data)
data = data.subset(fields)
return self._apply_nested_privacy(data)
def _filter_fields(self, data):
if '_type' not in data:
return data
try:
model_cls = engine.get_document_cls(data['_type'])
except ValueError as ex:
log.error(str(ex))
return data
public_fields = set(getattr(model_cls, '_public_fields', None) or [])
auth_fields = set(getattr(model_cls, '_auth_fields', None) or [])
hidden_fields = set(getattr(model_cls, '_hidden_fields', None) or [])
fields = set(data.keys())
user = getattr(self.request, 'user', None)
if self.request:
# User authenticated
if user:
# User not admin
if not self.is_admin:
fields &= auth_fields
# User not authenticated
else:
fields &= public_fields
if self.drop_hidden:
if not self.is_admin:
fields -= hidden_fields
else:
fields.update(hidden_fields)
fields.update(['_type', '_pk', '_self'])
if not isinstance(data, dictset):
data = dictset(data)
data = data.subset(fields)
return self._apply_nested_privacy(data)
def setup_fields_processors(config, model_cls, schema):
""" Set up model fields' processors.
:param config: Pyramid Configurator instance.
:param model_cls: Model class for field of which processors should be
set up.
:param schema: Dict of model JSON schema.
"""
properties = schema.get('properties', {})
for field_name, props in properties.items():
if not props:
continue
processors = props.get('_processors')
backref_processors = props.get('_backref_processors')
if processors:
processors = [resolve_to_callable(val) for val in processors]
setup_kwargs = {'model': model_cls, 'field': field_name}
config.add_field_processors(processors, **setup_kwargs)
if backref_processors:
db_settings = props.get('_db_settings', {})
is_relationship = db_settings.get('type') == 'relationship'
document = db_settings.get('document')
backref_name = db_settings.get('backref_name')
if not (is_relationship and document and backref_name):
continue
backref_processors = [
resolve_to_callable(val) for val in backref_processors]
setup_kwargs = {
'model': engine.get_document_cls(document),
'field': backref_name
}
config.add_field_processors(
backref_processors, **setup_kwargs)
