def setup_fields_processors(config, model_cls, schema): """ Set up model fields' processors. :param config: Pyramid Configurator instance. :param model_cls: Model class for field of which processors should be set up. :param schema: Dict of model JSON schema. """ properties = schema.get("properties", {}) for field_name, props in properties.items(): if not props: continue processors = props.get("_processors") backref_processors = props.get("_backref_processors") if processors: processors = [resolve_to_callable(val) for val in processors] setup_kwargs = {"model": model_cls, "field": field_name} config.add_field_processors(processors, **setup_kwargs) if backref_processors: db_settings = props.get("_db_settings", {}) is_relationship = db_settings.get("type") == "relationship" document = db_settings.get("document") backref_name = db_settings.get("backref_name") if not (is_relationship and document and backref_name): continue backref_processors = [resolve_to_callable(val) for val in backref_processors] setup_kwargs = {"model": engine.get_document_cls(document), "field": backref_name} config.add_field_processors(backref_processors, **setup_kwargs)
def run(self): from nefertari.elasticsearch import ES ES.setup(self.settings) model_names = split_strip(self.options.models) for model_name in model_names: model = engine.get_document_cls(model_name) params = self.options.params or '' params = dict([ [k, v[0]] for k, v in urlparse.parse_qs(params).items() ]) params.setdefault('_limit', params.get('_limit', 10000)) chunk_size = self.options.chunk or params['_limit'] es = ES(source=model_name, index_name=self.options.index) query_set = model.get_collection(**params) documents = to_dicts(query_set) if self.options.force: es.index(documents, chunk_size=chunk_size) else: es.index_missing_documents(documents, chunk_size=chunk_size) return 0
def run(self): if self.options.models: model_names = split_strip(self.options.models) models = [engine.get_document_cls(name) for name in model_names] else: models = None try: from_ace = json.loads(self.options.from_ace) except ValueError as ex: raise ValueError('--from_ace: {}'.format(ex)) try: to_ace = json.loads(self.options.to_ace) except ValueError as ex: raise ValueError('--to_ace: {}'.format(ex)) six.print_('Updating documents ACE') update_ace(from_ace=from_ace, to_ace=to_ace, models=models) try: import transaction transaction.commit() except: pass six.print_('Done')
def run(self): ES.setup(self.settings) model_names = split_strip(self.options.models) for model_name in model_names: self.log.info('Processing model `{}`'.format(model_name)) model = engine.get_document_cls(model_name) params = self.options.params or '' params = dict([[k, v[0]] for k, v in urllib.parse.parse_qs(params).items()]) params.setdefault('_limit', params.get('_limit', 10000)) chunk_size = self.options.chunk or params['_limit'] es = ES(source=model_name, index_name=self.options.index, chunk_size=chunk_size) query_set = model.get_collection(**params) documents = to_dicts(query_set) if self.options.force: self.log.info('Recreating `{}` ES mapping'.format(model_name)) es.delete_mapping() es.put_mapping(body=model.get_es_mapping()) self.log.info('Indexing all `{}` documents'.format(model_name)) es.index(documents) else: self.log.info( 'Indexing missing `{}` documents'.format(model_name)) es.index_missing_documents(documents) return 0
def index_models(self, model_names): self.log.info('Indexing models documents') params = self.options.params or '' params = dict([[k, v[0]] for k, v in urllib.parse.parse_qs(params).items()]) for model_name in model_names: self.log.info('Processing model `{}`'.format(model_name)) model = engine.get_document_cls(model_name) local_params = dict() local_params.update(params) if '_limit' not in local_params: limit = model.get_collection().count() local_params['_limit'] = limit chunk_size = int(self.options.chunk or local_params['_limit']) es = ES(source=model_name, index_name=self.options.index, chunk_size=chunk_size) query_set = model.get_collection(**local_params) documents = to_indexable_dicts(query_set) self.log.info('Indexing missing `{}` documents'.format( model_name)) es.index_missing_documents(documents)
def run(self): ES.setup(self.settings) model_names = split_strip(self.options.models) for model_name in model_names: self.log.info('Processing model `{}`'.format(model_name)) model = engine.get_document_cls(model_name) params = self.options.params or '' params = dict([ [k, v[0]] for k, v in urllib.parse.parse_qs(params).items() ]) params.setdefault('_limit', params.get('_limit', 10000)) chunk_size = self.options.chunk or params['_limit'] es = ES(source=model_name, index_name=self.options.index, chunk_size=chunk_size) query_set = model.get_collection(**params) documents = to_dicts(query_set) if self.options.force: self.log.info('Recreating `{}` ES mapping'.format(model_name)) es.delete_mapping() es.put_mapping(body=model.get_es_mapping()) self.log.info('Indexing all `{}` documents'.format( model_name)) es.index(documents) else: self.log.info('Indexing missing `{}` documents'.format( model_name)) es.index_missing_documents(documents) return 0
def reindextask(model, boxsize=5000): """Index model by small chunks (ie: a box, with a reasonable size) """ global log mcls = engine.get_document_cls(model) # proceed by chunks of 'boxsize' count = mcls.get_collection(_count=True) if count < 1: # Hu ? nothing in DB return # Let us be aware of some numbers boxes = count // boxsize rest = count % boxsize es = ES(source=model) # humm quick & dirty: get a connector log.info('Processing model `{}` with {} documents in {} boxes'.format( model, count, boxes)) # dump by 'boxes' ; add one for the rest (NB: if rest=0 the last box will be # empty anyway ) for n in range(boxes + 1): log.info('Indexing missing `{}` documents (box: {}/{})'.format( model, n, boxes + 1)) query_set = mcls.get_collection( _limit=boxsize, _page=n, _sort=mcls.pk_field()) ## don't forget the sort documents = to_dicts(query_set) log.debug('---> from db {} documents ; send to ES'.format( len(documents))) ## TODO: add a control ? The last box size should be equal to 'rest' es.index(documents)
def __init__(self, user_model, check=None, credentials_callback=None): """ Init the policy. Arguments: :user_model: String name or class of a User model for which ApiKey model is to be generated :check: A callback passed the username, api_key and the request, expected to return None if user doesn't exist or a sequence of principal identifiers (possibly empty) if the user does exist. If callback is None, the username will be assumed to exist with no principals. Optional. :credentials_callback: A callback passed the username and current request, expected to return and user's api key. Is used to generate 'WWW-Authenticate' header with a value of valid 'Authorization' request header that should be used to perform requests. """ self.user_model = user_model if isinstance(self.user_model, six.string_types): self.user_model = engine.get_document_cls(self.user_model) create_apikey_model(self.user_model) self.check = check self.credentials_callback = credentials_callback super(ApiKeyAuthenticationPolicy, self).__init__()
def _filter_fields(self, data): if '_type' not in data: return data try: model_cls = engine.get_document_cls(data['_type']) except ValueError as ex: log.error(str(ex)) return data public_fields = set(getattr(model_cls, '_public_fields', None) or []) auth_fields = set(getattr(model_cls, '_auth_fields', None) or []) fields = set(data.keys()) user = getattr(self.request, 'user', None) if self.request: # User authenticated if user: # User not admin if not self.is_admin: fields &= auth_fields # User not authenticated else: fields &= public_fields fields.add('_type') fields.add('self') return data.subset(fields)
def recreate_mapping(model): """ Delete and Put the mapping of a model. Effect: delete all document linked to this mapping in the working index. """ global log mcls = engine.get_document_cls(model) es = ES(model) # delete: work with elasticsearch=1.7.0 es.api.indices.delete_mapping(es.index_name, doc_type=model) # put good old mapping. es.put_mapping(body=mcls.get_es_mapping())
def get_existing_model(model_name): """ Try to find existing model class named `model_name`. :param model_name: String name of the model class. """ try: model_cls = engine.get_document_cls(model_name) log.debug("Model `{}` already exists. Using existing one".format(model_name)) return model_cls except ValueError: log.debug("Model `{}` does not exist".format(model_name))
def get_existing_model(model_name): """ Try to find existing model class named `model_name`. :param model_name: String name of the model class. """ try: model_cls = engine.get_document_cls(model_name) log.debug('Model `{}` already exists. Using existing one'.format( model_name)) return model_cls except ValueError: log.debug('Model `{}` does not exist'.format(model_name))
def create_apikey_model(user_model): """ Generate ApiKey model class and connect it with :user_model:. ApiKey is generated with relationship to user model class :user_model: as a One-to-One relationship with a backreference. ApiKey is set up to be auto-generated when a new :user_model: is created. Returns ApiKey document class. If ApiKey is already defined, it is not generated. Arguments: :user_model: Class that represents user model for which api keys will be generated and with which ApiKey will have relationship. """ try: return engine.get_document_cls('ApiKey') except ValueError: pass fk_kwargs = { 'ref_column': None, } if hasattr(user_model, '__tablename__'): fk_kwargs['ref_column'] = '.'.join([ user_model.__tablename__, user_model.pk_field()]) fk_kwargs['ref_column_type'] = user_model.pk_field_type() class ApiKey(engine.BaseDocument): __tablename__ = 'nefertari_apikey' id = engine.IdField(primary_key=True) token = engine.StringField(default=create_apikey_token) user = engine.Relationship( document=user_model.__name__, uselist=False, backref_name='api_key', backref_uselist=False) user_id = engine.ForeignKeyField( ref_document=user_model.__name__, **fk_kwargs) def reset_token(self): self.update({'token': create_apikey_token()}) return self.token # Setup ApiKey autogeneration on :user_model: creation ApiKey.autogenerate_for(user_model, 'user') return ApiKey
def setup(req, examples): """ Setup database state for tests. NOTE: For objects to be created, when using SQLA transaction needs to be commited as follows: import transaction transaction.commit() """ from nefertari import engine Item = engine.get_document_cls('Item') if req.match(exclude='POST /items'): if Item.get_collection(_count=True) == 0: example = examples.build('item') Item(**example).save()
def create_apikey_model(user_model): """ Generate ApiKey model class and connect it with :user_model:. ApiKey is generated with relationship to user model class :user_model: as a One-to-One relationship with a backreference. ApiKey is set up to be auto-generated when a new :user_model: is created. Returns ApiKey document class. If ApiKey is already defined, it is not generated. Arguments: :user_model: Class that represents user model for which api keys will be generated and with which ApiKey will have relationship. """ try: return engine.get_document_cls('ApiKey') except ValueError: pass fk_kwargs = { 'ref_column': None, } if hasattr(user_model, '__tablename__'): fk_kwargs['ref_column'] = '.'.join( [user_model.__tablename__, user_model.pk_field()]) fk_kwargs['ref_column_type'] = user_model.pk_field_type() class ApiKey(engine.BaseDocument): __tablename__ = 'nefertari_apikey' id = engine.IdField(primary_key=True) token = engine.StringField(default=create_apikey_token) user = engine.Relationship(document=user_model.__name__, uselist=False, backref_name='api_key', backref_uselist=False) user_id = engine.ForeignKeyField(ref_document=user_model.__name__, **fk_kwargs) def reset_token(self): self.update({'token': create_apikey_token()}) return self.token # Setup ApiKey autogeneration on :user_model: creation ApiKey.autogenerate_for(user_model, 'user') return ApiKey
def run(self): if self.options.models: model_names = split_strip(self.options.models) models = [engine.get_document_cls(name) for name in model_names] else: models = None try: ace = json.loads(self.options.ace) except ValueError as ex: raise ValueError('--ace: {}'.format(ex)) counts = count_ace(ace=ace, models=models) six.print_('Model,Count') for model, count in counts.items(): if count is None: count = 'Not es-based' six.print_('{},{}'.format(model.__name__, count))
def index_models(self, model_names): self.log.info('Indexing models documents') params = self.options.params or '' params = dict([[k, v[0]] for k, v in urllib.parse.parse_qs(params).items()]) params.setdefault('_limit', params.get('_limit', 10000)) chunk_size = self.options.chunk or params['_limit'] for model_name in model_names: self.log.info('Processing model `{}`'.format(model_name)) model = engine.get_document_cls(model_name) es = ES(source=model_name, index_name=self.options.index, chunk_size=chunk_size) query_set = model.get_collection(**params) documents = to_dicts(query_set) self.log.info('Indexing missing `{}` documents'.format(model_name)) es.index_missing_documents(documents)
def index_models(self, model_names): self.log.info('Indexing models documents') params = self.options.params or '' params = dict([ [k, v[0]] for k, v in urllib.parse.parse_qs(params).items() ]) params.setdefault('_limit', params.get('_limit', 10000)) chunk_size = self.options.chunk or params['_limit'] for model_name in model_names: self.log.info('Processing model `{}`'.format(model_name)) model = engine.get_document_cls(model_name) es = ES(source=model_name, index_name=self.options.index, chunk_size=chunk_size) query_set = model.get_collection(**params) documents = to_dicts(query_set) self.log.info('Indexing missing `{}` documents'.format( model_name)) es.index_missing_documents(documents)
def _group_by_type(documents, models=None): """ Group documents by document class. :param documents: List of documents to group. :param models: List models classes of documents. :returns: Dict of grouped documents of format {Model: [doc1, doc2, ...]}. """ doc_classes = {} if models is not None: doc_classes.update({model.__name__: model for model in models}) grouped = defaultdict(list) for doc in documents: if doc._type not in doc_classes: doc_classes[doc._type] = engine.get_document_cls(doc._type) doc_cls = doc_classes[doc._type] grouped[doc_cls].append(doc) return grouped
def _filter_fields(self, data): if '_type' not in data: return data try: model_cls = engine.get_document_cls(data['_type']) except ValueError as ex: log.error(str(ex)) return data public_fields = set(getattr(model_cls, '_public_fields', None) or []) auth_fields = set(getattr(model_cls, '_auth_fields', None) or []) hidden_fields = set(getattr(model_cls, '_hidden_fields', None) or []) fields = set(data.keys()) user = getattr(self.request, 'user', None) if self.request: # User authenticated if user: # User not admin if not self.is_admin: fields &= auth_fields # User not authenticated else: fields &= public_fields if self.drop_hidden: if not self.is_admin: fields -= hidden_fields else: fields.update(hidden_fields) fields.update(['_type', '_pk', '_self']) if not isinstance(data, dictset): data = dictset(data) data = data.subset(fields) return self._apply_nested_privacy(data)
def setup_fields_processors(config, model_cls, schema): """ Set up model fields' processors. :param config: Pyramid Configurator instance. :param model_cls: Model class for field of which processors should be set up. :param schema: Dict of model JSON schema. """ properties = schema.get('properties', {}) for field_name, props in properties.items(): if not props: continue processors = props.get('_processors') backref_processors = props.get('_backref_processors') if processors: processors = [resolve_to_callable(val) for val in processors] setup_kwargs = {'model': model_cls, 'field': field_name} config.add_field_processors(processors, **setup_kwargs) if backref_processors: db_settings = props.get('_db_settings', {}) is_relationship = db_settings.get('type') == 'relationship' document = db_settings.get('document') backref_name = db_settings.get('backref_name') if not (is_relationship and document and backref_name): continue backref_processors = [ resolve_to_callable(val) for val in backref_processors] setup_kwargs = { 'model': engine.get_document_cls(document), 'field': backref_name } config.add_field_processors( backref_processors, **setup_kwargs)