def update_index_for_objects(session, cache): indexer = get_indexer() for obj in cache.new: indexer.create_record(fts_record_for(obj), commit=False) for obj in cache.dirty: indexer.update_record(fts_record_for(obj), commit=False) for obj in cache.deleted: indexer.delete_record(obj.id, obj.__class__.__name__, commit=False) session.commit()
def do_reindex(): """ update the full text search index """ indexer = get_indexer() indexer.delete_all_records(False) # Remove model base classes and non searchable objects excluded_models = { all_models.Directive, all_models.Option, all_models.SystemOrProcess } indexed_models = { model for model in all_models.all_models if model_is_indexed(model) } indexed_models -= excluded_models for model in indexed_models: mapper_class = model._sa_class_manager.mapper.base_mapper.class_ query = model.query.options( db.undefer_group(mapper_class.__name__ + '_complete'), ) for query_chunk in generate_query_chunks(query): for instance in query_chunk: indexer.create_record(fts_record_for(instance), False) db.session.commit()
def do_reindex(): """Update the full text search index.""" indexer = get_indexer() indexer.delete_all_records(False) # Remove model base classes and non searchable objects excluded_models = { all_models.Directive, all_models.Option, all_models.SystemOrProcess, all_models.Role, } indexed_models = {model for model in all_models.all_models if model_is_indexed(model)} indexed_models -= excluded_models for model in indexed_models: # pylint: disable=protected-access mapper_class = model._sa_class_manager.mapper.base_mapper.class_ query = model.query.options( db.undefer_group(mapper_class.__name__ + '_complete'), ) for query_chunk in generate_query_chunks(query): for instance in query_chunk: indexer.create_record(fts_record_for(instance), False) db.session.commit() reindex_snapshots()
def ensure_backlog_workflow_exists(cls): """Ensures there is at least one backlog workflow with an active cycle. If such workflow does not exist it creates one.""" def any_active_cycle(workflows): """Checks if any active cycle exists from given workflows""" for workflow in workflows: for cur_cycle in workflow.cycles: if cur_cycle.is_current: return True return False # Check if backlog workflow already exists backlog_workflows = Workflow.query\ .filter(and_ (Workflow.kind == "Backlog", Workflow.frequency == "one_time"))\ .all() if len(backlog_workflows) > 0 and any_active_cycle(backlog_workflows): return "At least one backlog workflow already exists" # Create a backlog workflow backlog_workflow = Workflow(description="Backlog workflow", title="Backlog (one time)", frequency="one_time", status="Active", recurrences=0, kind="Backlog") # create wf context wf_ctx = backlog_workflow.get_or_create_object_context(context=1) backlog_workflow.context = wf_ctx db.session.flush(backlog_workflow) # create a cycle backlog_cycle = cycle.Cycle( description="Backlog workflow", title="Backlog (one time)", is_current=1, status="Assigned", start_date=None, end_date=None, context=backlog_workflow.get_or_create_object_context(), workflow=backlog_workflow) # create a cycletaskgroup backlog_ctg = cycle_task_group\ .CycleTaskGroup(description="Backlog workflow taskgroup", title="Backlog TaskGroup", cycle=backlog_cycle, status="InProgress", start_date=None, end_date=None, context=backlog_workflow .get_or_create_object_context()) db.session.add_all([backlog_workflow, backlog_cycle, backlog_ctg]) db.session.flush() # add fulltext entries get_indexer().create_record(fts_record_for(backlog_workflow)) return "Backlog workflow created"
def do_reindex(): """Update the full text search index.""" indexer = get_indexer() with benchmark('Delete all records'): indexer.delete_all_records(False) indexed_models = get_indexed_model_names() people = db.session.query(all_models.Person.id, all_models.Person.name, all_models.Person.email) g.people_map = {p.id: (p.name, p.email) for p in people} for model in sorted(indexed_models): # pylint: disable=protected-access logger.info("Updating index for: %s", model) with benchmark("Create records for %s" % model): model = get_model(model) mapper_class = model._sa_class_manager.mapper.base_mapper.class_ query = model.query.options( db.undefer_group(mapper_class.__name__ + '_complete'), ) for query_chunk in generate_query_chunks(query): for instance in query_chunk: indexer.create_record(fts_record_for(instance), False) db.session.commit() reindex_snapshots() delattr(g, "people_map")
def admin_reindex(): """Simple re-index of all indexable objects """ if not permissions.is_allowed_read("/admin", 1): raise Forbidden() from ggrc.fulltext import get_indexer from ggrc.fulltext.recordbuilder import fts_record_for indexer = get_indexer() indexer.delete_all_records(False) from ggrc.models import all_models from ggrc.app import db # Find all models then remove base classes models = set(all_models.all_models) -\ set([all_models.Directive, all_models.SystemOrProcess]) for model in models: mapper_class = model._sa_class_manager.mapper.base_mapper.class_ query = model.query.options( db.undefer_group(mapper_class.__name__+'_complete'), ) for instance in query.all(): indexer.create_record(fts_record_for(instance), False) db.session.commit() return app.make_response(( 'success', 200, [('Content-Type', 'text/html')]))
def create_user(email, **kwargs): user = Person(email=email, **kwargs) db.session.add(user) db.session.flush() log_event(db.session, user, user.id) user_context = Context( name='Personal Context for {0}'.format(email), description='', related_object=user, context_id=1, ) db.session.add(user_context) db.session.commit() get_indexer().create_record(fts_record_for(user)) return user
def _log_event(cls, instance): indexer = get_indexer() db.session.flush() user = cls._get_user() revision = models.Revision(instance, user.id, 'created', instance.log_json()) event = models.Event( modified_by=user, action="POST", resource_id=instance.id, resource_type=instance.type, context=instance.context, revisions=[revision], ) db.session.add(revision) db.session.add(event) indexer.update_record(fts_record_for(instance), commit=False)
def _log_event(cls, instance): indexer = get_indexer() db.session.flush() user = cls._get_user() revision = models.Revision( instance, user.id, 'created', instance.log_json()) event = models.Event( modified_by=user, action="POST", resource_id=instance.id, resource_type=instance.type, context=instance.context, revisions=[revision], ) db.session.add(revision) db.session.add(event) indexer.update_record(fts_record_for(instance), commit=False)
def do_reindex(): """ update the full text search index """ indexer = get_indexer() indexer.delete_all_records(False) # Find all models then remove base classes # (If we don't remove base classes, we get duplicates in the index.) inheritance_base_models = [ all_models.Directive, all_models.SystemOrProcess ] models_ = set(all_models.all_models) - set(inheritance_base_models) models_ = [model for model in models_ if model_is_indexed(model)] for model in models_: mapper_class = model._sa_class_manager.mapper.base_mapper.class_ query = model.query.options( db.undefer_group(mapper_class.__name__ + '_complete'), ) for query_chunk in generate_query_chunks(query): for instance in query_chunk: indexer.create_record(fts_record_for(instance), False) db.session.commit()
def ensure_backlog_workflow_exists(cls): """Ensures there is at least one backlog workflow with an active cycle. If such workflow does not exist it creates one.""" def any_active_cycle(workflows): """Checks if any active cycle exists from given workflows""" for workflow in workflows: for cur_cycle in workflow.cycles: if cur_cycle.is_current: return True return False # Check if backlog workflow already exists backlog_workflows = Workflow.query\ .filter(and_ (Workflow.kind == "Backlog", Workflow.frequency == "one_time"))\ .all() if len(backlog_workflows) > 0 and any_active_cycle(backlog_workflows): return "At least one backlog workflow already exists" # Create a backlog workflow backlog_workflow = Workflow(description="Backlog workflow", title="Backlog (one time)", frequency="one_time", status="Active", recurrences=0, kind="Backlog") # create wf context wf_ctx = backlog_workflow.get_or_create_object_context(context=1) backlog_workflow.context = wf_ctx db.session.flush(backlog_workflow) # create a cycle backlog_cycle = cycle.Cycle(description="Backlog workflow", title="Backlog (one time)", is_current=1, status="Assigned", start_date=None, end_date=None, context=backlog_workflow .get_or_create_object_context(), workflow=backlog_workflow) # create a cycletaskgroup backlog_ctg = cycle_task_group\ .CycleTaskGroup(description="Backlog workflow taskgroup", title="Backlog TaskGroup", cycle=backlog_cycle, status="InProgress", start_date=None, end_date=None, context=backlog_workflow .get_or_create_object_context()) db.session.add_all([backlog_workflow, backlog_cycle, backlog_ctg]) db.session.flush() # add fulltext entries get_indexer().create_record(fts_record_for(backlog_workflow)) get_indexer().create_record(fts_record_for(backlog_cycle)) get_indexer().create_record(fts_record_for(backlog_ctg)) return "Backlog workflow created"
if not permissions.is_allowed_create(self.model.__name__, self.get_context_id_from_json(src)): raise Forbidden() if src.get("private") == True and src.get("context") is not None and src["context"].get("id") is not None: raise BadRequest('context MUST be "null" when creating a private resource.') elif "context" not in src: raise BadRequest("context MUST be specified.") else: if not permissions.is_allowed_create(self.model.__name__, self.get_context_id_from_json(src)): raise Forbidden() self.json_create(obj, src) self.model_posted.send(obj.__class__, obj=obj, src=src, service=self) obj.modified_by_id = get_current_user_id() db.session.add(obj) log_event(db.session, obj) db.session.commit() get_indexer().create_record(fts_record_for(obj)) return self.json_success_response(self.object_for_json(obj), self.modified_at(obj), id=obj.id, status=201) @classmethod def add_to(cls, app, url, model_class=None, decorators=()): if model_class: service_class = type(model_class.__name__, (cls,), {"_model": model_class}) import ggrc.services setattr(ggrc.services, model_class.__name__, service_class) else: service_class = cls view_func = service_class.as_view(service_class.endpoint_name()) view_func = cls.decorate_view_func(view_func, decorators) app.add_url_rule(url, defaults={cls.pk: None}, view_func=view_func, methods=["GET", "POST"]) app.add_url_rule(
class Resource(ModelView): """View base class for Views handling. Will typically be registered with an application following a collection style for routes. Collection `GET` and `POST` will have a route like `/resources` while collection member resource routes will have routes likej `/resources/<pk:pk_type>`. To register a Resource subclass FooCollection with a Flask application: .. FooCollection.add_to(app, '/foos') By default will only support the `application/json` content-type. """ def dispatch_request(self, *args, **kwargs): method = request.method.lower() if method == 'get': if self.pk in kwargs and kwargs[self.pk] is not None: return self.get(*args, **kwargs) else: return self.collection_get() elif method == 'post': if self.pk in kwargs and kwargs[self.pk] is not None: return self.post(*args, **kwargs) else: return self.collection_post() elif method == 'put': return self.put(*args, **kwargs) elif method == 'delete': return self.delete(*args, **kwargs) else: raise NotImplementedError() def post(*args, **kwargs): raise NotImplementedError() # Default JSON request handlers def get(self, id): obj = self.get_object(id) if obj is None: return self.not_found_response() if 'Accept' in self.request.headers and \ 'application/json' not in self.request.headers['Accept']: return current_app.make_response( ('application/json', 406, [('Content-Type', 'text/plain')])) object_for_json = self.object_for_json(obj) if 'If-None-Match' in self.request.headers and \ self.request.headers['If-None-Match'] == self.etag(object_for_json): return current_app.make_response( ('', 304, [('Etag', self.etag(object_for_json))])) return self.json_success_response(self.object_for_json(obj), self.modified_at(obj)) def validate_headers_for_put_or_delete(self, obj): missing_headers = [] if 'If-Match' not in self.request.headers: missing_headers.append('If-Match') if 'If-Unmodified-Since' not in self.request.headers: missing_headers.append('If-Unmodified-Since') if missing_headers: # rfc 6585 defines a new status code for missing required headers return current_app.make_response(('If-Match is required.', 428, [ ('Content-Type', 'text/plain') ])) if request.headers['If-Match'] != self.etag(self.object_for_json(obj)) or \ request.headers['If-Unmodified-Since'] != \ self.http_timestamp(self.modified_at(obj)): return current_app.make_response(( 'The resource has been changed. The conflict must be resolved and ' 'the request resubmitted with an up to date Etag for If-Match ' 'header.', 409, [('Content-Type', 'text/plain')])) return None def put(self, id): obj = self.get_object(id) if obj is None: return self.not_found_response() if self.request.headers['Content-Type'] != 'application/json': return current_app.make_response( ('Content-Type must be application/json', 415, [])) header_error = self.validate_headers_for_put_or_delete(obj) if header_error: return header_error src = UnicodeSafeJsonWrapper(self.request.json) root_attribute = self.model._inflector.table_singular try: src = src[root_attribute] except KeyError, e: return current_app.make_response( ('Required attribute "{0}" not found'.format(root_attribute), 400, [])) ggrc.builder.json.update(obj, src) #FIXME Fake the modified_by_id until we have that information in session. obj.modified_by_id = get_current_user_id() db.session.add(obj) db.session.commit() obj = self.get_object(id) get_indexer().update_record(fts_record_for(obj)) return self.json_success_response(self.object_for_json(obj), self.modified_at(obj))
('Content-Type must be application/json', 415, [])) obj = self.model() src = UnicodeSafeJsonWrapper(self.request.json) root_attribute = self.model._inflector.table_singular try: src = src[root_attribute] except KeyError, e: return current_app.make_response( ('Required attribute "{0}" not found'.format(root_attribute), 400, [])) ggrc.builder.json.create(obj, src) #FIXME Fake the modified_by_id until we have that information in session. obj.modified_by_id = get_current_user_id() db.session.add(obj) db.session.commit() get_indexer().create_record(fts_record_for(obj)) return self.json_success_response(self.object_for_json(obj), self.modified_at(obj), id=obj.id, status=201) @classmethod def add_to(cls, app, url, model_class=None, decorators=()): if model_class: service_class = type(model_class.__name__, (Resource, ), { '_model': model_class, }) import ggrc.services setattr(ggrc.services, model_class.__name__, service_class) else: service_class = cls