def ensure_backlog_workflow_exists(cls):
"""Ensures there is at least one backlog workflow with an active cycle.
If such workflow does not exist it creates one."""
def any_active_cycle(workflows):
"""Checks if any active cycle exists from given workflows"""
for workflow in workflows:
for cur_cycle in workflow.cycles:
if cur_cycle.is_current:
return True
return False
# Check if backlog workflow already exists
backlog_workflows = Workflow.query.filter(
and_(Workflow.kind == "Backlog",
# the following means one_time wf
Workflow.unit is None)
).all()
if len(backlog_workflows) > 0 and any_active_cycle(backlog_workflows):
return "At least one backlog workflow already exists"
# Create a backlog workflow
backlog_workflow = Workflow(description="Backlog workflow",
title="Backlog (one time)",
status="Active",
recurrences=0,
kind="Backlog")
# create wf context
wf_ctx = backlog_workflow.get_or_create_object_context(context=1)
backlog_workflow.context = wf_ctx
db.session.flush(backlog_workflow)
# create a cycle
backlog_cycle = cycle.Cycle(description="Backlog workflow",
title="Backlog (one time)",
is_current=1,
status="Assigned",
start_date=None,
end_date=None,
context=backlog_workflow
.get_or_create_object_context(),
workflow=backlog_workflow)
# create a cycletaskgroup
backlog_ctg = cycle_task_group\
.CycleTaskGroup(description="Backlog workflow taskgroup",
title="Backlog TaskGroup",
cycle=backlog_cycle,
status=cycle_task_group.CycleTaskGroup.IN_PROGRESS,
start_date=None,
end_date=None,
context=backlog_workflow
.get_or_create_object_context())
db.session.add_all([backlog_workflow, backlog_cycle, backlog_ctg])
db.session.flush()
# add fulltext entries
get_indexer().create_record(backlog_workflow)
return "Backlog workflow created"
def ensure_backlog_workflow_exists(cls):
"""Ensures there is at least one backlog workflow with an active cycle.
If such workflow does not exist it creates one."""
def any_active_cycle(workflows):
"""Checks if any active cycle exists from given workflows"""
for workflow in workflows:
for cur_cycle in workflow.cycles:
if cur_cycle.is_current:
return True
return False
# Check if backlog workflow already exists
backlog_workflows = Workflow.query.filter(
and_(Workflow.kind == "Backlog",
# the following means one_time wf
Workflow.unit is None)
).all()
if len(backlog_workflows) > 0 and any_active_cycle(backlog_workflows):
return "At least one backlog workflow already exists"
# Create a backlog workflow
backlog_workflow = Workflow(description="Backlog workflow",
title="Backlog (one time)",
status="Active",
recurrences=0,
kind="Backlog")
# create wf context
wf_ctx = backlog_workflow.get_or_create_object_context(context=1)
backlog_workflow.context = wf_ctx
db.session.flush(backlog_workflow)
# create a cycle
backlog_cycle = cycle.Cycle(description="Backlog workflow",
title="Backlog (one time)",
is_current=1,
status="Assigned",
start_date=None,
end_date=None,
context=backlog_workflow
.get_or_create_object_context(),
workflow=backlog_workflow)
# create a cycletaskgroup
backlog_ctg = cycle_task_group\
.CycleTaskGroup(description="Backlog workflow taskgroup",
title="Backlog TaskGroup",
cycle=backlog_cycle,
status=cycle_task_group.CycleTaskGroup.IN_PROGRESS,
start_date=None,
end_date=None,
context=backlog_workflow
.get_or_create_object_context())
db.session.add_all([backlog_workflow, backlog_cycle, backlog_ctg])
db.session.flush()
# add fulltext entries
get_indexer().create_record(backlog_workflow)
return "Backlog workflow created"
def create_reindexed_snapshots(audit_id, objects):
"""Create snapshots for list of provided objects and reindex them"""
# pylint: disable=protected-access
snapshottable_objects = [o for o in objects if o.type in Types.all]
audit = all_models.Audit.query.get(audit_id)
snapshots = TestCase._create_snapshots(audit, snapshottable_objects)
reindex_snapshots_ids = [snap.id for snap in snapshots]
get_indexer().delete_records_by_ids("Snapshot",
reindex_snapshots_ids,
commit=False)
reindex_snapshots(reindex_snapshots_ids)
def create_reindexed_snapshots(audit_id, objects):
"""Create snapshots for list of provided objects and reindex them"""
# pylint: disable=protected-access
snapshottable_objects = [o for o in objects if o.type in Types.all]
audit = all_models.Audit.query.get(audit_id)
snapshots = TestCase._create_snapshots(audit, snapshottable_objects)
reindex_snapshots_ids = [snap.id for snap in snapshots]
get_indexer().delete_records_by_ids("Snapshot",
reindex_snapshots_ids,
commit=False)
reindex_snapshots(reindex_snapshots_ids)
def delete(self, id):
obj = self.get_object(id)
if obj is None:
return self.not_found_response()
header_error = self.validate_headers_for_put_or_delete(obj)
if header_error:
return header_error
db.session.delete(obj)
db.session.commit()
get_indexer().delete_record(self.url_for(id=id))
return self.json_success_response(self.object_for_json(obj),
self.modified_at(obj))
def delete(self, id):
obj = self.get_object(id)
if obj is None:
return self.not_found_response()
header_error = self.validate_headers_for_put_or_delete(obj)
if header_error:
return header_error
db.session.delete(obj)
db.session.commit()
get_indexer().delete_record(self.url_for(id=id))
return self.json_success_response(
self.object_for_json(obj), self.modified_at(obj))
def delete(self, id):
obj = self.get_object(id)
if obj is None:
return self.not_found_response()
header_error = self.validate_headers_for_put_or_delete(obj)
if header_error:
return header_error
if not permissions.is_allowed_delete(self.model.__name__, obj.context_id):
raise Forbidden()
db.session.delete(obj)
log_event(db.session, obj)
db.session.commit()
get_indexer().delete_record(id, self.model.__name__)
return self.json_success_response(self.object_for_json(obj), self.modified_at(obj))
def create_user(email, **kwargs):
user = Person(email=email, **kwargs)
db.session.add(user)
db.session.flush()
log_event(db.session, user, user.id)
user_context = Context(
name='Personal Context for {0}'.format(email),
description='',
related_object=user,
context_id=1,
)
db.session.add(user_context)
db.session.commit()
get_indexer().create_record(fts_record_for(user))
return user
def create_user(email, **kwargs):
user = Person(email=email, **kwargs)
db.session.add(user)
db.session.flush()
log_event(db.session, user, user.id)
user_context = Context(
name='Personal Context for {0}'.format(email),
description='',
related_object=user,
context_id=1,
)
db.session.add(user_context)
db.session.commit()
get_indexer().create_record(fts_record_for(user))
return user
def delete(self, id):
obj = self.get_object(id)
if obj is None:
return self.not_found_response()
header_error = self.validate_headers_for_put_or_delete(obj)
if header_error:
return header_error
if not permissions.is_allowed_delete(self.model.__name__,
obj.context_id):
raise Forbidden()
db.session.delete(obj)
db.session.commit()
get_indexer().delete_record(self.url_for(id=id))
return self.json_success_response(self.object_for_json(obj),
self.modified_at(obj))
def __init__(self, ie_job, dry_run=True, csv_data=None): self.user = getattr(g, '_current_user', None) self.ie_job = ie_job self.dry_run = dry_run self.csv_data = csv_data or [] self.indexer = get_indexer() super(ImportConverter, self).__init__()
def do_reindex():
"""Update the full text search index."""
indexer = get_indexer()
with benchmark('Delete all records'):
indexer.delete_all_records(False)
indexed_models = get_indexed_model_names()
people = db.session.query(all_models.Person.id, all_models.Person.name,
all_models.Person.email)
g.people_map = {p.id: (p.name, p.email) for p in people}
for model in sorted(indexed_models):
# pylint: disable=protected-access
logger.info("Updating index for: %s", model)
with benchmark("Create records for %s" % model):
model = get_model(model)
mapper_class = model._sa_class_manager.mapper.base_mapper.class_
query = model.query.options(
db.undefer_group(mapper_class.__name__ + '_complete'), )
for query_chunk in generate_query_chunks(query):
for instance in query_chunk:
indexer.create_record(fts_record_for(instance), False)
db.session.commit()
reindex_snapshots()
delattr(g, "people_map")
def test_simple_reindex(self):
"""Test for check simple reindex procedure."""
self.client.get("/login")
# Hack to index the person object that we do not delete between different
# tests.
self.client.post("/admin/full_reindex")
with ggrc_factories.single_commit():
for factory in self.INDEXED_MODEL_FACTORIES:
for _ in range(3):
factory()
indexer = fulltext.get_indexer()
count = indexer.record_type.query.filter(
MysqlRecordProperty.type != "Context").count()
indexer.record_type.query.delete()
self.assertNotEqual(count, 0)
db.session.commit()
self.assertEqual(indexer.record_type.query.count(), 0)
self.client.get("/login")
self.client.post("/admin/full_reindex")
# ACR roles are created in migration and aren't removed in setup
# Index for them will be created only after reindexing
reindexed_count = indexer.record_type.query.filter(
MysqlRecordProperty.type != "Context").count()
self.assertEqual(count, reindexed_count)
def test_simple_reindex(self):
"""Test for check simple reindex procedure."""
self.client.get("/login")
# Hack to index the person object that we do not delete between different
# tests.
self.client.post("/admin/full_reindex")
with ggrc_factories.single_commit():
for factory in self.INDEXED_MODEL_FACTORIES:
for _ in range(3):
factory()
indexer = fulltext.get_indexer()
count = indexer.record_type.query.filter(
MysqlRecordProperty.type != "Context"
).count()
indexer.record_type.query.delete()
self.assertNotEqual(count, 0)
db.session.commit()
self.assertEqual(indexer.record_type.query.count(), 0)
self.client.get("/login")
self.client.post("/admin/full_reindex")
# ACR roles are created in migration and aren't removed in setup
# Index for them will be created only after reindexing
reindexed_count = indexer.record_type.query.filter(
MysqlRecordProperty.type != "Context"
).count()
self.assertEqual(count, reindexed_count)
def do_reindex():
"""Update the full text search index."""
indexer = get_indexer()
indexed_models = {
m.__name__: m for m in all_models.all_models
if issubclass(m, mixin.Indexed) and m.REQUIRED_GLOBAL_REINDEX
}
people_query = db.session.query(all_models.Person.id,
all_models.Person.name,
all_models.Person.email)
indexer.cache["people_map"] = {p.id: (p.name, p.email) for p in people_query}
indexer.cache["ac_role_map"] = dict(db.session.query(
all_models.AccessControlRole.id,
all_models.AccessControlRole.name,
))
for model_name in sorted(indexed_models.keys()):
logger.info("Updating index for: %s", model_name)
with benchmark("Create records for %s" % model_name):
model = indexed_models[model_name]
for query_chunk in generate_query_chunks(db.session.query(model.id)):
model.bulk_record_update_for([i.id for i in query_chunk])
db.session.commit()
logger.info("Updating index for: %s", "Snapshot")
with benchmark("Create records for %s" % "Snapshot"):
reindex_snapshots()
indexer.invalidate_cache()
def do_search(terms, list_for_type, types=None, permission_type='read',
permission_model=None, contact_id=None, extra_params=None,
relevant_objects=None):
indexer = get_indexer()
with benchmark("Search"):
results = indexer.search(
terms, types=types, permission_type=permission_type,
permission_model=permission_model, contact_id=contact_id,
extra_params=extra_params
)
related_filter = _build_relevant_filter(types, relevant_objects)
seen_results = {}
for result in results:
id = result.key
model_type = result.type
result_pair = (model_type, id)
if result_pair not in seen_results and related_filter(result_pair):
seen_results[result_pair] = True
entries_list = list_for_type(model_type)
entries_list.append({
'id': id,
'type': model_type,
'href': url_for(model_type, id=id),
})
def admin_reindex():
"""Simple re-index of all indexable objects
"""
if not permissions.is_allowed_read("/admin", 1):
raise Forbidden()
from ggrc.fulltext import get_indexer
from ggrc.fulltext.recordbuilder import fts_record_for
indexer = get_indexer()
indexer.delete_all_records(False)
from ggrc.models import all_models
from ggrc.app import db
# Find all models then remove base classes
models = set(all_models.all_models) -\
set([all_models.Directive, all_models.SystemOrProcess])
for model in models:
mapper_class = model._sa_class_manager.mapper.base_mapper.class_
query = model.query.options(
db.undefer_group(mapper_class.__name__+'_complete'),
)
for instance in query.all():
indexer.create_record(fts_record_for(instance), False)
db.session.commit()
return app.make_response((
'success', 200, [('Content-Type', 'text/html')]))
def do_search(terms, list_for_type, types=None, permission_type='read',
contact_id=None, extra_params=None,
relevant_objects=None):
indexer = get_indexer()
with benchmark("Search"):
results = indexer.search(
terms, types=types, permission_type=permission_type,
contact_id=contact_id,
extra_params=extra_params
)
related_filter = _build_relevant_filter(types, relevant_objects)
seen_results = {}
for result in results:
id = result.key
model_type = result.type
result_pair = (model_type, id)
if result_pair not in seen_results and related_filter(result_pair):
seen_results[result_pair] = True
entries_list = list_for_type(model_type)
entries_list.append({
'id': id,
'type': model_type,
'href': url_for(model_type, id=id),
})
def do_reindex(with_reindex_snapshots=False):
"""Update the full text search index."""
indexer = get_indexer()
indexed_models = {
m.__name__: m for m in all_models.all_models
if issubclass(m, mixin.Indexed) and m.REQUIRED_GLOBAL_REINDEX
}
people_query = db.session.query(all_models.Person.id,
all_models.Person.name,
all_models.Person.email)
indexer.cache["people_map"] = {p.id: (p.name, p.email) for p in people_query}
indexer.cache["ac_role_map"] = dict(db.session.query(
all_models.AccessControlRole.id,
all_models.AccessControlRole.name,
))
for model_name in sorted(indexed_models.keys()):
logger.info("Updating index for: %s", model_name)
with benchmark("Create records for %s" % model_name):
model = indexed_models[model_name]
ids = [obj.id for obj in model.query]
ids_count = len(ids)
handled_ids = 0
for ids_chunk in utils.list_chunks(ids, chunk_size=REINDEX_CHUNK_SIZE):
handled_ids += len(ids_chunk)
logger.info("%s: %s / %s", model.__name__, handled_ids, ids_count)
model.bulk_record_update_for(ids_chunk)
db.session.commit()
if with_reindex_snapshots:
logger.info("Updating index for: %s", "Snapshot")
with benchmark("Create records for %s" % "Snapshot"):
snapshot_indexer.reindex()
indexer.invalidate_cache()
def do_reindex():
"""Update the full text search index."""
indexer = get_indexer()
indexed_models = get_indexed_model_names()
people = db.session.query(all_models.Person.id, all_models.Person.name,
all_models.Person.email)
indexer.cache["people_map"] = {p.id: (p.name, p.email) for p in people}
for model in sorted(indexed_models):
# pylint: disable=protected-access
logger.info("Updating index for: %s", model)
with benchmark("Create records for %s" % model):
model = get_model(model)
mapper_class = model._sa_class_manager.mapper.base_mapper.class_
if issubclass(model, mixin.Indexed):
for query_chunk in generate_query_chunks(
db.session.query(model.id)):
model.bulk_record_update_for([i.id for i in query_chunk])
db.session.commit()
else:
logger.warning(
"Try to index model that not inherited from Indexed mixin: %s",
model.__name__)
indexer.delete_records_by_type(model.__name__)
query = model.query.options(
db.undefer_group(mapper_class.__name__ + '_complete'), )
for query_chunk in generate_query_chunks(query):
for instance in query_chunk:
indexer.create_record(indexer.fts_record_for(instance),
False)
db.session.commit()
reindex_snapshots()
indexer.invalidate_cache()
def do_reindex():
"""Update the full text search index."""
indexer = get_indexer()
indexer.delete_all_records(False)
# Remove model base classes and non searchable objects
excluded_models = {
all_models.Directive,
all_models.Option,
all_models.SystemOrProcess,
all_models.Role,
}
indexed_models = {model for model in all_models.all_models
if model_is_indexed(model)}
indexed_models -= excluded_models
for model in indexed_models:
# pylint: disable=protected-access
mapper_class = model._sa_class_manager.mapper.base_mapper.class_
query = model.query.options(
db.undefer_group(mapper_class.__name__ + '_complete'),
)
for query_chunk in generate_query_chunks(query):
for instance in query_chunk:
indexer.create_record(fts_record_for(instance), False)
db.session.commit()
reindex_snapshots()
def do_reindex():
"""
update the full text search index
"""
indexer = get_indexer()
indexer.delete_all_records(False)
# Remove model base classes and non searchable objects
excluded_models = {
all_models.Directive, all_models.Option, all_models.SystemOrProcess
}
indexed_models = {
model
for model in all_models.all_models if model_is_indexed(model)
}
indexed_models -= excluded_models
for model in indexed_models:
mapper_class = model._sa_class_manager.mapper.base_mapper.class_
query = model.query.options(
db.undefer_group(mapper_class.__name__ + '_complete'), )
for query_chunk in generate_query_chunks(query):
for instance in query_chunk:
indexer.create_record(fts_record_for(instance), False)
db.session.commit()
def do_counts(terms,
types=None,
contact_id=None,
extra_params={},
extra_columns={}):
# FIXME: ? This would make the query more efficient, but will also prune
# objects the user is allowed to read in other contexts.
# Remove types that the user can't read
# types = [type for type in types if permissions.is_allowed_read(type, None)]
indexer = get_indexer()
with benchmark("Counts"):
results = indexer.counts(terms,
types=types,
contact_id=contact_id,
extra_params=extra_params,
extra_columns=extra_columns)
results = [(r[2] if r[2] != "" else r[0], r[1]) for r in results]
return current_app.make_response((
json.dumps(
{'results': {
'selfLink': request.url,
'counts': dict(results)
}},
cls=GrcEncoder),
200,
[('Content-Type', 'application/json')],
))
def do_reindex():
"""Update the full text search index."""
indexer = get_indexer()
indexed_models = {
m.__name__: m
for m in all_models.all_models
if issubclass(m, mixin.Indexed) and m.REQUIRED_GLOBAL_REINDEX
}
people_query = db.session.query(all_models.Person.id,
all_models.Person.name,
all_models.Person.email)
indexer.cache["people_map"] = {
p.id: (p.name, p.email)
for p in people_query
}
indexer.cache["ac_role_map"] = dict(
db.session.query(
all_models.AccessControlRole.id,
all_models.AccessControlRole.name,
))
for model_name in sorted(indexed_models.keys()):
logger.info("Updating index for: %s", model_name)
with benchmark("Create records for %s" % model_name):
model = indexed_models[model_name]
for query_chunk in generate_query_chunks(db.session.query(
model.id)):
model.bulk_record_update_for([i.id for i in query_chunk])
db.session.commit()
logger.info("Updating index for: %s", "Snapshot")
with benchmark("Create records for %s" % "Snapshot"):
reindex_snapshots()
indexer.invalidate_cache()
start_compute_attributes("all_latest")
def do_reindex():
"""Update the full text search index."""
indexer = get_indexer()
indexed_models = {
m.__name__: m for m in all_models.all_models
if issubclass(m, mixin.Indexed) and m.REQUIRED_GLOBAL_REINDEX
}
people_query = db.session.query(all_models.Person.id,
all_models.Person.name,
all_models.Person.email)
indexer.cache["people_map"] = {p.id: (p.name, p.email) for p in people_query}
indexer.cache["ac_role_map"] = dict(db.session.query(
all_models.AccessControlRole.id,
all_models.AccessControlRole.name,
))
for model_name in sorted(indexed_models.keys()):
logger.info("Updating index for: %s", model_name)
with benchmark("Create records for %s" % model_name):
model = indexed_models[model_name]
ids = [obj.id for obj in model.query]
ids_count = len(ids)
handled_ids = 0
for ids_chunk in utils.list_chunks(ids):
handled_ids += len(ids_chunk)
logger.info("%s: %s / %s", model_name, handled_ids, ids_count)
model.bulk_record_update_for(ids_chunk)
db.session.commit()
indexer.invalidate_cache()
def __init__(self, ie_job, dry_run=True, csv_data=None):
self.user = getattr(g, '_current_user', None)
self.dry_run = dry_run
self.csv_data = csv_data or []
self.indexer = get_indexer()
self.comment_created_notif_type = all_models.NotificationType.query. \
filter_by(name="comment_created").one().id
super(ImportConverter, self).__init__(ie_job)
def get_delete_query_for(cls, ids):
"""Return delete class record query. If ids are empty, will return None."""
if not ids:
return
indexer = fulltext.get_indexer()
return indexer.record_type.__table__.delete().where(
indexer.record_type.type == cls.__name__).where(
indexer.record_type.key.in_(ids))
def update_index_for_objects(session, cache):
indexer = get_indexer()
for obj in cache.new:
indexer.create_record(fts_record_for(obj), commit=False)
for obj in cache.dirty:
indexer.update_record(fts_record_for(obj), commit=False)
for obj in cache.deleted:
indexer.delete_record(obj.id, obj.__class__.__name__, commit=False)
session.commit()
def get_insert_query_for(cls, ids):
"""Return insert class record query. It will return None, if it's empty."""
if not ids:
return
instances = cls.indexed_query().filter(cls.id.in_(ids))
indexer = fulltext.get_indexer()
rows = itertools.chain(*[indexer.records_generator(i) for i in instances])
values = list(rows)
if values:
return indexer.record_type.__table__.insert().values(values)
def __init__(self, **kwargs):
self.dry_run = kwargs.get("dry_run", True)
self.csv_data = kwargs.get("csv_data", [])
self.ids_by_type = kwargs.get("ids_by_type", [])
self.block_converters = []
self.new_objects = defaultdict(structures.CaseInsensitiveDict)
self.shared_state = {}
self.response_data = []
self.exportable = get_exportables()
self.indexer = get_indexer()
def test_simple_reindex(self):
"""Test for check simple reindex procedure."""
with ggrc_factories.single_commit():
for factory in self.INDEXED_MODEL_FACTORIES:
for _ in range(5):
factory()
indexer = fulltext.get_indexer()
count = indexer.record_type.query.count()
views.do_reindex()
self.assertEqual(count, indexer.record_type.query.count())
def __init__(self, **kwargs):
self.dry_run = kwargs.get("dry_run", True)
self.csv_data = kwargs.get("csv_data", [])
self.ids_by_type = kwargs.get("ids_by_type", [])
self.block_converters = []
self.new_objects = defaultdict(structures.CaseInsensitiveDict)
self.shared_state = {}
self.response_data = []
self.exportable = get_exportables()
self.indexer = get_indexer()
def test_simple_reindex(self):
"""Test for check simple reindex procedure."""
with ggrc_factories.single_commit():
for factory in self.INDEXED_MODEL_FACTORIES:
for _ in range(5):
factory()
indexer = fulltext.get_indexer()
count = indexer.record_type.query.count()
views.do_reindex()
self.assertEqual(count, indexer.record_type.query.count())
def get_person_data(rec, person):
"""Get list of Person properties for fulltext indexing
"""
indexer = get_indexer()
builder = indexer.get_builder(models.Person)
subprops = builder.build_person_subprops(person)
for key, val in subprops.items():
newrec = rec.copy()
newrec.update({"subproperty": key, "content": val})
yield newrec
def get_person_sort_subprop(rec, people):
"""Get a special subproperty for sorting
"""
indexer = get_indexer()
builder = indexer.get_builder(models.Person)
subprops = builder.build_list_sort_subprop(people)
for key, val in subprops.items():
newrec = rec.copy()
newrec.update({"subproperty": key, "content": val})
yield newrec
def get_person_data(rec, person):
"""Get list of Person properties for fulltext indexing
"""
indexer = get_indexer()
builder = indexer.get_builder(models.Person)
subprops = builder.build_person_subprops(person)
for key, val in subprops.items():
newrec = rec.copy()
newrec.update({"subproperty": key, "content": val})
yield newrec
def get_person_sort_subprop(rec, people):
"""Get a special subproperty for sorting
"""
indexer = get_indexer()
builder = indexer.get_builder(models.Person)
subprops = builder.build_list_sort_subprop(people)
for key, val in subprops.items():
newrec = rec.copy()
newrec.update({"subproperty": key, "content": val})
yield newrec
def get_delete_query_for(cls, ids):
"""Return delete class record query. If ids are empty, will return None."""
if not ids:
return
indexer = fulltext.get_indexer()
return indexer.record_type.__table__.delete().where(
indexer.record_type.type == cls.__name__
).where(
indexer.record_type.key.in_(ids)
)
def get_insert_query_for(cls, ids):
"""Return insert class record query. It will return None, if it's empty."""
if not ids:
return
instances = cls.indexed_query().filter(cls.id.in_(ids))
indexer = fulltext.get_indexer()
rows = itertools.chain(
*[indexer.records_generator(i) for i in instances])
values = list(rows)
if values:
return indexer.record_type.__table__.insert().values(values)
def register_fulltext_listeners():
"""Indexing initialization procedure"""
ggrc_indexer = fulltext.get_indexer()
for model in all_models.all_models:
for action in ACTIONS:
event.listen(model, action, _runner)
if not issubclass(model, Indexed):
continue
for sub_model in model.mro():
for rule in getattr(sub_model, "AUTO_REINDEX_RULES", []):
ggrc_indexer.indexer_rules[rule.model].append(rule.rule)
def __init__(self, ie_job, dry_run=True, csv_data=None, bulk_import=False):
self.user = login.get_current_user()
self.dry_run = dry_run
self.csv_data = csv_data or []
self._bulk_import = bulk_import
self.indexer = get_indexer()
self.comment_created_notif_type = all_models.NotificationType.query. \
filter_by(name="comment_created").one().id
super(ImportConverter, self).__init__(ie_job)
self.exportable.update(get_importables())
self.bulk_import = bulk_import
self.failed_slugs = []
def get_insert_query_for(cls, ids):
"""Return insert class record query. It will return None, if it's empty."""
if not ids:
return
instances = cls.indexed_query().filter(cls.id.in_(ids))
indexer = fulltext.get_indexer()
keys = inspect(indexer.record_type).c
records = (indexer.fts_record_for(i) for i in instances)
rows = itertools.chain(*[indexer.records_generator(i) for i in records])
values = [{c.name: getattr(r, a) for a, c in keys.items()} for r in rows]
if values:
return indexer.record_type.__table__.insert().values(values)
def get_insert_query_for(cls, ids):
"""Return insert class record query. It will return None, if it's empty."""
if not ids:
return
instances = cls.indexed_query().filter(cls.id.in_(ids))
indexer = fulltext.get_indexer()
keys = inspect(indexer.record_type).c
records = (indexer.fts_record_for(i) for i in instances)
rows = itertools.chain(*[indexer.records_generator(i) for i in records])
values = [{c.name: getattr(r, a) for a, c in keys.items()} for r in rows]
if values:
return indexer.record_type.__table__.insert().values(values)
def do_search(terms, list_for_type):
indexer = get_indexer()
results = indexer.search(terms)
for result in results:
id = result.key
model_type = result.type
entries_list = list_for_type(model_type)
entries_list.append({
'id': id,
'type': model_type,
'href': url_for(model_type, id=id),
})
def get_access_control_role_data(rec, ac_list_item):
"""Get list of access control data for fulltext indexing
"""
indexer = get_indexer()
builder = indexer.get_builder(models.Person)
ac_role_name, person_id = (builder.get_ac_role_person_id(ac_list_item))
if ac_role_name:
for key, val in builder.build_person_subprops({"id": person_id}).items():
newrec = rec.copy()
newrec.update({"property": ac_role_name,
"subproperty": key,
"content": val})
yield newrec
def _runner(mapper, content, target): # pylint:disable=unused-argument
"""Collect all reindex models in session"""
ggrc_indexer = fulltext.get_indexer()
db.session.reindex_set = getattr(db.session, "reindex_set", set())
getters = ggrc_indexer.indexer_rules.get(target.__class__.__name__) or []
for getter in getters:
to_index_list = getter(target)
if not isinstance(to_index_list, Iterable):
to_index_list = [to_index_list]
for to_index in to_index_list:
db.session.reindex_set.add(to_index)
if isinstance(target, Indexed):
db.session.reindex_set.add(target)
def get_access_control_role_data(rec, ac_list_item):
"""Get list of access control data for fulltext indexing
"""
indexer = get_indexer()
builder = indexer.get_builder(models.Person)
ac_role_name, person_id = (builder.get_ac_role_person_id(ac_list_item))
if ac_role_name:
for key, val in builder.build_person_subprops({"id": person_id}).items():
newrec = rec.copy()
newrec.update({"property": ac_role_name,
"subproperty": key,
"content": val})
yield newrec
def do_counts(terms, types=None):
indexer = get_indexer()
results = indexer.counts(terms, types=types)
return current_app.make_response((
json.dumps({ 'results': {
'selfLink': request.url,
'counts': dict(results)
}
}, cls=DateTimeEncoder),
200,
[('Content-Type', 'application/json')],
))
def get_access_control_sort_subprop(rec, access_control_list):
"""Get a special access_control_list subproperty for sorting
"""
builder = get_indexer().get_builder(models.Person)
collection = defaultdict(list)
for ac_list_item in access_control_list:
ac_role_name, person_id = builder.get_ac_role_person_id(ac_list_item)
if ac_role_name:
collection[ac_role_name].append({"id": person_id})
for ac_role_name, people in collection.iteritems():
for prop in get_person_sort_subprop({"property": ac_role_name}, people):
newrec = rec.copy()
newrec.update(prop)
yield newrec
def get_access_control_sort_subprop(rec, access_control_list):
"""Get a special access_control_list subproperty for sorting
"""
builder = get_indexer().get_builder(models.Person)
collection = defaultdict(list)
for ac_list_item in access_control_list:
ac_role_name, person_id = builder.get_ac_role_person_id(ac_list_item)
if ac_role_name:
collection[ac_role_name].append({"id": person_id})
for ac_role_name, people in collection.iteritems():
for prop in get_person_sort_subprop({"property": ac_role_name}, people):
newrec = rec.copy()
newrec.update(prop)
yield newrec
def insert_records(cls, ids):
"""Calculate and insert records into fulltext_record_properties table."""
instances = cls.indexed_query().filter(cls.id.in_(ids))
indexer = fulltext.get_indexer()
rows = itertools.chain(*[indexer.records_generator(i) for i in instances])
for vals_chunk in utils.iter_chunks(rows, chunk_size=10000):
query = """
INSERT INTO fulltext_record_properties (
`key`, type, tags, property, subproperty, content
) VALUES (:key, :type, :tags, :property, :subproperty, :content)
"""
values = list(vals_chunk)
if not values:
return
db.session.execute(query, values)
def test_simple_reindex(self):
"""Test for check simple reindex procedure."""
with ggrc_factories.single_commit():
for factory in self.INDEXED_MODEL_FACTORIES:
for _ in range(5):
factory()
indexer = fulltext.get_indexer()
count = indexer.record_type.query.count()
views.do_reindex()
# ACR roles are created in migration and aren't removed in setup
# Index for them will be created only after reindexing
reindexed_count = indexer.record_type.query.filter(
MysqlRecordProperty.type != "AccessControlRole"
).count()
self.assertEqual(count, reindexed_count)
def do_reindex(with_reindex_snapshots=False, delete=False):
"""Update the full text search index."""
indexer = fulltext.get_indexer()
indexed_models = {
m.__name__: m
for m in models.all_models.all_models
if issubclass(m, mixin.Indexed) and m.REQUIRED_GLOBAL_REINDEX
}
people_query = db.session.query(models.all_models.Person.id,
models.all_models.Person.name,
models.all_models.Person.email)
indexer.cache["people_map"] = {
p.id: (p.name, p.email)
for p in people_query
}
indexer.cache["ac_role_map"] = dict(
db.session.query(
models.all_models.AccessControlRole.id,
models.all_models.AccessControlRole.name,
))
_remove_dead_reindex_objects(indexed_models)
for model_name in sorted(indexed_models.keys()):
if delete:
with benchmark("Deleting records for %s" % model_name):
pass
logger.info("Updating index for: %s", model_name)
with benchmark("Create records for %s" % model_name):
model = indexed_models[model_name]
ids = [id_[0] for id_ in db.session.query(model.id)]
ids_count = len(ids)
handled_ids = 0
ids_chunks = ggrc_utils.list_chunks(ids,
chunk_size=REINDEX_CHUNK_SIZE)
for ids_chunk in ids_chunks:
handled_ids += len(ids_chunk)
logger.info("%s: %s / %s", model.__name__, handled_ids,
ids_count)
model.bulk_record_update_for(ids_chunk)
db.session.plain_commit()
if with_reindex_snapshots:
logger.info("Updating index for: %s", "Snapshot")
with benchmark("Create records for %s" % "Snapshot"):
snapshot_indexer.reindex()
indexer.invalidate_cache()
def _log_event(cls, instance, action="POST"):
indexer = get_indexer()
db.session.flush()
user = cls._get_user()
revision = models.Revision(
instance, user.id, 'created', instance.log_json())
event = models.Event(
modified_by=user,
action=action,
resource_id=instance.id,
resource_type=instance.type,
context=instance.context,
revisions=[revision],
)
db.session.add(revision)
db.session.add(event)
indexer.update_record(instance, commit=False)
def test_simple_reindex(self):
"""Test for check simple reindex procedure."""
with ggrc_factories.single_commit():
for factory in self.INDEXED_MODEL_FACTORIES:
for _ in range(5):
factory()
indexer = fulltext.get_indexer()
count = indexer.record_type.query.count()
count = indexer.record_type.query.delete()
self.client.get("/login")
self.client.post("/admin/full_reindex")
# ACR roles are created in migration and aren't removed in setup
# Index for them will be created only after reindexing
reindexed_count = indexer.record_type.query.filter(
MysqlRecordProperty.type != "AccessControlRole"
).count()
self.assertEqual(count, reindexed_count)
def test_reindex(self):
"""Test reindex of big portion of objects."""
obj_count = listeners.ReindexSet.CHUNK_SIZE + 1
with factories.single_commit():
audit = factories.AuditFactory()
for _ in range(obj_count):
factories.AssessmentFactory(audit=audit)
indexer = fulltext.get_indexer()
archived_index = indexer.record_type.query.filter(
mysql.MysqlRecordProperty.type == "Assessment",
mysql.MysqlRecordProperty.property == "archived",
mysql.MysqlRecordProperty.content == "True"
)
self.assertEqual(archived_index.count(), 0)
# Reindex of Audit.archived lead to reindex of all related assessments
self.api.put(audit, {"archived": True})
# Check that all Assessment.archived were properly reindexed
self.assertEqual(archived_index.count(), obj_count)
def do_search(
terms, list_for_type, types=None, permission_type='read',
permission_model=None):
indexer = get_indexer()
results = indexer.search(
terms, types=types, permission_type=permission_type,
permission_model=permission_model)
seen_results = {}
for result in results:
id = result.key
model_type = result.type
result_pair = (model_type, id)
if result_pair not in seen_results:
seen_results[result_pair] = True
entries_list = list_for_type(model_type)
entries_list.append({
'id': id,
'type': model_type,
'href': url_for(model_type, id=id),
})
