def test_index_on_commit(self, factory, obj_count):
"""Test count number of queries on eager query procedure."""
model = factory._meta.model # pylint: disable=protected-access
with ggrc_factories.single_commit():
# pylint: disable=expression-not-assigned
{factory() for _ in range(obj_count)}
with QueryCounter() as counter:
self._post([{
"object_name": model.__name__,
"filters": {"expression": {}},
"limit": [0, obj_count],
"order_by": [{"name": "updated_at", "desc": True}]
}])
self.assertEqual(
counter.get,
self.QUERY_API_LIMIT[model.__name__],
"Eager query of {model} has too much queries: "
"{counted} not equal to {expected} for query "
"{obj_count} instances".format(
model=model,
expected=self.QUERY_API_LIMIT[model.__name__],
counted=counter.get,
obj_count=obj_count,
)
)
def test_create_mapping_cache(self, count):
"""Test creation of mapping cache for export."""
regulations = [factories.RegulationFactory() for _ in range(count)]
markets = [factories.MarketFactory() for _ in range(count)]
controls = [factories.ControlFactory() for _ in range(count)]
expected_cache = defaultdict(lambda: defaultdict(list))
for i in range(count):
for j in range(i):
factories.RelationshipFactory(
source=regulations[j] if i % 2 == 0 else markets[i],
destination=regulations[j] if i % 2 == 1 else markets[i],
)
factories.RelationshipFactory(
source=regulations[j] if i % 2 == 0 else controls[i],
destination=regulations[j] if i % 2 == 1 else controls[i],
)
expected_cache[regulations[j].id]["Control"].append(
controls[i].slug)
expected_cache[regulations[j].id]["Market"].append(
markets[i].slug)
block = base_block.BlockConverter(mock.MagicMock())
block.object_class = models.Regulation
block.object_ids = [r.id for r in regulations]
with QueryCounter() as counter:
cache = block._create_mapping_cache()
self.assertEqual(
self.dd_to_dict(cache),
self.dd_to_dict(expected_cache),
)
self.assertLess(counter.get, self.QUERY_LIMIT)
def test_queries_for_dashboard(self):
"""Test query count for dashboard page."""
with QueryCounter() as counter:
res = self.client.get("/dashboard")
self.assertEqual(res.status_code, 200)
self.assertLess(counter.get, self.DEFAULT_LIMIT,
"Query count for dashboard")
def test_queries_for_permissions(self):
"""Test query count for permissions loading."""
with QueryCounter() as counter:
res = self.client.get("/permissions")
self.assertEqual(res.status_code, 200)
self.assertLess(counter.get, self.DEFAULT_LIMIT,
"Query count for permissions")
def test_queries_per_object_page(self):
"""Import comprehensive_sheet1 and count db requests per collection get.
Query count should be <LIMIT for all model types.
"""
errors = set()
with QueryCounter() as counter:
for view in all_object_views():
try:
model = view.model_class
if model not in self.MODELS:
continue
instance = model.query.first()
if instance is None or getattr(instance, "id",
None) is None:
continue
counter.queries = []
res = self.client.get("/{}/{}".format(
view.url, instance.id))
self.assertEqual(res.status_code, 200)
self.assertLess(
counter.get, self.LIMIT,
"Query count for object {} exceeded: {}/{}".format(
model.__name__, counter.get, self.LIMIT))
except AssertionError as e:
errors.add(e.message)
self.assertEqual(errors, set())
def test_index_on_commit(self, factory, obj_count):
"""Test count number of queries on reindex procedure."""
model = factory._meta.model # pylint: disable=protected-access
with ggrc_factories.single_commit():
obj_to_index = {factory() for _ in range(obj_count)}
db.session.expire_all()
db.session.reindex_set = model.query.filter(
model.id.in_([i.id for i in obj_to_index]),
).options(
orm.Load(model).load_only("id")
).all()
with QueryCounter() as counter:
mysql.update_indexer(db.session)
self.assertLessEqual(
counter.get,
self.INDEX_QUERY_LIMIT[model.__name__],
"Index {model} has too much queries: "
"{counted} greater than {expected} for generate "
"{obj_count} instances".format(
model=model,
expected=self.INDEX_QUERY_LIMIT[model.__name__],
counted=counter.get,
obj_count=obj_count,
)
)
def test_queries_per_api_call(self, model):
"""Import comprehensive_sheet1 and count db requests per collection get.
Query count should be <LIMIT for all model types.
"""
with QueryCounter() as counter:
counter.queries = []
self.generator.api.get_query(model, "")
if counter.get > self.LIMIT:
print collections.Counter(counter.queries).most_common(1)
self.assertLess(
counter.get, self.LIMIT,
"Query count for object {} exceeded: {}/{}".format(
model.__name__, counter.get, self.LIMIT))
def test_create_mapping_cache(self, count):
"""Test creation of mapping cache for export."""
with factories.single_commit():
regulations = [factories.RegulationFactory() for _ in range(count)]
requirements = [
factories.RequirementFactory() for _ in range(count)
]
controls = [factories.ControlFactory() for _ in range(count)]
expected_cache = defaultdict(lambda: defaultdict(list))
for i in range(count):
for j in range(i):
factories.RelationshipFactory(
source=regulations[j] if i %
2 == 0 else requirements[i],
destination=regulations[j] if i %
2 == 1 else requirements[i],
)
with mock.patch(
'ggrc.models.relationship.is_external_app_user',
return_value=True):
factories.RelationshipFactory(
source=regulations[j] if i %
2 == 0 else controls[i],
destination=regulations[j] if i %
2 == 1 else controls[i],
is_external=True,
)
expected_cache[regulations[j].id]["Control"].append(
controls[i].slug)
expected_cache[regulations[j].id]["Requirement"].append(
requirements[i].slug)
block = base_block.ExportBlockConverter(
mock.MagicMock(),
object_class=models.Regulation,
fields="all",
object_ids=[r.id for r in regulations],
class_name=models.Regulation.__name__,
)
with QueryCounter() as counter:
cache = block._create_mapping_cache()
self.assertEqual(
self.dd_to_dict(cache),
self.dd_to_dict(expected_cache),
)
self.assertLess(counter.get, self.QUERY_LIMIT)
def test_queries_per_api_call(self, model):
"""Import comprehensive_sheet1 and count db requests per {0.__name__}
collection get
Query count should be <LIMIT for all model types.
"""
limit = self.LIMIT_DICT["LIST"].get(model, self.DEFAULT_LIMIT)
with QueryCounter() as counter:
counter.queries = []
self.generator.api.get_query(model, "")
if counter.get > limit:
print collections.Counter(counter.queries).most_common(1)
self.assertLess(counter.get, limit,
"Query count for object {} exceeded: {}/{}".format(
model.__name__, counter.get, limit)
)
def test_queries_per_object_page(self, view):
"""Import comprehensive_sheet1 and count db requests per collection get.
Query count should be <LIMIT for all model types.
"""
with QueryCounter() as counter:
model = view.model_class
if model not in self.MODELS:
return
instance = model.query.first()
if instance is None or getattr(instance, "id", None) is None:
return
counter.queries = []
res = self.client.get("/{}/{}".format(view.url, instance.id))
self.assertEqual(res.status_code, 200)
self.assertLessEqual(
counter.get, self.LIMIT,
"Query count for object {} exceeded: {}/{}".format(
model.__name__, counter.get, self.LIMIT))
def test_queries_per_api_call(self):
"""Import comprehensive_sheet1 and count db requests per collection get.
Query count should be <LIMIT for all model types.
"""
errors = set()
with QueryCounter() as counter:
for model in self.MODELS:
try:
counter.queries = []
self.generator.api.get_query(model, "")
if counter.get > self.LIMIT:
print collections.Counter(counter.queries).most_common(1)
self.assertLess(counter.get, self.LIMIT,
"Query count for object {} exceeded: {}/{}".format(
model.__name__, counter.get, self.LIMIT)
)
except AssertionError as error:
errors.add(error.message)
self.assertEqual(errors, set())
def test_export_query_count(self):
"""Test for a constant number of queries for snapshot export
The number of database queries should not be linked to the amount of data
that is being exported.
"""
# pylint: disable=too-many-locals
self._create_cads("product")
self.import_file("product_snapshot_data_multiple.csv")
product = models.Product.query.all()
with factories.single_commit():
audit = factories.AuditFactory()
snapshots = self._create_snapshots(audit, product)
count = len(snapshots)
assessments = [factories.AssessmentFactory() for _ in range(count)]
issues = [factories.IssueFactory() for _ in range(count)]
for snapshot, assessment, issue in zip(snapshots, assessments,
issues):
factories.RelationshipFactory(source=snapshot,
destination=assessment)
factories.RelationshipFactory(source=issue,
destination=snapshot)
with QueryCounter() as counter:
search_request = [{
"object_name": "Snapshot",
"filters": {
"expression": {
"left": {
"left": "child_type",
"op": {
"name": "="
},
"right": "Product",
},
"op": {
"name": "AND"
},
"right": {
"left": "Code",
"op": {
"name": "="
},
"right": "Product 1",
},
},
},
"fields": ["mappings"],
}]
self.assertEqual(
len(
self.export_parsed_csv(search_request)
["Product Snapshot"]),
1,
)
single_query_count = counter.get
with QueryCounter() as counter:
search_request = [{
"object_name": "Snapshot",
"filters": {
"expression": {
"left": "child_type",
"op": {
"name": "="
},
"right": "Product",
},
},
"fields": ["mappings"],
}]
self.assertEqual(
len(
self.export_parsed_csv(search_request)
["Product Snapshot"]),
5,
)
multiple_query_count = counter.get
self.assertEqual(multiple_query_count, single_query_count)
def test_export_query_count(self):
"""Test for a constant number of queries for snapshot export
The number of database queries should not be linked to the amount of data
that is being exported.
"""
self._create_cads("control")
self.import_file("control_snapshot_data_multiple.csv")
# Duplicate import because we have a bug in logging revisions and this
# makes sure that the fixture created properly.
self.import_file("control_snapshot_data_multiple.csv")
controls = models.Control.query.all()
with factories.single_commit():
audit = factories.AuditFactory()
snapshots = self._create_snapshots(audit, controls)
count = len(snapshots)
assessments = [factories.AssessmentFactory() for _ in range(count)]
issues = [factories.IssueFactory() for _ in range(count)]
for snapshot, assessment, issue in zip(snapshots, assessments, issues):
factories.RelationshipFactory(source=snapshot, destination=assessment)
factories.RelationshipFactory(source=issue, destination=snapshot)
with QueryCounter() as counter:
search_request = [{
"object_name": "Snapshot",
"filters": {
"expression": {
"left": {
"left": "child_type",
"op": {"name": "="},
"right": "Control",
},
"op": {"name": "AND"},
"right": {
"left": "Code",
"op": {"name": "="},
"right": "Control 1",
},
},
},
"fields": ["mappings"],
}]
self.assertEqual(
len(self.export_parsed_csv(search_request)["Control Snapshot"]),
1,
)
single_query_count = counter.get
with QueryCounter() as counter:
search_request = [{
"object_name": "Snapshot",
"filters": {
"expression": {
"left": "child_type",
"op": {"name": "="},
"right": "Control",
},
},
"fields": ["mappings"],
}]
self.assertEqual(
len(self.export_parsed_csv(search_request)["Control Snapshot"]),
5,
)
multiple_query_count = counter.get
self.assertEqual(multiple_query_count, single_query_count)
def test_export_query_count(self):
"""Test for a constant number of queries for snapshot export
The number of database queries should not be linked to the amount of data
that is being exported.
"""
# pylint: disable=too-many-locals
self._create_cads("product")
user_details = ("Administrator", "Creator", "Editor", "Reader")
with factories.single_commit():
roles = {r.name: r for r in all_models.Role.query.all()}
for user in user_details:
person = factories.PersonFactory(
name=user, email="{}@example.com".format(user))
rbac_factories.UserRoleFactory(role=roles[user], person=person)
factories.ProductFactory(title="Product {}".format(user))
product = models.Product.query.all()
audit = factories.AuditFactory()
snapshots = self._create_snapshots(audit, product)
count = len(snapshots)
assessments = [factories.AssessmentFactory() for _ in range(count)]
issues = [factories.IssueFactory() for _ in range(count)]
for snapshot, assessment, issue in zip(snapshots, assessments,
issues):
factories.RelationshipFactory(source=snapshot,
destination=assessment)
factories.RelationshipFactory(source=issue,
destination=snapshot)
with QueryCounter() as counter:
search_request = [{
"object_name": "Snapshot",
"filters": {
"expression": {
"left": {
"left": "child_type",
"op": {
"name": "="
},
"right": "Product",
},
"op": {
"name": "AND"
},
"right": {
"left": "title",
"op": {
"name": "="
},
"right": "Product Editor",
},
},
},
"fields": ["mappings"],
}]
self.assertEqual(
len(
self.export_parsed_csv(search_request)
["Product Snapshot"]),
1,
)
single_query_count = counter.get
with QueryCounter() as counter:
search_request = [{
"object_name": "Snapshot",
"filters": {
"expression": {
"left": "child_type",
"op": {
"name": "="
},
"right": "Product",
},
},
"fields": ["mappings"],
}]
self.assertEqual(
len(
self.export_parsed_csv(search_request)
["Product Snapshot"]),
4,
)
multiple_query_count = counter.get
self.assertEqual(multiple_query_count, single_query_count)
