def test_index_on_commit(self, factory, obj_count): """Test count number of queries on eager query procedure.""" model = factory._meta.model # pylint: disable=protected-access with ggrc_factories.single_commit(): # pylint: disable=expression-not-assigned {factory() for _ in range(obj_count)} with QueryCounter() as counter: self._post([{ "object_name": model.__name__, "filters": {"expression": {}}, "limit": [0, obj_count], "order_by": [{"name": "updated_at", "desc": True}] }]) self.assertEqual( counter.get, self.QUERY_API_LIMIT[model.__name__], "Eager query of {model} has too much queries: " "{counted} not equal to {expected} for query " "{obj_count} instances".format( model=model, expected=self.QUERY_API_LIMIT[model.__name__], counted=counter.get, obj_count=obj_count, ) )
def test_create_mapping_cache(self, count): """Test creation of mapping cache for export.""" regulations = [factories.RegulationFactory() for _ in range(count)] markets = [factories.MarketFactory() for _ in range(count)] controls = [factories.ControlFactory() for _ in range(count)] expected_cache = defaultdict(lambda: defaultdict(list)) for i in range(count): for j in range(i): factories.RelationshipFactory( source=regulations[j] if i % 2 == 0 else markets[i], destination=regulations[j] if i % 2 == 1 else markets[i], ) factories.RelationshipFactory( source=regulations[j] if i % 2 == 0 else controls[i], destination=regulations[j] if i % 2 == 1 else controls[i], ) expected_cache[regulations[j].id]["Control"].append( controls[i].slug) expected_cache[regulations[j].id]["Market"].append( markets[i].slug) block = base_block.BlockConverter(mock.MagicMock()) block.object_class = models.Regulation block.object_ids = [r.id for r in regulations] with QueryCounter() as counter: cache = block._create_mapping_cache() self.assertEqual( self.dd_to_dict(cache), self.dd_to_dict(expected_cache), ) self.assertLess(counter.get, self.QUERY_LIMIT)
def test_queries_for_dashboard(self): """Test query count for dashboard page.""" with QueryCounter() as counter: res = self.client.get("/dashboard") self.assertEqual(res.status_code, 200) self.assertLess(counter.get, self.DEFAULT_LIMIT, "Query count for dashboard")
def test_queries_for_permissions(self): """Test query count for permissions loading.""" with QueryCounter() as counter: res = self.client.get("/permissions") self.assertEqual(res.status_code, 200) self.assertLess(counter.get, self.DEFAULT_LIMIT, "Query count for permissions")
def test_queries_per_object_page(self): """Import comprehensive_sheet1 and count db requests per collection get. Query count should be <LIMIT for all model types. """ errors = set() with QueryCounter() as counter: for view in all_object_views(): try: model = view.model_class if model not in self.MODELS: continue instance = model.query.first() if instance is None or getattr(instance, "id", None) is None: continue counter.queries = [] res = self.client.get("/{}/{}".format( view.url, instance.id)) self.assertEqual(res.status_code, 200) self.assertLess( counter.get, self.LIMIT, "Query count for object {} exceeded: {}/{}".format( model.__name__, counter.get, self.LIMIT)) except AssertionError as e: errors.add(e.message) self.assertEqual(errors, set())
def test_index_on_commit(self, factory, obj_count): """Test count number of queries on reindex procedure.""" model = factory._meta.model # pylint: disable=protected-access with ggrc_factories.single_commit(): obj_to_index = {factory() for _ in range(obj_count)} db.session.expire_all() db.session.reindex_set = model.query.filter( model.id.in_([i.id for i in obj_to_index]), ).options( orm.Load(model).load_only("id") ).all() with QueryCounter() as counter: mysql.update_indexer(db.session) self.assertLessEqual( counter.get, self.INDEX_QUERY_LIMIT[model.__name__], "Index {model} has too much queries: " "{counted} greater than {expected} for generate " "{obj_count} instances".format( model=model, expected=self.INDEX_QUERY_LIMIT[model.__name__], counted=counter.get, obj_count=obj_count, ) )
def test_queries_per_api_call(self, model): """Import comprehensive_sheet1 and count db requests per collection get. Query count should be <LIMIT for all model types. """ with QueryCounter() as counter: counter.queries = [] self.generator.api.get_query(model, "") if counter.get > self.LIMIT: print collections.Counter(counter.queries).most_common(1) self.assertLess( counter.get, self.LIMIT, "Query count for object {} exceeded: {}/{}".format( model.__name__, counter.get, self.LIMIT))
def test_create_mapping_cache(self, count): """Test creation of mapping cache for export.""" with factories.single_commit(): regulations = [factories.RegulationFactory() for _ in range(count)] requirements = [ factories.RequirementFactory() for _ in range(count) ] controls = [factories.ControlFactory() for _ in range(count)] expected_cache = defaultdict(lambda: defaultdict(list)) for i in range(count): for j in range(i): factories.RelationshipFactory( source=regulations[j] if i % 2 == 0 else requirements[i], destination=regulations[j] if i % 2 == 1 else requirements[i], ) with mock.patch( 'ggrc.models.relationship.is_external_app_user', return_value=True): factories.RelationshipFactory( source=regulations[j] if i % 2 == 0 else controls[i], destination=regulations[j] if i % 2 == 1 else controls[i], is_external=True, ) expected_cache[regulations[j].id]["Control"].append( controls[i].slug) expected_cache[regulations[j].id]["Requirement"].append( requirements[i].slug) block = base_block.ExportBlockConverter( mock.MagicMock(), object_class=models.Regulation, fields="all", object_ids=[r.id for r in regulations], class_name=models.Regulation.__name__, ) with QueryCounter() as counter: cache = block._create_mapping_cache() self.assertEqual( self.dd_to_dict(cache), self.dd_to_dict(expected_cache), ) self.assertLess(counter.get, self.QUERY_LIMIT)
def test_queries_per_api_call(self, model): """Import comprehensive_sheet1 and count db requests per {0.__name__} collection get Query count should be <LIMIT for all model types. """ limit = self.LIMIT_DICT["LIST"].get(model, self.DEFAULT_LIMIT) with QueryCounter() as counter: counter.queries = [] self.generator.api.get_query(model, "") if counter.get > limit: print collections.Counter(counter.queries).most_common(1) self.assertLess(counter.get, limit, "Query count for object {} exceeded: {}/{}".format( model.__name__, counter.get, limit) )
def test_queries_per_object_page(self, view): """Import comprehensive_sheet1 and count db requests per collection get. Query count should be <LIMIT for all model types. """ with QueryCounter() as counter: model = view.model_class if model not in self.MODELS: return instance = model.query.first() if instance is None or getattr(instance, "id", None) is None: return counter.queries = [] res = self.client.get("/{}/{}".format(view.url, instance.id)) self.assertEqual(res.status_code, 200) self.assertLessEqual( counter.get, self.LIMIT, "Query count for object {} exceeded: {}/{}".format( model.__name__, counter.get, self.LIMIT))
def test_queries_per_api_call(self): """Import comprehensive_sheet1 and count db requests per collection get. Query count should be <LIMIT for all model types. """ errors = set() with QueryCounter() as counter: for model in self.MODELS: try: counter.queries = [] self.generator.api.get_query(model, "") if counter.get > self.LIMIT: print collections.Counter(counter.queries).most_common(1) self.assertLess(counter.get, self.LIMIT, "Query count for object {} exceeded: {}/{}".format( model.__name__, counter.get, self.LIMIT) ) except AssertionError as error: errors.add(error.message) self.assertEqual(errors, set())
def test_export_query_count(self): """Test for a constant number of queries for snapshot export The number of database queries should not be linked to the amount of data that is being exported. """ # pylint: disable=too-many-locals self._create_cads("product") self.import_file("product_snapshot_data_multiple.csv") product = models.Product.query.all() with factories.single_commit(): audit = factories.AuditFactory() snapshots = self._create_snapshots(audit, product) count = len(snapshots) assessments = [factories.AssessmentFactory() for _ in range(count)] issues = [factories.IssueFactory() for _ in range(count)] for snapshot, assessment, issue in zip(snapshots, assessments, issues): factories.RelationshipFactory(source=snapshot, destination=assessment) factories.RelationshipFactory(source=issue, destination=snapshot) with QueryCounter() as counter: search_request = [{ "object_name": "Snapshot", "filters": { "expression": { "left": { "left": "child_type", "op": { "name": "=" }, "right": "Product", }, "op": { "name": "AND" }, "right": { "left": "Code", "op": { "name": "=" }, "right": "Product 1", }, }, }, "fields": ["mappings"], }] self.assertEqual( len( self.export_parsed_csv(search_request) ["Product Snapshot"]), 1, ) single_query_count = counter.get with QueryCounter() as counter: search_request = [{ "object_name": "Snapshot", "filters": { "expression": { "left": "child_type", "op": { "name": "=" }, "right": "Product", }, }, "fields": ["mappings"], }] self.assertEqual( len( self.export_parsed_csv(search_request) ["Product Snapshot"]), 5, ) multiple_query_count = counter.get self.assertEqual(multiple_query_count, single_query_count)
def test_export_query_count(self): """Test for a constant number of queries for snapshot export The number of database queries should not be linked to the amount of data that is being exported. """ self._create_cads("control") self.import_file("control_snapshot_data_multiple.csv") # Duplicate import because we have a bug in logging revisions and this # makes sure that the fixture created properly. self.import_file("control_snapshot_data_multiple.csv") controls = models.Control.query.all() with factories.single_commit(): audit = factories.AuditFactory() snapshots = self._create_snapshots(audit, controls) count = len(snapshots) assessments = [factories.AssessmentFactory() for _ in range(count)] issues = [factories.IssueFactory() for _ in range(count)] for snapshot, assessment, issue in zip(snapshots, assessments, issues): factories.RelationshipFactory(source=snapshot, destination=assessment) factories.RelationshipFactory(source=issue, destination=snapshot) with QueryCounter() as counter: search_request = [{ "object_name": "Snapshot", "filters": { "expression": { "left": { "left": "child_type", "op": {"name": "="}, "right": "Control", }, "op": {"name": "AND"}, "right": { "left": "Code", "op": {"name": "="}, "right": "Control 1", }, }, }, "fields": ["mappings"], }] self.assertEqual( len(self.export_parsed_csv(search_request)["Control Snapshot"]), 1, ) single_query_count = counter.get with QueryCounter() as counter: search_request = [{ "object_name": "Snapshot", "filters": { "expression": { "left": "child_type", "op": {"name": "="}, "right": "Control", }, }, "fields": ["mappings"], }] self.assertEqual( len(self.export_parsed_csv(search_request)["Control Snapshot"]), 5, ) multiple_query_count = counter.get self.assertEqual(multiple_query_count, single_query_count)
def test_export_query_count(self): """Test for a constant number of queries for snapshot export The number of database queries should not be linked to the amount of data that is being exported. """ # pylint: disable=too-many-locals self._create_cads("product") user_details = ("Administrator", "Creator", "Editor", "Reader") with factories.single_commit(): roles = {r.name: r for r in all_models.Role.query.all()} for user in user_details: person = factories.PersonFactory( name=user, email="{}@example.com".format(user)) rbac_factories.UserRoleFactory(role=roles[user], person=person) factories.ProductFactory(title="Product {}".format(user)) product = models.Product.query.all() audit = factories.AuditFactory() snapshots = self._create_snapshots(audit, product) count = len(snapshots) assessments = [factories.AssessmentFactory() for _ in range(count)] issues = [factories.IssueFactory() for _ in range(count)] for snapshot, assessment, issue in zip(snapshots, assessments, issues): factories.RelationshipFactory(source=snapshot, destination=assessment) factories.RelationshipFactory(source=issue, destination=snapshot) with QueryCounter() as counter: search_request = [{ "object_name": "Snapshot", "filters": { "expression": { "left": { "left": "child_type", "op": { "name": "=" }, "right": "Product", }, "op": { "name": "AND" }, "right": { "left": "title", "op": { "name": "=" }, "right": "Product Editor", }, }, }, "fields": ["mappings"], }] self.assertEqual( len( self.export_parsed_csv(search_request) ["Product Snapshot"]), 1, ) single_query_count = counter.get with QueryCounter() as counter: search_request = [{ "object_name": "Snapshot", "filters": { "expression": { "left": "child_type", "op": { "name": "=" }, "right": "Product", }, }, "fields": ["mappings"], }] self.assertEqual( len( self.export_parsed_csv(search_request) ["Product Snapshot"]), 4, ) multiple_query_count = counter.get self.assertEqual(multiple_query_count, single_query_count)