def test_index_by_acr(self):
"""Test index by ACR."""
role_name = "Test name"
with factories.single_commit():
acr = factories.AccessControlRoleFactory(
name=role_name,
object_type="Control"
)
person = factories.PersonFactory(email="*****@*****.**", name='test')
control = factories.ControlFactory()
factories.AccessControlList(ac_role=acr, person=person, object=control)
revision = all_models.Revision.query.filter(
all_models.Revision.resource_id == control.id,
all_models.Revision.resource_type == control.type,
).one()
revision.content = control.log_json()
db.session.add(revision)
with factories.single_commit():
snapshot = factories.SnapshotFactory(
child_id=control.id,
child_type=control.type,
revision=revision)
db.session.expire_all()
do_reindex()
self.assert_indexed_fields(snapshot, role_name, {
"{}-email".format(person.id): person.email,
"{}-name".format(person.id): person.name,
"{}-user_name".format(person.id): person.user_name,
"__sort__": person.user_name,
})
def tests_ca_export_filters(self):
"""Test filtering on custom attribute values."""
filename = "custom_attribute_tests.csv"
self.import_file(filename)
# TODO: there is a bug that imports do not index all custom attributes.
# After fixing this bug remove the following two lines.
from ggrc import views
views.do_reindex()
data = [{
"object_name": "Product",
"filters": {
"expression": {
"left": "normal text",
"op": {"name": "="},
"right": "some text",
},
},
"fields": "all",
}]
response = self.client.post("/_service/export_csv", data=dumps(data),
headers=self.headers)
self.assert200(response)
self.assertIn("some text", response.data)
def test_index_deleted_acr(self):
"""Test index by removed ACR."""
role_name = "Test name"
with factories.single_commit():
acr = factories.AccessControlRoleFactory(name=role_name,
object_type="Control")
person = factories.PersonFactory(email="*****@*****.**",
name='test')
control = factories.ControlFactory()
factories.AccessControlList(ac_role=acr,
person=person,
object=control)
revision = all_models.Revision.query.filter(
all_models.Revision.resource_id == control.id,
all_models.Revision.resource_type == control.type,
).one()
revision.content = control.log_json()
db.session.add(revision)
with factories.single_commit():
snapshot = factories.SnapshotFactory(child_id=control.id,
child_type=control.type,
revision=revision)
db.session.expire_all()
db.session.delete(acr)
db.session.commit()
do_reindex()
all_found_records = dict(
Record.query.filter(Record.key == snapshot.id,
Record.type == snapshot.type,
Record.property == role_name.lower()).values(
"subproperty", "content"))
self.assertFalse(all_found_records)
def test_checkbox_fulltext(self, value, search_value):
"""Test filter by checkbox value."""
title = "checkbox"
checkbox_type = all_models.CustomAttributeDefinition.ValidTypes.CHECKBOX
with factories.single_commit():
market = factories.MarketFactory()
cad = factories.CustomAttributeDefinitionFactory(
title=title,
definition_type="market",
attribute_type=checkbox_type)
factories.CustomAttributeValueFactory(
custom_attribute=cad,
attributable=market,
attribute_value=value)
views.do_reindex()
contents = [
i.content
for i in mysql.MysqlRecordProperty.query.filter(
mysql.MysqlRecordProperty.property == title,
mysql.MysqlRecordProperty.type == market.type,
mysql.MysqlRecordProperty.key == market.id,
)
]
self.assertEqual([search_value], contents)
def tests_ca_export_filters(self):
"""Test filtering on custom attribute values."""
filename = "custom_attribute_tests.csv"
self.import_file(filename)
# TODO: there is a bug that imports do not index all custom attributes.
# After fixing this bug remove the following two lines.
from ggrc import views
views.do_reindex()
data = [{
"object_name": "Product",
"filters": {
"expression": {
"left": "normal text",
"op": {
"name": "="
},
"right": "some text",
},
},
"fields": "all",
}]
response = self.client.post("/_service/export_csv",
data=dumps(data),
headers=self.headers)
self.assert200(response)
self.assertIn("some text", response.data)
def test_filter_by_checkbox_cad(self, value, search_value):
"""Test index by Checkdoxed cad."""
checkbox_type = all_models.CustomAttributeDefinition.ValidTypes.CHECKBOX
cad_title = "Checkbox"
with factories.single_commit():
cad = factories.CustomAttributeDefinitionFactory(
attribute_type=checkbox_type,
definition_type="control",
title=cad_title,
)
control = factories.ControlFactory()
factories.CustomAttributeValueFactory(
custom_attribute=cad,
attributable=control,
attribute_value=value,
)
revision = all_models.Revision.query.filter(
all_models.Revision.resource_id == control.id,
all_models.Revision.resource_type == control.type,
).first()
revision.content = control.log_json()
db.session.add(revision)
with factories.single_commit():
snapshot = factories.SnapshotFactory(
child_id=control.id,
child_type=control.type,
revision=revision)
db.session.expire_all()
do_reindex()
self.assert_indexed_fields(snapshot, cad_title, {"": search_value})
def test_index_deleted_acr(self):
"""Test index by removed ACR."""
role_name = "Test name"
with factories.single_commit():
acr = factories.AccessControlRoleFactory(
name=role_name,
object_type="Control"
)
person = factories.PersonFactory(email="*****@*****.**", name='test')
control = factories.ControlFactory()
factories.AccessControlList(ac_role=acr, person=person, object=control)
revision = all_models.Revision.query.filter(
all_models.Revision.resource_id == control.id,
all_models.Revision.resource_type == control.type,
).one()
revision.content = control.log_json()
db.session.add(revision)
with factories.single_commit():
snapshot = factories.SnapshotFactory(
child_id=control.id,
child_type=control.type,
revision=revision)
db.session.expire_all()
db.session.delete(acr)
db.session.commit()
do_reindex()
all_found_records = dict(Record.query.filter(
Record.key == snapshot.id,
Record.type == snapshot.type,
Record.property == role_name.lower()
).values("subproperty", "content"))
self.assertFalse(all_found_records)
def test_full_reindex(self):
"""Test full reindex of all snapshots"""
self._import_file("snapshotter_create.csv")
program = db.session.query(models.Program).filter(
models.Program.slug == "Prog-13211"
).one()
self.create_audit(program)
audit = db.session.query(models.Audit).filter(
models.Audit.title.like("%Snapshotable audit%")).first()
snapshots = db.session.query(models.Snapshot).all()
records = get_records(audit, snapshots)
self.assertEqual(records.count(), 57)
delete_records({s.id for s in snapshots})
records = get_records(audit, snapshots)
self.assertEqual(records.count(), 0)
do_reindex()
records = get_records(audit, snapshots)
self.assertEqual(records.count(), 57)
def test_search_no_acl_in_content(self, field, role_name):
"""Test search older revisions without access_control_list."""
with factories.single_commit():
factories.AccessControlRoleFactory(name=role_name,
object_type="Control")
person = factories.PersonFactory(email="{}@example.com".format(field),
name=field)
control = factories.ControlFactory()
revision = all_models.Revision.query.filter(
all_models.Revision.resource_id == control.id,
all_models.Revision.resource_type == control.type,
).one()
with factories.single_commit():
snapshot = factories.SnapshotFactory(
child_id=control.id,
child_type=control.type,
revision=revision)
revision.content = revision.content.copy()
revision.content.pop("access_control_list")
revision.content[field] = {"id": person.id}
db.session.add(revision)
do_reindex()
self.assert_indexed_fields(snapshot, role_name, {
"{}-email".format(person.id): person.email,
"{}-name".format(person.id): person.name,
"{}-user_name".format(person.id): person.user_name,
"__sort__": person.user_name,
})
def test_full_reindex(self):
"""Test full reindex of all snapshots"""
self._import_file("snapshotter_create.csv")
program = db.session.query(
models.Program).filter(models.Program.slug == "Prog-13211").one()
self.create_audit(program)
audit = db.session.query(models.Audit).filter(
models.Audit.title.like("%Snapshotable audit%")).first()
snapshots = db.session.query(models.Snapshot).all()
records = get_records(audit, snapshots)
self.assertEqual(records.count(), 57)
delete_records({s.id for s in snapshots})
records = get_records(audit, snapshots)
self.assertEqual(records.count(), 0)
do_reindex()
records = get_records(audit, snapshots)
self.assertEqual(records.count(), 57)
def test_search_no_acl_in_content(self, field, role_name):
"""Test search older revisions without access_control_list."""
with factories.single_commit():
person = factories.PersonFactory(
email="{}@example.com".format(field), name=field)
control = factories.ControlFactory()
revision = all_models.Revision.query.filter(
all_models.Revision.resource_id == control.id,
all_models.Revision.resource_type == control.type,
).one()
with factories.single_commit():
snapshot = factories.SnapshotFactory(child_id=control.id,
child_type=control.type,
revision=revision)
old_content = revision.content.copy()
old_content.pop("access_control_list")
old_content[field] = {"id": person.id}
revision.content = old_content
db.session.add(revision)
do_reindex()
self.assert_indexed_fields(
snapshot, role_name, {
"{}-email".format(person.id): person.email,
"{}-name".format(person.id): person.name,
"{}-user_name".format(person.id): person.user_name,
"__sort__": person.user_name,
})
def test_index_by_acr(self):
"""Test index by ACR."""
role_name = "Test name"
with factories.single_commit():
acr = factories.AccessControlRoleFactory(name=role_name,
object_type="Control")
person = factories.PersonFactory(email="*****@*****.**",
name='test')
control = factories.ControlFactory()
factories.AccessControlList(ac_role=acr,
person=person,
object=control)
revision = all_models.Revision.query.filter(
all_models.Revision.resource_id == control.id,
all_models.Revision.resource_type == control.type,
).one()
revision.content = control.log_json()
db.session.add(revision)
with factories.single_commit():
snapshot = factories.SnapshotFactory(child_id=control.id,
child_type=control.type,
revision=revision)
db.session.expire_all()
do_reindex()
self.assert_indexed_fields(
snapshot, role_name, {
"{}-email".format(person.id): person.email,
"{}-name".format(person.id): person.name,
"{}-user_name".format(person.id): person.user_name,
"__sort__": person.user_name,
})
def test_simple_reindex(self):
"""Test for check simple reindex procedure."""
with ggrc_factories.single_commit():
for factory in self.INDEXED_MODEL_FACTORIES:
for _ in range(5):
factory()
indexer = fulltext.get_indexer()
count = indexer.record_type.query.count()
views.do_reindex()
self.assertEqual(count, indexer.record_type.query.count())
def test_simple_reindex(self):
"""Test for check simple reindex procedure."""
with ggrc_factories.single_commit():
for factory in self.INDEXED_MODEL_FACTORIES:
for _ in range(5):
factory()
indexer = fulltext.get_indexer()
count = indexer.record_type.query.count()
views.do_reindex()
self.assertEqual(count, indexer.record_type.query.count())
def test_type_cad(self):
"""Test CAD with the name 'type' """
title1 = "type"
cad1 = CAD(title=title1, definition_type="market")
factory = factories.MarketFactory()
CAV(custom_attribute=cad1, attributable=factory, attribute_value="x")
views.do_reindex()
title1_count = mysql.MysqlRecordProperty.query.filter(
mysql.MysqlRecordProperty.property == title1).count()
self.assertEqual(title1_count, 1)
def test_type_cad(self):
"""Test CAD with the name 'type' """
title1 = "type"
cad1 = CAD(title=title1, definition_type="market")
factory = factories.MarketFactory()
CAV(custom_attribute=cad1, attributable=factory, attribute_value="x")
views.do_reindex()
title1_count = mysql.MysqlRecordProperty.query.filter(
mysql.MysqlRecordProperty.property == title1
).count()
self.assertEqual(title1_count, 1)
def test_simple_reindex(self):
"""Test for check simple reindex procedure."""
with ggrc_factories.single_commit():
for factory in self.INDEXED_MODEL_FACTORIES:
for _ in range(5):
factory()
indexer = fulltext.get_indexer()
count = indexer.record_type.query.count()
views.do_reindex()
# ACR roles are created in migration and aren't removed in setup
# Index for them will be created only after reindexing
reindexed_count = indexer.record_type.query.filter(
MysqlRecordProperty.type != "AccessControlRole"
).count()
self.assertEqual(count, reindexed_count)
def _setup_objects():
"""Create and reindex objects needed for tests"""
text_cad = factories.CustomAttributeDefinitionFactory(
title="text cad",
definition_type="market",
)
date_cad = factories.CustomAttributeDefinitionFactory(
title="date cad",
definition_type="market",
attribute_type="Date",
)
audit = factories.AuditFactory()
for i in range(5):
factories.OrgGroupFactory()
market = factories.MarketFactory()
factories.CustomAttributeValueFactory(
custom_attribute=date_cad,
attributable=market,
attribute_value="2016-11-0{}".format(i + 3),
)
factories.CustomAttributeValueFactory(
custom_attribute=text_cad,
attributable=market,
attribute_value="2016-11-0{}".format(i + 1),
)
revisions = models.Revision.query.filter(
models.Revision.resource_type.in_(["OrgGroup", "Market"]),
models.Revision.id.in_(
db.session.query(func.max(models.Revision.id)).group_by(
models.Revision.resource_type,
models.Revision.resource_id,
)
),
)
for revision in revisions:
factories.SnapshotFactory(
child_id=revision.resource_id,
child_type=revision.resource_type,
revision=revision,
parent=audit,
)
views.do_reindex()
def test_cad_length(self):
"""Custom attribute titles must support all lengths that can be stored."""
title1 = "a" * 200 + "1"
title2 = "a" * 200 + "2"
cad1 = CAD(title=title1, definition_type="market")
cad2 = CAD(title=title2, definition_type="market",)
factory = factories.MarketFactory()
CAV(custom_attribute=cad1, attributable=factory, attribute_value="x")
CAV(custom_attribute=cad2, attributable=factory, attribute_value="x")
views.do_reindex()
title1_count = mysql.MysqlRecordProperty.query.filter(
mysql.MysqlRecordProperty.property == title1
).count()
self.assertEqual(title1_count, 1)
def test_cad_length(self):
"""Custom attribute titles must support all lengths that can be stored."""
title1 = "a" * 200 + "1"
title2 = "a" * 200 + "2"
cad1 = CAD(title=title1, definition_type="market")
cad2 = CAD(title=title2, definition_type="market",)
factory = factories.MarketFactory()
CAV(custom_attribute=cad1, attributable=factory, attribute_value="x")
CAV(custom_attribute=cad2, attributable=factory, attribute_value="x")
views.do_reindex()
title1_count = mysql.MysqlRecordProperty.query.filter(
mysql.MysqlRecordProperty.property == title1
).count()
self.assertEqual(title1_count, 1)
def test_total_count_with_ca(self, limit, expected_count):
"""Test total related assessments count for assessments with ca.
The left outer join in our eager query on custom attribute values breaks
the total count if on sa.func.count, but works if we use query.count()
"""
cad = factories.CustomAttributeDefinitionFactory
cads = [cad(definition_type="assessment") for _ in range(3)]
for cad in cads:
factories.CustomAttributeValueFactory(
attributable=self.assessment1,
custom_attribute=cad,
)
factories.CustomAttributeValueFactory(
attributable=self.assessment2,
custom_attribute=cad,
)
with mock.patch("ggrc.views.start_compute_attributes"):
views.do_reindex()
response = self._get_related_assessments(self.control, **limit).json
self.assertEqual(response["total"], expected_count)
def _setup_objects():
"""Create and reindex objects needed for tests"""
text_cad = factories.CustomAttributeDefinitionFactory(
title="text cad",
definition_type="market",
)
date_cad = factories.CustomAttributeDefinitionFactory(
title="date cad",
definition_type="market",
attribute_type="Date",
)
person_cad = factories.CustomAttributeDefinitionFactory(
title="CA person",
definition_type="market",
attribute_type="Map:Person",
)
users = [
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**"
]
for user in users:
factories.PersonFactory(email=user)
audit = factories.AuditFactory()
for i in range(5):
factories.ControlFactory(title="Control {}".format(i + 1))
factories.OrgGroupFactory()
market = factories.MarketFactory(title="Market {}".format(i + 1))
factories.CustomAttributeValueFactory(
custom_attribute=date_cad,
attributable=market,
attribute_value="2016-11-0{}".format(i + 3),
)
factories.CustomAttributeValueFactory(
custom_attribute=text_cad,
attributable=market,
attribute_value="2016-11-0{}".format(i + 1),
)
factories.CustomAttributeValueFactory(
custom_attribute=person_cad,
attributable=market,
attribute_value="user{}@example.com".format(i + 1),
)
revisions = models.Revision.query.filter(
models.Revision.resource_type.in_(["OrgGroup", "Market", "Control"]),
models.Revision.id.in_(
db.session.query(func.max(models.Revision.id)).group_by(
models.Revision.resource_type,
models.Revision.resource_id,
)
),
)
snapshots = [
factories.SnapshotFactory(
child_id=revision.resource_id,
child_type=revision.resource_type,
revision=revision,
parent=audit,
)
for revision in revisions
]
snapshot_map = {snapshot.revision.content["title"]: snapshot
for snapshot in snapshots}
# create Assessments and issues and map some snapshots to them
# Markets and Controls represent snapshots of those objects.
assessment_issues = (
factories.AssessmentFactory,
factories.IssueFactory,
)
for i in range(4):
for factory in assessment_issues:
# pylint: disable=protected-access
obj = factory(
title="{} {}".format(factory._meta.model.__name__, i + 1),
)
factories.RelationshipFactory(
source=audit if i % 2 == 0 else obj,
destination=audit if i % 2 == 1 else obj,
)
market = snapshot_map["Market {}".format(i + 1)]
factories.RelationshipFactory(
source=market if i % 2 == 0 else obj,
destination=market if i % 2 == 1 else obj,
)
for j in range(i):
control = snapshot_map["Control {}".format(j + 1)]
factories.RelationshipFactory(
source=control if i % 2 == 0 else obj,
destination=control if i % 2 == 1 else obj,
)
# Create an unmapped control for base tests
factories.ControlFactory(title="Control 0")
views.do_reindex()
def setUp(self):
views.do_reindex()
def _setup_objects(cls):
"""Create and reindex objects needed for tests"""
cls._create_snapshotable_objects()
revisions = models.Revision.query.filter(
models.Revision.resource_type.in_(["OrgGroup", "Market", "Control"]),
models.Revision.id.in_(
db.session.query(func.max(models.Revision.id)).group_by(
models.Revision.resource_type,
models.Revision.resource_id,
)
),
)
audit = factories.AuditFactory()
snapshots = [
factories.SnapshotFactory(
child_id=revision.resource_id,
child_type=revision.resource_type,
revision=revision,
parent=audit,
)
for revision in revisions
]
snapshot_map = {snapshot.revision.content["title"]: snapshot
for snapshot in snapshots}
# create Assessments and issues and map some snapshots to them
# Markets and Controls represent snapshots of those objects.
assessment_issues = (
factories.AssessmentFactory,
factories.IssueFactory,
)
for i in range(4):
for factory in assessment_issues:
# pylint: disable=protected-access
obj = factory(
title="{} {}".format(factory._meta.model.__name__, i + 1),
audit=audit,
)
factories.RelationshipFactory(
source=audit if i % 2 == 0 else obj,
destination=audit if i % 2 == 1 else obj,
)
market = snapshot_map["Market {}".format(i + 1)]
factories.RelationshipFactory(
source=market if i % 2 == 0 else obj,
destination=market if i % 2 == 1 else obj,
)
for j in range(i):
control = snapshot_map["Control {}".format(j + 1)]
factories.RelationshipFactory(
source=control if i % 2 == 0 else obj,
destination=control if i % 2 == 1 else obj,
)
# Create an unmapped control for base tests
factories.ControlFactory(title="Control 0")
views.do_reindex()
def _setup_objects():
"""Create and reindex objects needed for tests"""
text_cad = factories.CustomAttributeDefinitionFactory(
title="text cad",
definition_type="market",
)
date_cad = factories.CustomAttributeDefinitionFactory(
title="date cad",
definition_type="market",
attribute_type="Date",
)
person_cad = factories.CustomAttributeDefinitionFactory(
title="CA person",
definition_type="market",
attribute_type="Map:Person",
)
users = [
"*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"
]
for user in users:
factories.PersonFactory(email=user)
audit = factories.AuditFactory()
for i in range(5):
factories.ControlFactory(title="Control {}".format(i + 1))
factories.OrgGroupFactory()
market = factories.MarketFactory(title="Market {}".format(i + 1))
factories.CustomAttributeValueFactory(
custom_attribute=date_cad,
attributable=market,
attribute_value="2016-11-0{}".format(i + 3),
)
factories.CustomAttributeValueFactory(
custom_attribute=text_cad,
attributable=market,
attribute_value="2016-11-0{}".format(i + 1),
)
factories.CustomAttributeValueFactory(
custom_attribute=person_cad,
attributable=market,
attribute_value="user{}@example.com".format(i + 1),
)
revisions = models.Revision.query.filter(
models.Revision.resource_type.in_(
["OrgGroup", "Market", "Control"]),
models.Revision.id.in_(
db.session.query(func.max(models.Revision.id)).group_by(
models.Revision.resource_type,
models.Revision.resource_id,
)),
)
snapshots = [
factories.SnapshotFactory(
child_id=revision.resource_id,
child_type=revision.resource_type,
revision=revision,
parent=audit,
) for revision in revisions
]
snapshot_map = {
snapshot.revision.content["title"]: snapshot
for snapshot in snapshots
}
# create Assessments and issues and map some snapshots to them
# Markets and Controls represent snapshots of those objects.
assessment_issues = (
factories.AssessmentFactory,
factories.IssueFactory,
)
for i in range(4):
for factory in assessment_issues:
# pylint: disable=protected-access
obj = factory(title="{} {}".format(
factory._meta.model.__name__, i + 1), )
factories.RelationshipFactory(
source=audit if i % 2 == 0 else obj,
destination=audit if i % 2 == 1 else obj,
)
market = snapshot_map["Market {}".format(i + 1)]
factories.RelationshipFactory(
source=market if i % 2 == 0 else obj,
destination=market if i % 2 == 1 else obj,
)
for j in range(i):
control = snapshot_map["Control {}".format(j + 1)]
factories.RelationshipFactory(
source=control if i % 2 == 0 else obj,
destination=control if i % 2 == 1 else obj,
)
# Create an unmapped control for base tests
factories.ControlFactory(title="Control 0")
views.do_reindex()
