def test_commentable_import(self):
"""Test import of recipients and send_by_default fields"""
for model_name in COMMENTABLE_MODELS:
model_cls = inflector.get_model(model_name)
obj = model_cls.query.first()
self.assertEqual(sorted(obj.recipients.split(",")),
sorted(RECIPIENTS))
self.assertEqual(obj.send_by_default, True)
for model_name in SCOPING_MODELS:
model_cls = inflector.get_model(model_name)
obj = model_cls.query.first()
self.assertEqual(sorted(obj.recipients.split(",")),
sorted(SCOPING_RECIPIENTS))
self.assertEqual(obj.send_by_default, True)
def test_commentable_import(self):
"""Test import of recipients and send_by_default fields"""
for model_name in COMMENTABLE_MODELS:
model_cls = inflector.get_model(model_name)
obj = model_cls.query.first()
self.assertEqual(sorted(obj.recipients.split(",")),
sorted(RECIPIENTS))
self.assertEqual(obj.send_by_default, True)
for model_name in SCOPING_MODELS:
model_cls = inflector.get_model(model_name)
obj = model_cls.query.first()
self.assertEqual(sorted(obj.recipients.split(",")),
sorted(SCOPING_RECIPIENTS))
self.assertEqual(obj.send_by_default, True)
def update_indexer(session): # pylint:disable=unused-argument
"""General function to update index
for all updated related instance before commit"""
models_ids_to_reindex = defaultdict(set)
db.session.flush()
for for_index in getattr(db.session, 'reindex_set', set()):
if for_index not in db.session:
continue
type_name, id_value = for_index.get_reindex_pair()
if type_name:
models_ids_to_reindex[type_name].add(id_value)
db.session.reindex_set = set()
for model_name, ids in models_ids_to_reindex.iteritems():
get_model(model_name).bulk_record_update_for(ids)
def update_indexer(session): # pylint:disable=unused-argument
"""General function to update index
for all updated related instance before commit"""
models_ids_to_reindex = defaultdict(set)
db.session.flush()
for for_index in getattr(db.session, 'reindex_set', set()):
if for_index not in db.session:
continue
type_name, id_value = for_index.get_reindex_pair()
if type_name:
models_ids_to_reindex[type_name].add(id_value)
db.session.reindex_set = set()
for model_name, ids in models_ids_to_reindex.iteritems():
get_model(model_name).bulk_record_update_for(ids)
def _parse_query(cls, query):
"""Parse cloning parameters from input query.
Args:
query: Dict with cloning parameters.
Returns:
Tuple that include list objects to clone, destination object and
list of possible mapped types (source_objs, destination, mapped_types).
"""
if not query:
raise exceptions.BadRequest()
source_ids = query.get("sourceObjectIds", [])
if not source_ids:
raise exceptions.BadRequest(
"sourceObjectIds parameter wasn't provided")
source_objs = cls.query.options(
sa.orm.subqueryload('custom_attribute_definitions'),
sa.orm.subqueryload('custom_attribute_values'),
).filter(cls.id.in_(source_ids)).all()
dest_query = query.get("destination", {})
destination = None
if dest_query and dest_query.get("type") and dest_query.get("id"):
destination_cls = inflector.get_model(dest_query.get("type"))
destination = destination_cls.query.filter_by(
id=dest_query.get("id")).first()
mapped_types = {
type_
for type_ in query.get("mappedObjects", [])
if type_ in cls.CLONEABLE_CHILDREN
}
return source_objs, destination, mapped_types
def get_objects_by_query():
"""Return objects corresponding to a POST'ed query list."""
query = request.json
results = get_handler_results(query)
last_modified_list = [result["last_modified"] for result in results
if result["last_modified"]]
last_modified = max(last_modified_list) if last_modified_list else None
collections = []
collection_fields = ["ids", "values", "count", "total", "object_name"]
for result in results:
model = get_model(result["object_name"])
if model is not None:
collection = build_collection_representation(
model,
{
field: result[field] for field in collection_fields
if field in result
}
)
collections.append(collection)
return json_success_response(collections, last_modified)
def get_objects_to_reindex(obj):
"""Return list of Commentable objects related to provided comment."""
source_qs = db.session.query(
Relationship.destination_type, Relationship.destination_id
).filter(
Relationship.source_type == obj.type,
Relationship.source_id == obj.id
)
destination_qs = db.session.query(
Relationship.source_type, Relationship.source_id
).filter(
Relationship.destination_type == obj.type,
Relationship.destination_id == obj.id
)
result_qs = source_qs.union(destination_qs)
klass_dict = defaultdict(set)
for klass, object_id in result_qs:
klass_dict[klass].add(object_id)
queries = []
for klass, object_ids in klass_dict.iteritems():
model = inflector.get_model(klass)
if not model:
continue
if issubclass(model, (Indexed, Commentable, ExternalCommentable)):
queries.append(model.query.filter(model.id.in_(list(object_ids))))
return list(itertools.chain(*queries))
def get_objects_to_reindex(obj):
"""Return list of Commentable objects related to provided comment."""
source_qs = db.session.query(
Relationship.destination_type, Relationship.destination_id
).filter(
Relationship.source_type == obj.type,
Relationship.source_id == obj.id
)
destination_qs = db.session.query(
Relationship.source_type, Relationship.source_id
).filter(
Relationship.destination_type == obj.type,
Relationship.destination_id == obj.id
)
result_qs = source_qs.union(destination_qs)
klass_dict = defaultdict(set)
for klass, object_id in result_qs:
klass_dict[klass].add(object_id)
queries = []
for klass, object_ids in klass_dict.iteritems():
model = inflector.get_model(klass)
if not model:
continue
if issubclass(model, (Indexed, Commentable, ExternalCommentable)):
queries.append(model.query.filter(model.id.in_(list(object_ids))))
return list(itertools.chain(*queries))
def mark_to_cache(type_, id_):
"""Mark object for warmup"""
if not hasattr(flask.g, "referenced_objects_markers"):
flask.g.referenced_objects_markers = collections.defaultdict(set)
if not (isinstance(type_, type) and issubclass(type_, db.Model)):
type_ = inflector.get_model(type_)
flask.g.referenced_objects_markers[type_].add(id_)
def text_local_delete_relationship_scoping_directive(
self, scoping_factory, directive_factory):
"""Test that deleteion of relationship disabled for local users."""
# Set up relationships
self.api.set_user(self.person_ext)
with factories.single_commit():
scoping_object = scoping_factory()
directive = directive_factory()
mappings = [(scoping_object, directive), (directive, scoping_object)]
relationship_ids = []
for source, destination in mappings:
rel = self.create_relationship(source, destination, True,
self.person_ext)
relationship_ids.append(rel.id)
self.api.set_user(self.person)
for rel_id in relationship_ids:
relationship = all_models.Relationship.query.get(rel_id)
response = self.api.delete(relationship)
self.assert400(response)
# relationship allowed to be deleted when source or
# destination objects are deleted
directive_model = get_model(relationship.destination_type)
directive = directive_model.query.get(relationship.destination_id)
self.assertIsNone(directive)
response = self.api.delete(directive)
self.assert200(response)
relationship = all_models.Relationship.query.get(rel.id)
self.assertIsNone(relationship)
def get_objects_by_query():
"""Return objects corresponding to a POST'ed query list."""
query = request.json
query_helper = QueryAPIQueryHelper(query)
results = query_helper.get_results()
last_modified_list = [result["last_modified"] for result in results
if result["last_modified"]]
last_modified = max(last_modified_list) if last_modified_list else None
collections = []
collection_fields = ["ids", "values", "count", "total"]
for result in results:
if last_modified is None:
last_modified = result["last_modified"]
elif result["last_modified"] is not None:
last_modified = max(last_modified, result["last_modified"])
model = get_model(result["object_name"])
collection = build_collection_representation(
model,
{
field: result[field] for field in collection_fields
if field in result
}
)
collections.append(collection)
return json_success_response(collections, last_modified)
def get_objects_to_reindex(self):
"""Return list required objects for reindex if comment C.U.D."""
source_qs = db.session.query(
Relationship.destination_type, Relationship.destination_id
).filter(
Relationship.source_type == self.__class__.__name__,
Relationship.source_id == self.id
)
destination_qs = db.session.query(
Relationship.source_type, Relationship.source_id
).filter(
Relationship.destination_type == self.__class__.__name__,
Relationship.destination_id == self.id
)
result_qs = source_qs.union(destination_qs)
klass_dict = defaultdict(set)
for klass, object_id in result_qs:
klass_dict[klass].add(object_id)
queries = []
for klass, object_ids in klass_dict.iteritems():
model = inflector.get_model(klass)
if not model:
continue
if issubclass(model, (Indexed, Commentable)):
queries.append(model.query.filter(model.id.in_(list(object_ids))))
return list(itertools.chain(*queries))
def get_objects_to_reindex(self):
"""Return list required objects for reindex if comment C.U.D."""
source_qs = db.session.query(
Relationship.destination_type, Relationship.destination_id
).filter(
Relationship.source_type == self.__class__.__name__,
Relationship.source_id == self.id
)
destination_qs = db.session.query(
Relationship.source_type, Relationship.source_id
).filter(
Relationship.destination_type == self.__class__.__name__,
Relationship.destination_id == self.id
)
result_qs = source_qs.union(destination_qs)
klass_dict = defaultdict(set)
for klass, object_id in result_qs:
klass_dict[klass].add(object_id)
queries = []
for klass, object_ids in klass_dict.iteritems():
model = inflector.get_model(klass)
if not model:
continue
if issubclass(model, (Indexed, Commentable)):
queries.append(model.query.filter(model.id.in_(list(object_ids))))
return list(itertools.chain(*queries))
def similar(exp, object_class, target_class, query):
"""Filter by relationships similarity.
Note: only the first id from the list of ids is used.
Args:
object_name: the name of the class of the objects to which similarity
will be computed.
ids: the ids of similar objects of type `object_name`.
Returns:
sqlalchemy.sql.elements.BinaryExpression if an object of `object_class`
is similar to one the given objects.
"""
similar_class = inflector.get_model(exp['object_name'])
if not hasattr(similar_class, "get_similar_objects_query"):
raise BadQueryException(u"{} does not define weights to count "
u"relationships similarity".format(
similar_class.__name__))
similar_objects_query = similar_class.get_similar_objects_query(
id_=exp['ids'][0],
types=[object_class.__name__],
)
flask.g.similar_objects_query = similar_objects_query
similar_objects_ids = [obj.id for obj in similar_objects_query]
if similar_objects_ids:
return object_class.id.in_(similar_objects_ids)
return sqlalchemy.sql.false()
def get_objects_by_query():
"""Return objects corresponding to a POST'ed query list."""
query = request.json
results = get_handler_results(query)
last_modified_list = [result["last_modified"] for result in results
if result["last_modified"]]
last_modified = max(last_modified_list) if last_modified_list else None
collections = []
collection_fields = ["ids", "values", "count", "total", "object_name"]
for result in results:
model = get_model(result["object_name"])
collection = build_collection_representation(
model,
{
field: result[field] for field in collection_fields
if field in result
}
)
collections.append(collection)
return json_success_response(collections, last_modified)
def similar(exp, object_class, target_class, query):
"""Filter by relationships similarity.
Note: only the first id from the list of ids is used.
Args:
object_name: the name of the class of the objects to which similarity
will be computed.
ids: the ids of similar objects of type `object_name`.
Returns:
sqlalchemy.sql.elements.BinaryExpression if an object of `object_class`
is similar to one the given objects.
"""
similar_class = inflector.get_model(exp['object_name'])
if not hasattr(similar_class, "get_similar_objects_query"):
raise BadQueryException(u"{} does not define weights to count "
u"relationships similarity"
.format(similar_class.__name__))
similar_objects_query = similar_class.get_similar_objects_query(
id_=exp['ids'][0],
type_=object_class.__name__,
)
similar_objects_ids = {obj[0] for obj in similar_objects_query}
if similar_objects_ids:
return object_class.id.in_(similar_objects_ids)
return sqlalchemy.sql.false()
def do_reindex():
"""Update the full text search index."""
indexer = get_indexer()
with benchmark('Delete all records'):
indexer.delete_all_records(False)
indexed_models = get_indexed_model_names()
people = db.session.query(all_models.Person.id, all_models.Person.name,
all_models.Person.email)
g.people_map = {p.id: (p.name, p.email) for p in people}
for model in sorted(indexed_models):
# pylint: disable=protected-access
logger.info("Updating index for: %s", model)
with benchmark("Create records for %s" % model):
model = get_model(model)
mapper_class = model._sa_class_manager.mapper.base_mapper.class_
query = model.query.options(
db.undefer_group(mapper_class.__name__ + '_complete'), )
for query_chunk in generate_query_chunks(query):
for instance in query_chunk:
indexer.create_record(fts_record_for(instance), False)
db.session.commit()
reindex_snapshots()
delattr(g, "people_map")
def _get_missing_models_query(role):
"""Get query for objects with a missing ACL entry for the given role.
Note that the filter flag here is just for optimization. When creating a new
access control role we can be sure that no object will have the ACL entry for
that role so we can skip the filter.
Args:
role: role object for which we want to check acl entries.
filter_: A flag to filter the possible objects that might already have the
given acl entry
Returns:
sqlalchemy query for the object type specified in the access control role.
"""
model = inflector.get_model(role.object_type)
if not model:
# We only log info instead of warning here because we still leave access
# control roles of obsolete objects in our database, so that we can use
# them with old revisions in our history log.
logger.info("Trying to handle role '%s' for non existent object '%s'",
role.name, role.object_type)
return None
query = model.query.outerjoin(
all_models.AccessControlList,
sa.and_(all_models.AccessControlList.object_type == model.__name__,
all_models.AccessControlList.object_id == model.id,
all_models.AccessControlList.ac_role_id == role.id)).filter(
all_models.AccessControlList.id.is_(None)).order_by(
model.id)
return query
def import_model(self, model_name, audit, recipients, send_by_default):
"""Import model data with commentable fields"""
# pylint: disable=protected-access
import_data = [
("object_type", model_name),
("Code", "{}-1".format(model_name)),
("Title", "{}-Title".format(model_name)),
("Admin", "*****@*****.**"),
("Recipients", recipients),
("Send by default", send_by_default),
]
model_cls = inflector.get_model(model_name)
if model_name in SCOPING_MODELS:
import_data.append(("Assignee", "*****@*****.**"))
import_data.append(("Verifier", "*****@*****.**"))
if model_name == "Control":
import_data.append(("Assertions*", "Privacy"))
if issubclass(model_cls, AuditRelationship):
import_data.append(("Map:Audit", audit))
if (issubclass(model_cls, Described) and "description"
not in model_cls._aliases) or model_name == "Risk":
import_data.append(
("description", "{}-Description".format(model_name)))
response = self.import_data(OrderedDict(import_data))
self._check_csv_response(response, {})
def do_reindex():
"""Update the full text search index."""
indexer = get_indexer()
indexed_models = get_indexed_model_names()
people = db.session.query(all_models.Person.id, all_models.Person.name,
all_models.Person.email)
indexer.cache["people_map"] = {p.id: (p.name, p.email) for p in people}
for model in sorted(indexed_models):
# pylint: disable=protected-access
logger.info("Updating index for: %s", model)
with benchmark("Create records for %s" % model):
model = get_model(model)
mapper_class = model._sa_class_manager.mapper.base_mapper.class_
if issubclass(model, mixin.Indexed):
for query_chunk in generate_query_chunks(
db.session.query(model.id)):
model.bulk_record_update_for([i.id for i in query_chunk])
db.session.commit()
else:
logger.warning(
"Try to index model that not inherited from Indexed mixin: %s",
model.__name__)
indexer.delete_records_by_type(model.__name__)
query = model.query.options(
db.undefer_group(mapper_class.__name__ + '_complete'), )
for query_chunk in generate_query_chunks(query):
for instance in query_chunk:
indexer.create_record(indexer.fts_record_for(instance),
False)
db.session.commit()
reindex_snapshots()
indexer.invalidate_cache()
def load_objs(cls, data):
"""Load objects by their ids and types.
Args:
data: List of stubs [(_, type, id),] for objects to load.
Returns:
Dict with object type and id as keys and instance as value.
"""
# Combine ids of one type together to load in one query
type_ids = defaultdict(set)
for _, type_, id_ in data:
type_ids[type_].add(id_)
type_id_objs = defaultdict(dict)
# We can't load all objects with different types in one step, so we
# load them for each type separately
for type_, ids in type_ids.items():
related_model = inflector.get_model(type_)
related_query = related_model.query.options(
sa.orm.subqueryload('custom_attribute_definitions'), ).filter(
related_model.id.in_(ids))
for related in related_query:
type_id_objs[type_][related.id] = related
return type_id_objs
def get_objects_by_query():
"""Return objects corresponding to a POST'ed query list."""
query = request.json
query_helper = QueryAPIQueryHelper(query)
results = query_helper.get_results()
last_modified_list = [
result["last_modified"] for result in results
if result["last_modified"]
]
last_modified = max(last_modified_list) if last_modified_list else None
collections = []
collection_fields = ["ids", "values", "count", "total"]
for result in results:
if last_modified is None:
last_modified = result["last_modified"]
elif result["last_modified"] is not None:
last_modified = max(last_modified, result["last_modified"])
model = get_model(result["object_name"])
collection = build_collection_representation(model, {
field: result[field]
for field in collection_fields if field in result
})
collections.append(collection)
return json_success_response(collections, last_modified)
def load_objs(cls, data):
"""Load objects by their ids and types.
Args:
data: List of stubs [(_, type, id),] for objects to load.
Returns:
Dict with object type and id as keys and instance as value.
"""
# Combine ids of one type together to load in one query
type_ids = defaultdict(set)
for _, type_, id_ in data:
type_ids[type_].add(id_)
type_id_objs = defaultdict(dict)
# We can't load all objects with different types in one step, so we
# load them for each type separately
for type_, ids in type_ids.items():
related_model = inflector.get_model(type_)
related_query = related_model.query.options(
sa.orm.subqueryload('custom_attribute_definitions'),
).filter(related_model.id.in_(ids))
for related in related_query:
type_id_objs[type_][related.id] = related
return type_id_objs
def _parse_query(cls, query):
"""Parse cloning parameters from input query.
Args:
query: Dict with cloning parameters.
Returns:
Tuple that include list objects to clone, destination object and
list of possible mapped types (source_objs, destination, mapped_types).
"""
if not query:
raise exceptions.BadRequest()
source_ids = query.get("sourceObjectIds", [])
if not source_ids:
raise exceptions.BadRequest("sourceObjectIds parameter wasn't provided")
source_objs = cls.query.options(
sa.orm.subqueryload('custom_attribute_definitions'),
sa.orm.subqueryload('custom_attribute_values'),
).filter(cls.id.in_(source_ids)).all()
dest_query = query.get("destination", {})
destination = None
if dest_query and dest_query.get("type") and dest_query.get("id"):
destination_cls = inflector.get_model(dest_query.get("type"))
destination = destination_cls.query.filter_by(
id=dest_query.get("id")
).first()
mapped_types = {
type_ for type_ in query.get("mappedObjects", [])
if type_ in cls.CLONEABLE_CHILDREN
}
return source_objs, destination, mapped_types
def test_commentable_import(self):
"""Test import of recipients and send_by_default fields"""
for model_name, recipients in RECIPIENTS_MAPPING.iteritems():
model_cls = inflector.get_model(model_name)
obj = model_cls.query.first()
self.assertEqual(sorted(obj.recipients.split(",")),
sorted(recipients))
self.assertEqual(obj.send_by_default, True)
def mark_to_cache(type_, id_):
"""Mark object for warmup"""
if not hasattr(flask.g, "referenced_objects_markers"):
flask.g.referenced_objects_markers = collections.defaultdict(set)
if not (isinstance(type_, type) and issubclass(type_, db.Model)):
type_ = inflector.get_model(type_)
if type_ is not None:
flask.g.referenced_objects_markers[type_].add(id_)
def get_by_id(obj):
"""Get object instance by id"""
if not obj:
return
model = inflector.get_model(obj['type'])
if not model:
return
return model.query.get(obj["id"])
def _slugs_to_ids(object_name, slugs):
"""Convert SLUG to proper ids for the given objec."""
object_class = inflector.get_model(object_name)
if not object_class:
return []
ids = [c.id for c in object_class.query.filter(
object_class.slug.in_(slugs)).all()]
return ids
def _slugs_to_ids(object_name, slugs):
"""Convert SLUG to proper ids for the given objec."""
object_class = inflector.get_model(object_name)
if not object_class:
return []
ids = [c.id for c in object_class.query.filter(
object_class.slug.in_(slugs)).all()]
return ids
def _ensure_program_relationships(snapshots):
"""Ensure that snapshotted object is related to audit program.
This function is made to handle multiple snapshots for a single audit.
Args:
snapshots: list of snapshot objects with child_id, child_type and parent.
"""
# assert that every parent is an Audit as the code relies on program_id field
assert {s.parent_type for s in snapshots} == {"Audit"}
rel = relationship.Relationship
program_children = {}
for obj in snapshots:
program_children.setdefault(obj.parent.program, set()).add(
(obj.child_type, obj.child_id)
)
for program, children_set in program_children.items():
query = db.session.query(
rel.destination_type, rel.destination_id
).filter(
and_(
rel.source_type == "Program",
rel.source_id == program.id,
tuple_(rel.destination_type, rel.destination_id).in_(children_set)
)
).union_all(
db.session.query(
rel.source_type, rel.source_id
).filter(
and_(
rel.destination_type == "Program",
rel.destination_id == program.id,
tuple_(rel.source_type, rel.source_id).in_(children_set)
)
)
)
children_set.difference_update(query.all())
child_objects = {}
type_ids = {}
for child in children_set:
type_ids.setdefault(child[0], set()).add(child[1])
for child_type, ids in type_ids.items():
child_model = inflector.get_model(child_type)
query = child_model.query.filter(child_model.id.in_(ids))
for child in query:
child_objects[(child.type, child.id)] = child
for child in children_set:
if child in child_objects:
db.session.add(
relationship.Relationship(
source=program,
destination=child_objects[child],
)
)
def original_object_deleted(self):
"""Flag if the snapshot has the last revision and action is deleted."""
if not self.revisions:
return False
deleted = self.revisions[-1].action == "deleted"
external_deleted = (
self.revisions[-1].content["status"] == "Deprecated" and
issubclass(inflector.get_model(self.child_type), Synchronizable))
return bool(deleted or external_deleted)
def validate_instance_type(self, _, instance_type):
"""Validate instance_type attribute.
We preventing creation of proposals for external models.
"""
instance_class = inflector.get_model(instance_type)
if issubclass(instance_class, synchronizable.Synchronizable):
raise ValueError("Trying to create proposal for external model.")
return instance_type
def validate_reviewable_type(self, _, reviewable_type):
"""Validate reviewable_type attribute.
We preventing creation of reviews for external models.
"""
reviewable_class = inflector.get_model(reviewable_type)
if issubclass(reviewable_class, synchronizable.Synchronizable):
raise ValueError("Trying to create review for external model.")
return reviewable_type
def validate_reviewable_type(self, _, reviewable_type):
"""Validate reviewable_type attribute.
We preventing creation of reviews for external models.
"""
reviewable_class = inflector.get_model(reviewable_type)
if issubclass(reviewable_class, synchronizable.Synchronizable):
raise ValueError("Trying to create review for external model.")
return reviewable_type
def test_asmnt_procedure_export(self, model):
"""Test export of Assessment Procedure. {}"""
with factories.single_commit():
program = factories.ProgramFactory()
audit = factories.AuditFactory(program=program)
import_queries = []
for i in range(3):
import_queries.append(
collections.OrderedDict([
("object_type", model),
("Assessment Procedure", "Procedure-{}".format(i)),
("Title", "Title {}".format(i)),
("Code*", "{}-{}".format(model, i)),
("Admin", "*****@*****.**"),
("Assignees", "*****@*****.**"),
("Creators", "*****@*****.**"),
("Description", "{} description".format(model)),
("Program", program.slug),
("Audit", audit.slug),
("Start Date", ""),
("End Date", ""),
]))
if model == "Control":
import_queries[-1]["Assertions"] = "Privacy"
if model == "Risk":
import_queries[-1]["Risk Type"] = "Risk type"
if model.replace(" ", "") in self.SCOPING_MODELS_NAMES:
import_queries[-1]["Assignee"] = "*****@*****.**"
import_queries[-1]["Verifier"] = "*****@*****.**"
self.check_import_errors(self.import_data(*import_queries))
model_cls = inflector.get_model(model)
objects = model_cls.query.order_by(model_cls.test_plan).all()
self.assertEqual(len(objects), 3)
for num, obj in enumerate(objects):
self.assertEqual(obj.test_plan, "Procedure-{}".format(num))
obj_dicts = [{
"Code*": obj.slug,
"Assessment Procedure": "Procedure-{}".format(i)
} for i, obj in enumerate(objects)]
search_request = [{
"object_name": model_cls.__name__,
"filters": {
"expression": {},
"order_by": {
"name": "id"
}
},
"fields": ["slug", "test_plan"],
}]
exported_data = self.export_parsed_csv(search_request)[model]
self.assertEqual(exported_data, obj_dicts)
def validate_instance_type(self, _, instance_type):
"""Validate instance_type attribute.
We preventing creation of proposals for external models.
"""
instance_class = inflector.get_model(instance_type)
if issubclass(instance_class, synchronizable.Synchronizable):
raise ValueError("Trying to create proposal for external model.")
return instance_type
def update_indexer(session): # pylint:disable=unused-argument
"""General function to update index
for all updated related instance before commit"""
with benchmark("Update indexer before commit"):
if not hasattr(db.session, "reindex_set"):
return
models_ids_to_reindex = defaultdict(set)
db.session.flush()
for for_index in db.session.reindex_set:
if for_index not in db.session:
continue
type_name, id_value = for_index.get_reindex_pair()
if type_name:
models_ids_to_reindex[type_name].add(id_value)
# expire required to fix declared_attr cached value
db.session.expire_all()
db.session.reindex_set.invalidate()
for model_name, ids in models_ids_to_reindex.iteritems():
get_model(model_name).bulk_record_update_for(ids)
def expected_single_equal(self, setup_objs):
"""Calculate expected values for single test case, equal operator"""
values = []
for data in setup_objs:
search_obj = inflector.get_model(data.searchable_type).query.get(
data.searchable_id)
for subprop in self.Meta.subprops:
values.append(([
create_tuple_data(data.obj_id, search_obj,
self.Meta.subprops)
], getattr(search_obj, subprop)))
return values
def test_unified_hint_state(self, model):
"""Tests if {} type attribute state has hint expected"""
data = {
"export_to": "csv",
"objects": [
{"object_name": model, "fields": ["title", "status"]},
],
}
response = self.client.post("/_service/export_csv",
data=dumps(data), headers=self.headers)
self.assertIn("Allowed values are:\n{}".format(
'\n'.join(inflector.get_model(model).VALID_STATES)), response.data)
def _get_ids(self, object_query):
"""Get a set of ids of objects described in the filters."""
object_name = object_query["object_name"]
expression = object_query.get("filters", {}).get("expression")
if expression is None:
return set()
object_class = inflector.get_model(object_name)
query = db.session.query(object_class.id)
tgt_class = object_class
if object_name == "Snapshot":
child_type = self._get_snapshot_child_type(object_query)
tgt_class = getattr(models.all_models, child_type, object_class)
requested_permissions = object_query.get("permissions", "read")
with benchmark("Get permissions: _get_ids > _get_type_query"):
type_query = self._get_type_query(object_class, requested_permissions)
if type_query is not None:
query = query.filter(type_query)
with benchmark("Parse filter query: _get_ids > _build_expression"):
filter_expression = custom_operators.build_expression(
expression,
object_class,
tgt_class,
self.query
)
if filter_expression is not None:
query = query.filter(filter_expression)
if object_query.get("order_by"):
with benchmark("Sorting: _get_ids > order_by"):
query = self._apply_order_by(
object_class,
query,
object_query["order_by"],
tgt_class,
)
with benchmark("Apply limit"):
limit = object_query.get("limit")
if limit:
ids, total = self._apply_limit(query, limit)
else:
ids = [obj.id for obj in query]
total = len(ids)
object_query["total"] = total
if hasattr(flask.g, "similar_objects_query"):
# delete similar_objects_query for the case when several queries are
# POSTed in one request, the first one filters by similarity and the
# second one doesn't but tries to sort by __similarity__
delattr(flask.g, "similar_objects_query")
return ids
def update_indexer(session): # pylint:disable=unused-argument
"""General function to update index
for all updated related instance before commit"""
with benchmark("Update indexer before commit"):
if not hasattr(db.session, "reindex_set"):
return
models_ids_to_reindex = defaultdict(set)
db.session.flush()
for for_index in db.session.reindex_set:
if for_index not in db.session:
continue
type_name, id_value = for_index.get_reindex_pair()
if type_name:
models_ids_to_reindex[type_name].add(id_value)
# expire required to fix declared_attr cached value
db.session.expire_all()
db.session.reindex_set.invalidate()
for model_name, ids in models_ids_to_reindex.iteritems():
get_model(model_name).bulk_record_update_for(ids)
def _get_ids(self, object_query):
"""Get a set of ids of objects described in the filters."""
object_name = object_query["object_name"]
expression = object_query.get("filters", {}).get("expression")
if expression is None:
return set()
object_class = inflector.get_model(object_name)
if object_class is None:
return set()
query = db.session.query(object_class.id)
tgt_class = object_class
if object_name == "Snapshot":
child_type = self._get_snapshot_child_type(object_query)
tgt_class = getattr(models.all_models, child_type, object_class)
requested_permissions = object_query.get("permissions", "read")
with benchmark("Get permissions: _get_ids > _get_type_query"):
type_query = self._get_type_query(object_class, requested_permissions)
if type_query is not None:
query = query.filter(type_query)
with benchmark("Parse filter query: _get_ids > _build_expression"):
filter_expression = custom_operators.build_expression(
expression,
object_class,
tgt_class,
self.query
)
if filter_expression is not None:
query = query.filter(filter_expression)
if object_query.get("order_by"):
with benchmark("Sorting: _get_ids > order_by"):
query = pagination.apply_order_by(
object_class,
query,
object_query["order_by"],
tgt_class,
)
with benchmark("Apply limit"):
limit = object_query.get("limit")
if limit:
limit_query = pagination.apply_limit(query, limit)
total = pagination.get_total_count(query)
ids = [obj.id for obj in limit_query]
else:
ids = [obj.id for obj in query]
total = len(ids)
object_query["total"] = total
return ids
def _get_ids(self, object_query):
"""Get a set of ids of objects described in the filters."""
object_name = object_query["object_name"]
expression = object_query.get("filters", {}).get("expression")
if expression is None:
return set()
object_class = inflector.get_model(object_name)
if object_class is None:
return set()
query = db.session.query(object_class.id)
tgt_class = object_class
if object_name == "Snapshot":
child_type = self._get_snapshot_child_type(object_query)
tgt_class = getattr(models.all_models, child_type, object_class)
requested_permissions = object_query.get("permissions", "read")
with benchmark("Get permissions: _get_ids > _get_type_query"):
type_query = self._get_type_query(object_class, requested_permissions)
if type_query is not None:
query = query.filter(type_query)
with benchmark("Parse filter query: _get_ids > _build_expression"):
filter_expression = custom_operators.build_expression(
expression,
object_class,
tgt_class,
self.query
)
if filter_expression is not None:
query = query.filter(filter_expression)
if object_query.get("order_by"):
with benchmark("Sorting: _get_ids > order_by"):
query = pagination.apply_order_by(
object_class,
query,
object_query["order_by"],
tgt_class,
)
with benchmark("Apply limit"):
limit = object_query.get("limit")
if limit:
limit_query = pagination.apply_limit(query, limit)
total = pagination.get_total_count(query)
ids = [obj.id for obj in limit_query]
else:
ids = [obj.id for obj in query]
total = len(ids)
object_query["total"] = total
return ids
def expected_single_equal(self, setup_objs):
"""Calculate expected values for single test case, equal operator"""
values = []
for data in setup_objs:
search_obj = inflector.get_model(data.searchable_type).query.get(
data.searchable_id
)
for subprop in self.Meta.subprops:
values.append((
[create_tuple_data(data.obj_id, search_obj, self.Meta.subprops)],
getattr(search_obj, subprop)
))
return values
def expected_multiple_not_equal(self, setup_objs):
"""Calculate expected values for multiple test case, not equal operator"""
values, expected_data = [], []
search_obj = inflector.get_model(
setup_objs[0].searchable_type).query.get(
setup_objs[0].searchable_id)
for data in setup_objs:
if data.searchable_id != setup_objs[0].searchable_id:
expected_data.append(
create_tuple_data(data.obj_id, search_obj,
self.Meta.subprops))
for subprop in self.Meta.subprops:
values.append((expected_data, getattr(search_obj, subprop)))
return values
def get_issuetracked_objects(obj_type, obj_ids):
"""Fetch issuetracked objects from db."""
issuetracked_model = inflector.get_model(obj_type)
return issuetracked_model.query.join(
all_models.IssuetrackerIssue,
sa.and_(
all_models.IssuetrackerIssue.object_type == obj_type,
all_models.IssuetrackerIssue.object_id ==
issuetracked_model.id)).filter(
all_models.IssuetrackerIssue.object_id.in_(obj_ids),
all_models.IssuetrackerIssue.issue_id.is_(None),
).options(
sa.orm.Load(issuetracked_model).undefer_group(
"{}_complete".format(obj_type), ))
def _get_ids(self, object_query):
"""Get a set of ids of objects described in the filters."""
object_name = object_query["object_name"]
expression = object_query.get("filters", {}).get("expression")
if expression is None:
return set()
object_class = inflector.get_model(object_name)
query = db.session.query(object_class.id)
tgt_class = object_class
if object_name == "Snapshot":
child_type = self._get_snapshot_child_type(object_query)
tgt_class = getattr(models.all_models, child_type, object_class)
requested_permissions = object_query.get("permissions", "read")
with benchmark("Get permissions: _get_ids > _get_type_query"):
type_query = self._get_type_query(object_class,
requested_permissions)
if type_query is not None:
query = query.filter(type_query)
with benchmark("Parse filter query: _get_ids > _build_expression"):
filter_expression = custom_operators.build_expression(
expression, object_class, tgt_class, self.query)
if filter_expression is not None:
query = query.filter(filter_expression)
if object_query.get("order_by"):
with benchmark("Sorting: _get_ids > order_by"):
query = self._apply_order_by(
object_class,
query,
object_query["order_by"],
tgt_class,
)
with benchmark("Apply limit"):
limit = object_query.get("limit")
if limit:
ids, total = self._apply_limit(query, limit)
else:
ids = [obj.id for obj in query]
total = len(ids)
object_query["total"] = total
if hasattr(flask.g, "similar_objects_query"):
# delete similar_objects_query for the case when several queries are
# POSTed in one request, the first one filters by similarity and the
# second one doesn't but tries to sort by __similarity__
delattr(flask.g, "similar_objects_query")
return ids
def test_asmnt_procedure_export(self, model):
"""Test export of Assessment Procedure. {}"""
with factories.single_commit():
program = factories.ProgramFactory()
audit = factories.AuditFactory(program=program)
import_queries = []
for i in range(3):
import_queries.append(collections.OrderedDict([
("object_type", model),
("Assessment Procedure", "Procedure-{}".format(i)),
("Title", "Title {}".format(i)),
("Code*", "{}-{}".format(model, i)),
("Admin", "*****@*****.**"),
("Assignees", "*****@*****.**"),
("Creators", "*****@*****.**"),
("Description", "{} description".format(model)),
("Program", program.slug),
("Audit", audit.slug),
("Start Date", "01/02/2019"),
("End Date", "03/03/2019"),
]))
if model.replace(" ", "") in all_models.get_scope_model_names():
import_queries[-1]["Assignee"] = "*****@*****.**"
import_queries[-1]["Verifier"] = "*****@*****.**"
self.check_import_errors(self.import_data(*import_queries))
model_cls = inflector.get_model(model)
objects = model_cls.query.order_by(model_cls.test_plan).all()
self.assertEqual(len(objects), 3)
for num, obj in enumerate(objects):
self.assertEqual(obj.test_plan, "Procedure-{}".format(num))
obj_dicts = [
{
"Code*": obj.slug,
"Assessment Procedure": "Procedure-{}".format(i)
} for i, obj in enumerate(objects)
]
search_request = [{
"object_name": model_cls.__name__,
"filters": {
"expression": {},
"order_by": {"name": "id"}
},
"fields": ["slug", "test_plan"],
}]
exported_data = self.export_parsed_csv(search_request)[model]
self.assertEqual(exported_data, obj_dicts)
def _is_parent_for(parent, obj):
"""Check that 'parent' is parent for 'obj'
Cache parents ids for mega"""
if hasattr(flask.g, "mega_parents_cache"):
mega_parents_cache = flask.g.mega_parents_cache
else:
mega_parents_cache = flask.g.mega_parents_cache = defaultdict(dict)
if obj.id in mega_parents_cache[obj.type]:
parents_ids = mega_parents_cache[obj.type][obj.id]
else:
obj = get_model(obj.type).query.get(obj.id)
parents_ids = obj.get_all_relatives_ids("parents")
mega_parents_cache[obj.type][obj.id] = parents_ids
return parent.id in parents_ids
def get(type_, id_):
"""Check flask.g.referenced_objects for the object or get it from the DB."""
if not id_:
return None
ref_objects = getattr(flask.g, "referenced_objects", {})
if not (isinstance(type_, type) and issubclass(type_, db.Model)):
type_ = inflector.get_model(type_)
result = ref_objects.get(type_, {}).get(id_, None)
if not result:
result = type_.query.get(id_)
return result
def expected_multiple_not_equal(self, setup_objs):
"""Calculate expected values for multiple test case, not equal operator"""
values, expected_data = [], []
search_obj = inflector.get_model(setup_objs[0].searchable_type).query.get(
setup_objs[0].searchable_id
)
for data in setup_objs:
if data.searchable_id != setup_objs[0].searchable_id:
expected_data.append(
create_tuple_data(data.obj_id, search_obj, self.Meta.subprops)
)
for subprop in self.Meta.subprops:
values.append(
(expected_data, getattr(search_obj, subprop))
)
return values
def get_issuetracked_objects(obj_type, obj_ids):
"""Fetch issuetracked objects from db."""
issuetracked_model = inflector.get_model(obj_type)
return issuetracked_model.query.join(
all_models.IssuetrackerIssue,
sa.and_(
all_models.IssuetrackerIssue.object_type == obj_type,
all_models.IssuetrackerIssue.object_id == issuetracked_model.id
)
).filter(
all_models.IssuetrackerIssue.object_id.in_(obj_ids),
all_models.IssuetrackerIssue.issue_id.isnot(None),
all_models.IssuetrackerIssue.enabled != 0,
).options(
sa.orm.Load(issuetracked_model).undefer_group(
"{}_complete".format(obj_type),
)
)
def import_model(self, model_name, audit, recipients, send_by_default):
"""Import model data with commentable fields"""
# pylint: disable=protected-access
import_data = [
("object_type", model_name),
("Code", "{}-1".format(model_name)),
("Title", "{}-Title".format(model_name)),
("Admin", "*****@*****.**"),
("Recipients", recipients),
("Send by default", send_by_default),
]
model_cls = inflector.get_model(model_name)
if issubclass(model_cls, AuditRelationship):
import_data.append(("Map:Audit", audit))
if (issubclass(model_cls, Described) and
"description" not in model_cls._aliases) or model_name == "Risk":
import_data.append(("description", "{}-Description".format(model_name)))
response = self.import_data(OrderedDict(import_data))
self._check_csv_response(response, {})
def get(type_, id_):
"""Check flask.g.referenced_objects for the object or get it from the DB."""
# id == 0 is a valid case if id is an int; therefore "not id" doesn't fit
if id_ is None:
return None
ref_objects = getattr(flask.g, "referenced_objects", {})
if not (isinstance(type_, type) and issubclass(type_, db.Model)):
type_ = inflector.get_model(type_)
# model for type_ has been removed
if type_ is None:
return None
result = ref_objects.get(type_, {}).get(id_, None)
if not result:
result = type_.query.get(id_)
return result
def _get_objects(self, object_query):
"""Get a set of objects described in the filters."""
with benchmark("Get ids: _get_objects -> _get_ids"):
ids = self._get_ids(object_query)
if not ids:
return set()
object_name = object_query["object_name"]
object_class = inflector.get_model(object_name)
query = object_class.eager_query()
query = query.filter(object_class.id.in_(ids))
with benchmark("Get objects by ids: _get_objects -> obj in query"):
id_object_map = {obj.id: obj for obj in query}
with benchmark("Order objects by ids: _get_objects"):
objects = [id_object_map[id_] for id_ in ids]
return objects
def expected_single_not_equal(self, setup_objs):
"""Calculate expected values for single test case, not equal operator"""
values = []
# Think that data type of all searchable objects is the same
model = inflector.get_model(setup_objs[0].searchable_type)
id_searchables = dict(
db.session.query(model.id, model).filter(getattr(model, "id").in_(
s.searchable_id for s in setup_objs
))
)
for data in setup_objs:
exp_data = [so for so in setup_objs if so.obj_id != data.obj_id]
search_obj = id_searchables[data.searchable_id]
for subprop in self.Meta.subprops:
values.append((
[create_tuple_data(
ed.obj_id, id_searchables[ed.searchable_id], self.Meta.subprops
) for ed in exp_data],
getattr(search_obj, subprop))
)
return values
def __init__(self):
"""Set up basic test fixture with the following data:
- Persons and Roles
- Searchable model instance
- basic test cases
"""
if not db.engine.dialect.has_table(db.engine, SetupData.__tablename__):
SetupData.__table__.create(db.engine)
objects = []
init_globals(Types.all)
with factories.single_commit():
audit_id = factories.AuditFactory().id
for model in Types.all:
for field in FIELD_SETTERS:
if field_exists(inflector.get_model(model), field):
for operation in OPERATIONS:
objects += self.base_single_setup(model, field, operation)
objects += self.base_multiple_setup(model, field, operation)
with app.app.app_context():
create_reindexed_snapshots(audit_id, objects)
def generate_classes(models, field):
"""Generate test class containing all BasePersonQueryApiTest tests"""
def generate(model):
"""Create test class with meta information"""
meta_fields = {"field": field, "model": model}
meta = type("Meta", (object,), meta_fields)
class_name = "Test{}FilterBy{}".format(
model.__name__.replace(" ", ""),
field.replace(" ", "_")
)
instance = type(
class_name,
(BasePersonQueryApiTest,),
{"Meta": meta}
)
return instance
module = sys.modules[__name__]
for model in models:
test_model = generate(inflector.get_model(model))
setattr(module, test_model.__name__, test_model)
def _get_missing_models_query(role):
"""Get query for objects with a missing ACL entry for the given role.
Note that the filter flag here is just for optimization. When creating a new
access control role we can be sure that no object will have the ACL entry for
that role so we can skip the filter.
Args:
role: role object for which we want to check acl entries.
filter_: A flag to filter the possible objects that might already have the
given acl entry
Returns:
sqlalchemy query for the object type specified in the access control role.
"""
model = inflector.get_model(role.object_type)
if not model:
# We only log info instead of warning here because we still leave access
# control roles of obsolete objects in our database, so that we can use
# them with old revisions in our history log.
logger.info("Trying to handle role '%s' for non existent object '%s'",
role.name, role.object_type)
return None
query = model.query.outerjoin(
all_models.AccessControlList,
sa.and_(
all_models.AccessControlList.object_type == model.__name__,
all_models.AccessControlList.object_id == model.id,
all_models.AccessControlList.ac_role_id == role.id
)
).filter(
all_models.AccessControlList.id.is_(None)
).order_by(
model.id
)
return query
def _get_object(self, request_json):
"""Return existing object or raise HTTP error"""
object_type = request_json.get('object_type')
object_id = request_json.get('object_id')
if None in (object_type, object_id):
raise exceptions.BadRequest(
"'object_id' and 'object_type' must be specified")
model = inflector.get_model(object_type)
if model is None or not issubclass(model, mixins.Folderable):
raise exceptions.BadRequest("Model {} not found".format(object_type))
obj = model.query.get(object_id)
if obj is None:
raise exceptions.NotFound(
"{} with id {} not found".format(model.__name__, object_id))
self._ensure_has_permissions(obj)
self._validate_readonly_access(obj)
return obj
