def upsert_all(sender, obj=None, src=None, service=None, event=None): # noqa # pylint: disable=unused-argument
"""Update snapshots globally"""
snapshot_settings = src.get("snapshots")
if snapshot_settings:
if snapshot_settings["operation"] == "upsert":
revisions = {
(Stub.from_dict(revision["parent"]),
Stub.from_dict(revision["child"])): revision["revision_id"]
for revision in snapshot_settings.get("revisions", {})}
upsert_snapshots(obj, event, revisions=revisions)
def upsert_all(sender, obj=None, src=None, service=None, event=None): # noqa # pylint: disable=unused-argument
"""Update snapshots globally"""
snapshot_settings = src.get("snapshots")
if snapshot_settings:
if snapshot_settings["operation"] == "upsert":
revisions = {
(Stub.from_dict(revision["parent"]),
Stub.from_dict(revision["child"])): revision["revision_id"]
for revision in snapshot_settings.get("revisions", {})
}
upsert_snapshots(obj, event, revisions=revisions)
def upsert_all(
sender, obj=None, src=None, service=None,
event=None, initial_state=None): # noqa
"""Updates snapshots globally."""
del sender, service, initial_state # Unused
snapshot_settings = src.get("snapshots")
if snapshot_settings:
if snapshot_settings["operation"] == "upsert":
revisions = {
(Stub.from_dict(revision["parent"]),
Stub.from_dict(revision["child"])): revision["revision_id"]
for revision in snapshot_settings.get("revisions", {})}
upsert_snapshots(obj, event, revisions=revisions)
def upsert_all(
sender, obj=None, src=None, service=None,
event=None, initial_state=None): # noqa
"""Updates snapshots globally."""
del sender, service, initial_state # Unused
snapshot_settings = src.get("snapshots")
if snapshot_settings:
if snapshot_settings["operation"] == "upsert":
revisions = {
(Stub.from_dict(revision["parent"]),
Stub.from_dict(revision["child"])): revision["revision_id"]
for revision in snapshot_settings.get("revisions", {})}
upsert_snapshots(obj, event, revisions=revisions)
def clone_scope(base_parent, new_parent, event):
"""Create exact copy of parent object scope.
Args:
base_parent: Old parent object
new_parent: New parent object
event: Event that triggered scope cloning
"""
with benchmark("clone_scope.clone audit scope"):
source_snapshots = db.session.query(
models.Snapshot.child_type, models.Snapshot.child_id,
models.Snapshot.revision_id).filter(
models.Snapshot.parent_type == base_parent.type,
models.Snapshot.parent_id == base_parent.id)
snapshot_revisions = {
Pair.from_4tuple((new_parent.type, new_parent.id, ctype, cid)):
revid
for ctype, cid, revid in source_snapshots
}
parent = Stub(new_parent.type, new_parent.id)
children = {pair.child for pair in snapshot_revisions}
generator = SnapshotGenerator(dry_run=False)
generator.add_family(parent, children)
generator.create(event, snapshot_revisions)
def _get_snapshottable_objects(self, obj):
"""Get snapshottable objects from parent object's neighborhood."""
with benchmark("Snapshot._get_snapshotable_objects"):
related_mappings = set()
object_rules = self.rules.rules[obj.type]
with benchmark("Snapshot._get_snapshotable_objects.related_mappings"):
relatable_rules = {
rule for rule in object_rules["fst"]
if isinstance(rule, basestring)
}
if relatable_rules:
related_mappings = obj.related_objects({
rule for rule in object_rules["fst"]
if isinstance(rule, basestring)})
with benchmark("Snapshot._get_snapshotable_objects.direct mappings"):
direct_mappings = {getattr(obj, rule.name)
for rule in object_rules["fst"]
if isinstance(rule, Attr)}
related_objects = {Stub.from_object(obj)
for obj in related_mappings | direct_mappings}
with benchmark("Snapshot._get_snapshotable_objects.fetch neighborhood"):
return self._fetch_neighborhood(obj, related_objects)
def _get_snapshottable_objects(self, obj):
"""Get snapshottable objects from parent object's neighborhood."""
with benchmark("Snapshot._get_snapshotable_objects"):
related_mappings = set()
object_rules = self.rules.rules[obj.type]
with benchmark("Snapshot._get_snapshotable_objects.related_mappings"):
relatable_rules = {
rule for rule in object_rules["fst"]
if isinstance(rule, basestring)
}
if relatable_rules:
related_mappings = obj.related_objects({
rule for rule in object_rules["fst"]
if isinstance(rule, basestring)})
with benchmark("Snapshot._get_snapshotable_objects.direct mappings"):
direct_mappings = {getattr(obj, rule.name)
for rule in object_rules["fst"]
if isinstance(rule, Attr)}
related_objects = {Stub.from_object(obj)
for obj in related_mappings | direct_mappings}
with benchmark("Snapshot._get_snapshotable_objects.fetch neighborhood"):
return self._fetch_neighborhood(obj, related_objects)
def get_revisions(pairs, revisions, filters=None):
"""Retrieve revision ids for pairs
If revisions dictionary is provided it will validate that the selected
revision exists in the objects revision history.
Args:
pairs: set([(parent_1, child_1), (parent_2, child_2), ...])
revisions: dict({(parent, child): revision_id, ...})
filters: predicate
"""
with benchmark("snapshotter.helpers.get_revisions"):
revision_id_cache = dict()
if pairs:
with benchmark("get_revisions.create caches"):
child_stubs = {pair.child for pair in pairs}
with benchmark("get_revisions.create child -> parents cache"):
parents_cache = collections.defaultdict(set)
for parent, child in pairs:
parents_cache[child].add(parent)
with benchmark("get_revisions.retrieve revisions"):
query = db.session.query(
models.Revision.id, models.Revision.resource_type,
models.Revision.resource_id).filter(
tuple_(models.Revision.resource_type,
models.Revision.resource_id).in_(
child_stubs)).order_by(
models.Revision.id.desc())
if filters:
for _filter in filters:
query = query.filter(_filter)
with benchmark("get_revisions.create revision_id cache"):
for revid, restype, resid in query:
child = Stub(restype, resid)
for parent in parents_cache[child]:
key = Pair(parent, child)
if key in revisions:
if revid == revisions[key]:
revision_id_cache[key] = revid
else:
logger.warning(
"Specified revision for object %s but couldn't find the"
"revision '%s' in object history", key,
revisions[key])
else:
if key not in revision_id_cache:
revision_id_cache[key] = revid
return revision_id_cache
def add_parent(self, obj):
"""Add parent object and automatically scan neighborhood for snapshottable
objects."""
with benchmark("Snapshot.add_parent_object"):
key = Stub.from_object(obj)
if key not in self.parents:
with benchmark("Snapshot.add_parent_object.add object"):
objs = self._get_snapshottable_objects(obj)
self.parents.add(key)
self.context_cache[key] = obj.context_id
self.children = self.children | objs
self.snapshots[key] = objs
return self.parents
def add_parent(self, obj):
"""Add parent object and automatically scan neighborhood for snapshottable
objects."""
with benchmark("Snapshot.add_parent_object"):
key = Stub.from_object(obj)
if key not in self.parents:
with benchmark("Snapshot.add_parent_object.add object"):
objs = self._get_snapshottable_objects(obj)
self.parents.add(key)
self.context_cache[key] = obj.context_id
self.children = self.children | objs
self.snapshots[key] = objs
return self.parents
def _fetch_neighborhood(self, parent_object, objects):
"""Fetch relationships for objects and parent."""
with benchmark("Snapshot._fetch_object_neighborhood"):
query_pairs = set()
for obj in objects:
for snd_obj in self.rules.rules[parent_object.type]["snd"]:
query_pairs.add((obj.type, obj.id, snd_obj))
columns = db.session.query(models.Relationship.source_type,
models.Relationship.source_id,
models.Relationship.destination_type,
models.Relationship.destination_id)
relationships = columns.filter(
tuple_(
models.Relationship.destination_type,
models.Relationship.destination_id,
models.Relationship.source_type,
).in_(query_pairs)).union(
columns.filter(
tuple_(
models.Relationship.source_type,
models.Relationship.source_id,
models.Relationship.destination_type,
).in_(query_pairs)))
neighborhood = set()
for (stype, sid, dtype, did) in relationships:
source = Stub(stype, sid)
destination = Stub(dtype, did)
if source in objects:
neighborhood.add(destination)
else:
neighborhood.add(source)
return neighborhood
def get_revisions(pairs, revisions, filters=None):
"""Retrieve revision ids for pairs
Args:
pairs: set([(parent_1, child_1), (parent_2, child_2), ...])
revisions: dict({(parent, child): revision_id, ...})
filters: predicate
"""
with benchmark("snapshotter.helpers.get_revisions"):
revision_id_cache = dict()
if pairs:
with benchmark("get_revisions.create caches"):
child_stubs = {pair.child for pair in pairs}
with benchmark("get_revisions.create child -> parents cache"):
parents_cache = collections.defaultdict(set)
for parent, child in pairs:
parents_cache[child].add(parent)
with benchmark("get_revisions.retrieve revisions"):
query = db.session.query(
models.Revision.id, models.Revision.resource_type,
models.Revision.resource_id).filter(
tuple_(models.Revision.resource_type,
models.Revision.resource_id).in_(
child_stubs)).order_by(
models.Revision.id.desc())
if filters:
for _filter in filters:
query = query.filter(_filter)
with benchmark("get_revisions.create revision_id cache"):
for revid, restype, resid in query:
child = Stub(restype, resid)
for parent in parents_cache[child]:
key = Pair(parent, child)
if key in revisions:
if revid == revisions[key]:
revision_id_cache[key] = revid
else:
if key not in revision_id_cache:
revision_id_cache[key] = revid
return revision_id_cache
def get_revisions(pairs, revisions, filters=None):
"""Retrieve revision ids for pairs
If revisions dictionary is provided it will validate that the selected
revision exists in the objects revision history.
Args:
pairs: set([(parent_1, child_1), (parent_2, child_2), ...])
revisions: dict({(parent, child): revision_id, ...})
filters: predicate
"""
with benchmark("snapshotter.helpers.get_revisions"):
if not pairs:
return {}
with benchmark("get_revisions.create child -> parents cache"):
parents_cache = collections.defaultdict(set)
child_stubs = set()
for parent, child in pairs:
parents_cache[child].add(parent)
child_stubs.add(child)
with benchmark("get_revisions.retrieve revisions"):
query = get_revisions_query(child_stubs, revisions, filters)
revision_id_cache = {}
with benchmark("get_revisions.create revision_id cache"):
for revid, restype, resid in query:
child = Stub(restype, resid)
for parent in parents_cache[child]:
key = Pair(parent, child)
if key in revisions and revisions[key] != revid:
logger.warning(
"Specified revision for object %s but couldn't find the"
"revision '%s' in object history",
key,
revisions[key],
)
else:
revision_id_cache[key] = revid
return revision_id_cache
def _create(self, for_create, event, revisions, _filter):
"""Create snapshots of parent objects neighhood and create revisions for
snapshots.
Args:
event: A ggrc.models.Event instance
revisions: A set of tuples of pairs with revisions to which it should
either create or update a snapshot of that particular audit
_filter: Callable that should return True if it should be updated
Returns:
OperationResponse
"""
# pylint: disable=too-many-locals,too-many-statements
with benchmark("Snapshot._create"):
with benchmark("Snapshot._create init"):
user_id = get_current_user_id()
missed_keys = set()
data_payload = list()
revision_payload = list()
relationship_payload = list()
response_data = dict()
if self.dry_run and event is None:
event_id = 0
else:
event_id = event.id
with benchmark("Snapshot._create.filter"):
if _filter:
for_create = {elem for elem in for_create if _filter(elem)}
with benchmark("Snapshot._create._get_revisions"):
revision_id_cache = get_revisions(for_create, revisions)
response_data["revisions"] = revision_id_cache
with benchmark("Snapshot._create.create payload"):
for pair in for_create:
if pair in revision_id_cache:
revision_id = revision_id_cache[pair]
context_id = self.context_cache[pair.parent]
data = create_snapshot_dict(pair, revision_id, user_id,
context_id)
data_payload += [data]
else:
missed_keys.add(pair)
if missed_keys:
logger.warning(
"Tried to create snapshots for the following objects but "
"found no revisions: %s", missed_keys)
with benchmark("Snapshot._create.write to database"):
self._execute(models.Snapshot.__table__.insert(), data_payload)
with benchmark("Snapshot._create.retrieve inserted snapshots"):
snapshots = get_snapshots(for_create)
with benchmark("Snapshot._create.access control list"):
acl_payload = get_acl_payload(snapshots)
with benchmark("Snapshot._create.write acls to database"):
self._execute(all_models.AccessControlList.__table__.insert(),
acl_payload)
with benchmark(
"Snapshot._create.create parent object -> snapshot rels"):
for snapshot in snapshots:
parent = Stub(snapshot.parent_type, snapshot.parent_id)
base = Stub(snapshot.child_type, snapshot.child_id)
relationship = create_relationship_dict(
parent, base, user_id, self.context_cache[parent])
relationship_payload += [relationship]
with benchmark("Snapshot._create.write relationships to database"):
self._execute(models.Relationship.__table__.insert(),
relationship_payload)
with benchmark("Snapshot._create.get created relationships"):
created_relationships = {
(rel["source_type"], rel["source_id"],
rel["destination_type"], rel["destination_id"])
for rel in relationship_payload
}
relationships = get_relationships(created_relationships)
with benchmark("Snapshot._create.create revision payload"):
with benchmark(
"Snapshot._create.create snapshots revision payload"):
for snapshot in snapshots:
parent = Stub(snapshot.parent_type, snapshot.parent_id)
context_id = self.context_cache[parent]
data = create_snapshot_revision_dict(
"created", event_id, snapshot, user_id, context_id)
revision_payload += [data]
with benchmark("Snapshot._create.create rel revision payload"):
for relationship in relationships:
parent = Stub(relationship.source_type,
relationship.source_id)
context_id = self.context_cache[parent]
data = create_relationship_revision_dict(
"created", event_id, relationship, user_id,
context_id)
revision_payload += [data]
with benchmark("Snapshot._create.write revisions to database"):
self._execute(models.Revision.__table__.insert(),
revision_payload)
return OperationResponse("create", True, for_create, response_data)
def _update(self, for_update, event, revisions, _filter):
"""Update (or create) parent objects' snapshots and create revisions for
them.
Args:
event: A ggrc.models.Event instance
revisions: A set of tuples of pairs with revisions to which it should
either create or update a snapshot of that particular audit
_filter: Callable that should return True if it should be updated
Returns:
OperationResponse
"""
# pylint: disable=too-many-locals
with benchmark("Snapshot._update"):
user_id = get_current_user_id()
missed_keys = set()
snapshot_cache = dict()
modified_snapshot_keys = set()
data_payload_update = list()
revision_payload = list()
response_data = dict()
if self.dry_run and event is None:
event_id = 0
else:
event_id = event.id
with benchmark("Snapshot._update.filter"):
if _filter:
for_update = {elem for elem in for_update if _filter(elem)}
with benchmark("Snapshot._update.get existing snapshots"):
existing_snapshots = db.session.query(
models.Snapshot.id,
models.Snapshot.revision_id,
models.Snapshot.parent_type,
models.Snapshot.parent_id,
models.Snapshot.child_type,
models.Snapshot.child_id,
).filter(
tuple_(models.Snapshot.parent_type,
models.Snapshot.parent_id,
models.Snapshot.child_type,
models.Snapshot.child_id).in_(
{pair.to_4tuple()
for pair in for_update}))
for esnap in existing_snapshots:
sid, rev_id, pair_tuple = esnap[0], esnap[1], esnap[2:]
pair = Pair.from_4tuple(pair_tuple)
snapshot_cache[pair] = (sid, rev_id)
with benchmark("Snapshot._update.retrieve latest revisions"):
revision_id_cache = get_revisions(
for_update,
filters=[
models.Revision.action.in_(["created", "modified"])
],
revisions=revisions)
response_data["revisions"] = {
"old":
{pair: values[1]
for pair, values in snapshot_cache.items()},
"new": revision_id_cache
}
with benchmark("Snapshot._update.build snapshot payload"):
for key in for_update:
if key in revision_id_cache:
sid, rev_id = snapshot_cache[key]
latest_rev = revision_id_cache[key]
if rev_id != latest_rev:
modified_snapshot_keys.add(key)
data_payload_update += [{
"_id": sid,
"_revision_id": latest_rev,
"_modified_by_id": user_id
}]
else:
missed_keys.add(key)
if missed_keys:
logger.warning(
"Tried to update snapshots for the following objects but "
"found no revisions: %s", missed_keys)
if not modified_snapshot_keys:
return OperationResponse("update", True, set(), response_data)
with benchmark("Snapshot._update.write snapshots to database"):
update_sql = models.Snapshot.__table__.update().where(
models.Snapshot.id == bindparam("_id")).values(
revision_id=bindparam("_revision_id"),
modified_by_id=bindparam("_modified_by_id"))
self._execute(update_sql, data_payload_update)
with benchmark("Snapshot._update.retrieve inserted snapshots"):
snapshots = get_snapshots(modified_snapshot_keys)
with benchmark(
"Snapshot._update.create snapshots revision payload"):
for snapshot in snapshots:
parent = Stub(snapshot.parent_type, snapshot.parent_id)
context_id = self.context_cache[parent]
data = create_snapshot_revision_dict(
"modified", event_id, snapshot, user_id, context_id)
revision_payload += [data]
with benchmark("Insert Snapshot entries into Revision"):
self._execute(models.Revision.__table__.insert(),
revision_payload)
return OperationResponse("update", True, for_update, response_data)
def _create(self, for_create, event, revisions, _filter):
"""Create snapshots of parent objects neighhood and create revisions for
snapshots.
Args:
event: A ggrc.models.Event instance
revisions: A set of tuples of pairs with revisions to which it should
either create or update a snapshot of that particular audit
_filter: Callable that should return True if it should be updated
Returns:
OperationResponse
"""
# pylint: disable=too-many-locals,too-many-statements
with benchmark("Snapshot._create"):
with benchmark("Snapshot._create init"):
user_id = get_current_user_id()
missed_keys = set()
data_payload = list()
revision_payload = list()
relationship_payload = list()
response_data = dict()
if self.dry_run and event is None:
event_id = 0
else:
event_id = event.id
with benchmark("Snapshot._create.filter"):
if _filter:
for_create = {elem for elem in for_create if _filter(elem)}
with benchmark("Snapshot._create._get_revisions"):
revision_id_cache = get_revisions(for_create, revisions)
response_data["revisions"] = revision_id_cache
with benchmark("Snapshot._create.create payload"):
for pair in for_create:
if pair in revision_id_cache:
revision_id = revision_id_cache[pair]
context_id = self.context_cache[pair.parent]
data = create_snapshot_dict(pair, revision_id, user_id, context_id)
data_payload += [data]
else:
missed_keys.add(pair)
with benchmark("Snapshot._create.write to database"):
self._execute(
models.Snapshot.__table__.insert(),
data_payload)
with benchmark("Snapshot._create.retrieve inserted snapshots"):
snapshots = get_snapshots(for_create)
with benchmark("Snapshot._create.create base object -> snapshot rels"):
for snapshot in snapshots:
base_object = Stub.from_tuple(snapshot, 6, 7)
snapshot_object = Stub("Snapshot", snapshot[0])
relationship = create_relationship_dict(base_object, snapshot_object,
user_id, snapshot[1])
relationship_payload += [relationship]
with benchmark("Snapshot._create.write relationships to database"):
self._execute(models.Relationship.__table__.insert(),
relationship_payload)
with benchmark("Snapshot._create.get created relationships"):
created_relationships = {
(rel["source_type"], rel["source_id"],
rel["destination_type"], rel["destination_id"])
for rel in relationship_payload}
relationships = get_relationships(created_relationships)
with benchmark("Snapshot._create.create revision payload"):
with benchmark("Snapshot._create.create snapshots revision payload"):
for snapshot in snapshots:
parent = Stub.from_tuple(snapshot, 4, 5)
context_id = self.context_cache[parent]
data = create_snapshot_revision_dict("created", event_id, snapshot,
user_id, context_id)
revision_payload += [data]
with benchmark("Snapshot._create.create rel revision payload"):
snapshot_parents = {pair.child: pair.parent for pair in for_create}
for relationship in relationships:
obj = Stub.from_tuple(relationship, 4, 5)
parent = snapshot_parents[obj]
context_id = self.context_cache[parent]
data = create_relationship_revision_dict(
"created", event_id, relationship, user_id, context_id)
revision_payload += [data]
with benchmark("Snapshot._create.write revisions to database"):
self._execute(models.Revision.__table__.insert(), revision_payload)
return OperationResponse("create", True, for_create, response_data)
