def _upsert(self, event, revisions, _filter):
"""Update and (if needed) create snapshots
Args:
event: A ggrc.models.Event instance
revisions: A set of tuples of pairs with revisions to which it should
either create or update a snapshot of that particular audit
_filter: Callable that should return True if it should be updated
Returns:
OperationResponse
"""
for_create, for_update = self.analyze()
create, update = None, None
created, updated = set(), set()
if for_update:
update = self._update(
for_update=for_update, event=event, revisions=revisions,
_filter=_filter)
updated = update.response
if for_create:
create = self._create(for_create=for_create, event=event,
revisions=revisions, _filter=_filter)
created = create.response
to_reindex = updated | created
if not self.dry_run:
reindex_pairs(to_reindex)
return OperationResponse("upsert", True, {
"create": create,
"update": update
}, {
"dry-run": self.dry_run
})
def _create(self, for_create, event, revisions, _filter):
"""Create snapshots of parent objects neighhood and create revisions for
snapshots.
Args:
event: A ggrc.models.Event instance
revisions: A set of tuples of pairs with revisions to which it should
either create or update a snapshot of that particular audit
_filter: Callable that should return True if it should be updated
Returns:
OperationResponse
"""
# pylint: disable=too-many-locals,too-many-statements
with benchmark("Snapshot._create"):
with benchmark("Snapshot._create init"):
user_id = get_current_user_id()
missed_keys = set()
data_payload = list()
revision_payload = list()
relationship_payload = list()
response_data = dict()
if self.dry_run and event is None:
event_id = 0
else:
event_id = event.id
with benchmark("Snapshot._create.filter"):
if _filter:
for_create = {elem for elem in for_create if _filter(elem)}
with benchmark("Snapshot._create._get_revisions"):
revision_id_cache = get_revisions(for_create, revisions)
response_data["revisions"] = revision_id_cache
with benchmark("Snapshot._create.create payload"):
for pair in for_create:
if pair in revision_id_cache:
revision_id = revision_id_cache[pair]
context_id = self.context_cache[pair.parent]
data = create_snapshot_dict(pair, revision_id, user_id,
context_id)
data_payload += [data]
else:
missed_keys.add(pair)
if missed_keys:
logger.warning(
"Tried to create snapshots for the following objects but "
"found no revisions: %s", missed_keys)
with benchmark("Snapshot._create.write to database"):
self._execute(models.Snapshot.__table__.insert(), data_payload)
with benchmark("Snapshot._create.retrieve inserted snapshots"):
snapshots = get_snapshots(for_create)
with benchmark("Snapshot._create.access control list"):
acl_payload = get_acl_payload(snapshots)
with benchmark("Snapshot._create.write acls to database"):
self._execute(all_models.AccessControlList.__table__.insert(),
acl_payload)
with benchmark(
"Snapshot._create.create parent object -> snapshot rels"):
for snapshot in snapshots:
parent = Stub(snapshot.parent_type, snapshot.parent_id)
base = Stub(snapshot.child_type, snapshot.child_id)
relationship = create_relationship_dict(
parent, base, user_id, self.context_cache[parent])
relationship_payload += [relationship]
with benchmark("Snapshot._create.write relationships to database"):
self._execute(models.Relationship.__table__.insert(),
relationship_payload)
with benchmark("Snapshot._create.get created relationships"):
created_relationships = {
(rel["source_type"], rel["source_id"],
rel["destination_type"], rel["destination_id"])
for rel in relationship_payload
}
relationships = get_relationships(created_relationships)
with benchmark("Snapshot._create.create revision payload"):
with benchmark(
"Snapshot._create.create snapshots revision payload"):
for snapshot in snapshots:
parent = Stub(snapshot.parent_type, snapshot.parent_id)
context_id = self.context_cache[parent]
data = create_snapshot_revision_dict(
"created", event_id, snapshot, user_id, context_id)
revision_payload += [data]
with benchmark("Snapshot._create.create rel revision payload"):
for relationship in relationships:
parent = Stub(relationship.source_type,
relationship.source_id)
context_id = self.context_cache[parent]
data = create_relationship_revision_dict(
"created", event_id, relationship, user_id,
context_id)
revision_payload += [data]
with benchmark("Snapshot._create.write revisions to database"):
self._execute(models.Revision.__table__.insert(),
revision_payload)
return OperationResponse("create", True, for_create, response_data)
def _update(self, for_update, event, revisions, _filter):
"""Update (or create) parent objects' snapshots and create revisions for
them.
Args:
event: A ggrc.models.Event instance
revisions: A set of tuples of pairs with revisions to which it should
either create or update a snapshot of that particular audit
_filter: Callable that should return True if it should be updated
Returns:
OperationResponse
"""
# pylint: disable=too-many-locals
with benchmark("Snapshot._update"):
user_id = get_current_user_id()
missed_keys = set()
snapshot_cache = dict()
modified_snapshot_keys = set()
data_payload_update = list()
revision_payload = list()
response_data = dict()
if self.dry_run and event is None:
event_id = 0
else:
event_id = event.id
with benchmark("Snapshot._update.filter"):
if _filter:
for_update = {elem for elem in for_update if _filter(elem)}
with benchmark("Snapshot._update.get existing snapshots"):
existing_snapshots = db.session.query(
models.Snapshot.id,
models.Snapshot.revision_id,
models.Snapshot.parent_type,
models.Snapshot.parent_id,
models.Snapshot.child_type,
models.Snapshot.child_id,
).filter(
tuple_(models.Snapshot.parent_type,
models.Snapshot.parent_id,
models.Snapshot.child_type,
models.Snapshot.child_id).in_(
{pair.to_4tuple()
for pair in for_update}))
for esnap in existing_snapshots:
sid, rev_id, pair_tuple = esnap[0], esnap[1], esnap[2:]
pair = Pair.from_4tuple(pair_tuple)
snapshot_cache[pair] = (sid, rev_id)
with benchmark("Snapshot._update.retrieve latest revisions"):
revision_id_cache = get_revisions(
for_update,
filters=[
models.Revision.action.in_(["created", "modified"])
],
revisions=revisions)
response_data["revisions"] = {
"old":
{pair: values[1]
for pair, values in snapshot_cache.items()},
"new": revision_id_cache
}
with benchmark("Snapshot._update.build snapshot payload"):
for key in for_update:
if key in revision_id_cache:
sid, rev_id = snapshot_cache[key]
latest_rev = revision_id_cache[key]
if rev_id != latest_rev:
modified_snapshot_keys.add(key)
data_payload_update += [{
"_id": sid,
"_revision_id": latest_rev,
"_modified_by_id": user_id
}]
else:
missed_keys.add(key)
if missed_keys:
logger.warning(
"Tried to update snapshots for the following objects but "
"found no revisions: %s", missed_keys)
if not modified_snapshot_keys:
return OperationResponse("update", True, set(), response_data)
with benchmark("Snapshot._update.write snapshots to database"):
update_sql = models.Snapshot.__table__.update().where(
models.Snapshot.id == bindparam("_id")).values(
revision_id=bindparam("_revision_id"),
modified_by_id=bindparam("_modified_by_id"))
self._execute(update_sql, data_payload_update)
with benchmark("Snapshot._update.retrieve inserted snapshots"):
snapshots = get_snapshots(modified_snapshot_keys)
with benchmark(
"Snapshot._update.create snapshots revision payload"):
for snapshot in snapshots:
parent = Stub(snapshot.parent_type, snapshot.parent_id)
context_id = self.context_cache[parent]
data = create_snapshot_revision_dict(
"modified", event_id, snapshot, user_id, context_id)
revision_payload += [data]
with benchmark("Insert Snapshot entries into Revision"):
self._execute(models.Revision.__table__.insert(),
revision_payload)
return OperationResponse("update", True, for_update, response_data)
def _create(self, for_create, event, revisions, _filter):
"""Create snapshots of parent objects neighhood and create revisions for
snapshots.
Args:
event: A ggrc.models.Event instance
revisions: A set of tuples of pairs with revisions to which it should
either create or update a snapshot of that particular audit
_filter: Callable that should return True if it should be updated
Returns:
OperationResponse
"""
# pylint: disable=too-many-locals,too-many-statements
with benchmark("Snapshot._create"):
with benchmark("Snapshot._create init"):
user_id = get_current_user_id()
missed_keys = set()
data_payload = list()
revision_payload = list()
relationship_payload = list()
response_data = dict()
if self.dry_run and event is None:
event_id = 0
else:
event_id = event.id
with benchmark("Snapshot._create.filter"):
if _filter:
for_create = {elem for elem in for_create if _filter(elem)}
with benchmark("Snapshot._create._get_revisions"):
revision_id_cache = get_revisions(for_create, revisions)
response_data["revisions"] = revision_id_cache
with benchmark("Snapshot._create.create payload"):
for pair in for_create:
if pair in revision_id_cache:
revision_id = revision_id_cache[pair]
context_id = self.context_cache[pair.parent]
data = create_snapshot_dict(pair, revision_id, user_id, context_id)
data_payload += [data]
else:
missed_keys.add(pair)
with benchmark("Snapshot._create.write to database"):
self._execute(
models.Snapshot.__table__.insert(),
data_payload)
with benchmark("Snapshot._create.retrieve inserted snapshots"):
snapshots = get_snapshots(for_create)
with benchmark("Snapshot._create.create base object -> snapshot rels"):
for snapshot in snapshots:
base_object = Stub.from_tuple(snapshot, 6, 7)
snapshot_object = Stub("Snapshot", snapshot[0])
relationship = create_relationship_dict(base_object, snapshot_object,
user_id, snapshot[1])
relationship_payload += [relationship]
with benchmark("Snapshot._create.write relationships to database"):
self._execute(models.Relationship.__table__.insert(),
relationship_payload)
with benchmark("Snapshot._create.get created relationships"):
created_relationships = {
(rel["source_type"], rel["source_id"],
rel["destination_type"], rel["destination_id"])
for rel in relationship_payload}
relationships = get_relationships(created_relationships)
with benchmark("Snapshot._create.create revision payload"):
with benchmark("Snapshot._create.create snapshots revision payload"):
for snapshot in snapshots:
parent = Stub.from_tuple(snapshot, 4, 5)
context_id = self.context_cache[parent]
data = create_snapshot_revision_dict("created", event_id, snapshot,
user_id, context_id)
revision_payload += [data]
with benchmark("Snapshot._create.create rel revision payload"):
snapshot_parents = {pair.child: pair.parent for pair in for_create}
for relationship in relationships:
obj = Stub.from_tuple(relationship, 4, 5)
parent = snapshot_parents[obj]
context_id = self.context_cache[parent]
data = create_relationship_revision_dict(
"created", event_id, relationship, user_id, context_id)
revision_payload += [data]
with benchmark("Snapshot._create.write revisions to database"):
self._execute(models.Revision.__table__.insert(), revision_payload)
return OperationResponse("create", True, for_create, response_data)