def upsert_all(sender, obj=None, src=None, service=None, event=None): # noqa # pylint: disable=unused-argument """Update snapshots globally""" snapshot_settings = src.get("snapshots") if snapshot_settings: if snapshot_settings["operation"] == "upsert": revisions = { (Stub.from_dict(revision["parent"]), Stub.from_dict(revision["child"])): revision["revision_id"] for revision in snapshot_settings.get("revisions", {})} upsert_snapshots(obj, event, revisions=revisions)
def upsert_all(sender, obj=None, src=None, service=None, event=None): # noqa # pylint: disable=unused-argument """Update snapshots globally""" snapshot_settings = src.get("snapshots") if snapshot_settings: if snapshot_settings["operation"] == "upsert": revisions = { (Stub.from_dict(revision["parent"]), Stub.from_dict(revision["child"])): revision["revision_id"] for revision in snapshot_settings.get("revisions", {}) } upsert_snapshots(obj, event, revisions=revisions)
def upsert_all( sender, obj=None, src=None, service=None, event=None, initial_state=None): # noqa """Updates snapshots globally.""" del sender, service, initial_state # Unused snapshot_settings = src.get("snapshots") if snapshot_settings: if snapshot_settings["operation"] == "upsert": revisions = { (Stub.from_dict(revision["parent"]), Stub.from_dict(revision["child"])): revision["revision_id"] for revision in snapshot_settings.get("revisions", {})} upsert_snapshots(obj, event, revisions=revisions)
def clone_scope(base_parent, new_parent, event): """Create exact copy of parent object scope. Args: base_parent: Old parent object new_parent: New parent object event: Event that triggered scope cloning """ with benchmark("clone_scope.clone audit scope"): source_snapshots = db.session.query( models.Snapshot.child_type, models.Snapshot.child_id, models.Snapshot.revision_id).filter( models.Snapshot.parent_type == base_parent.type, models.Snapshot.parent_id == base_parent.id) snapshot_revisions = { Pair.from_4tuple((new_parent.type, new_parent.id, ctype, cid)): revid for ctype, cid, revid in source_snapshots } parent = Stub(new_parent.type, new_parent.id) children = {pair.child for pair in snapshot_revisions} generator = SnapshotGenerator(dry_run=False) generator.add_family(parent, children) generator.create(event, snapshot_revisions)
def _get_snapshottable_objects(self, obj): """Get snapshottable objects from parent object's neighborhood.""" with benchmark("Snapshot._get_snapshotable_objects"): related_mappings = set() object_rules = self.rules.rules[obj.type] with benchmark("Snapshot._get_snapshotable_objects.related_mappings"): relatable_rules = { rule for rule in object_rules["fst"] if isinstance(rule, basestring) } if relatable_rules: related_mappings = obj.related_objects({ rule for rule in object_rules["fst"] if isinstance(rule, basestring)}) with benchmark("Snapshot._get_snapshotable_objects.direct mappings"): direct_mappings = {getattr(obj, rule.name) for rule in object_rules["fst"] if isinstance(rule, Attr)} related_objects = {Stub.from_object(obj) for obj in related_mappings | direct_mappings} with benchmark("Snapshot._get_snapshotable_objects.fetch neighborhood"): return self._fetch_neighborhood(obj, related_objects)
def get_revisions(pairs, revisions, filters=None): """Retrieve revision ids for pairs If revisions dictionary is provided it will validate that the selected revision exists in the objects revision history. Args: pairs: set([(parent_1, child_1), (parent_2, child_2), ...]) revisions: dict({(parent, child): revision_id, ...}) filters: predicate """ with benchmark("snapshotter.helpers.get_revisions"): revision_id_cache = dict() if pairs: with benchmark("get_revisions.create caches"): child_stubs = {pair.child for pair in pairs} with benchmark("get_revisions.create child -> parents cache"): parents_cache = collections.defaultdict(set) for parent, child in pairs: parents_cache[child].add(parent) with benchmark("get_revisions.retrieve revisions"): query = db.session.query( models.Revision.id, models.Revision.resource_type, models.Revision.resource_id).filter( tuple_(models.Revision.resource_type, models.Revision.resource_id).in_( child_stubs)).order_by( models.Revision.id.desc()) if filters: for _filter in filters: query = query.filter(_filter) with benchmark("get_revisions.create revision_id cache"): for revid, restype, resid in query: child = Stub(restype, resid) for parent in parents_cache[child]: key = Pair(parent, child) if key in revisions: if revid == revisions[key]: revision_id_cache[key] = revid else: logger.warning( "Specified revision for object %s but couldn't find the" "revision '%s' in object history", key, revisions[key]) else: if key not in revision_id_cache: revision_id_cache[key] = revid return revision_id_cache
def add_parent(self, obj): """Add parent object and automatically scan neighborhood for snapshottable objects.""" with benchmark("Snapshot.add_parent_object"): key = Stub.from_object(obj) if key not in self.parents: with benchmark("Snapshot.add_parent_object.add object"): objs = self._get_snapshottable_objects(obj) self.parents.add(key) self.context_cache[key] = obj.context_id self.children = self.children | objs self.snapshots[key] = objs return self.parents
def _fetch_neighborhood(self, parent_object, objects): """Fetch relationships for objects and parent.""" with benchmark("Snapshot._fetch_object_neighborhood"): query_pairs = set() for obj in objects: for snd_obj in self.rules.rules[parent_object.type]["snd"]: query_pairs.add((obj.type, obj.id, snd_obj)) columns = db.session.query(models.Relationship.source_type, models.Relationship.source_id, models.Relationship.destination_type, models.Relationship.destination_id) relationships = columns.filter( tuple_( models.Relationship.destination_type, models.Relationship.destination_id, models.Relationship.source_type, ).in_(query_pairs)).union( columns.filter( tuple_( models.Relationship.source_type, models.Relationship.source_id, models.Relationship.destination_type, ).in_(query_pairs))) neighborhood = set() for (stype, sid, dtype, did) in relationships: source = Stub(stype, sid) destination = Stub(dtype, did) if source in objects: neighborhood.add(destination) else: neighborhood.add(source) return neighborhood
def get_revisions(pairs, revisions, filters=None): """Retrieve revision ids for pairs Args: pairs: set([(parent_1, child_1), (parent_2, child_2), ...]) revisions: dict({(parent, child): revision_id, ...}) filters: predicate """ with benchmark("snapshotter.helpers.get_revisions"): revision_id_cache = dict() if pairs: with benchmark("get_revisions.create caches"): child_stubs = {pair.child for pair in pairs} with benchmark("get_revisions.create child -> parents cache"): parents_cache = collections.defaultdict(set) for parent, child in pairs: parents_cache[child].add(parent) with benchmark("get_revisions.retrieve revisions"): query = db.session.query( models.Revision.id, models.Revision.resource_type, models.Revision.resource_id).filter( tuple_(models.Revision.resource_type, models.Revision.resource_id).in_( child_stubs)).order_by( models.Revision.id.desc()) if filters: for _filter in filters: query = query.filter(_filter) with benchmark("get_revisions.create revision_id cache"): for revid, restype, resid in query: child = Stub(restype, resid) for parent in parents_cache[child]: key = Pair(parent, child) if key in revisions: if revid == revisions[key]: revision_id_cache[key] = revid else: if key not in revision_id_cache: revision_id_cache[key] = revid return revision_id_cache
def get_revisions(pairs, revisions, filters=None): """Retrieve revision ids for pairs If revisions dictionary is provided it will validate that the selected revision exists in the objects revision history. Args: pairs: set([(parent_1, child_1), (parent_2, child_2), ...]) revisions: dict({(parent, child): revision_id, ...}) filters: predicate """ with benchmark("snapshotter.helpers.get_revisions"): if not pairs: return {} with benchmark("get_revisions.create child -> parents cache"): parents_cache = collections.defaultdict(set) child_stubs = set() for parent, child in pairs: parents_cache[child].add(parent) child_stubs.add(child) with benchmark("get_revisions.retrieve revisions"): query = get_revisions_query(child_stubs, revisions, filters) revision_id_cache = {} with benchmark("get_revisions.create revision_id cache"): for revid, restype, resid in query: child = Stub(restype, resid) for parent in parents_cache[child]: key = Pair(parent, child) if key in revisions and revisions[key] != revid: logger.warning( "Specified revision for object %s but couldn't find the" "revision '%s' in object history", key, revisions[key], ) else: revision_id_cache[key] = revid return revision_id_cache
def _create(self, for_create, event, revisions, _filter): """Create snapshots of parent objects neighhood and create revisions for snapshots. Args: event: A ggrc.models.Event instance revisions: A set of tuples of pairs with revisions to which it should either create or update a snapshot of that particular audit _filter: Callable that should return True if it should be updated Returns: OperationResponse """ # pylint: disable=too-many-locals,too-many-statements with benchmark("Snapshot._create"): with benchmark("Snapshot._create init"): user_id = get_current_user_id() missed_keys = set() data_payload = list() revision_payload = list() relationship_payload = list() response_data = dict() if self.dry_run and event is None: event_id = 0 else: event_id = event.id with benchmark("Snapshot._create.filter"): if _filter: for_create = {elem for elem in for_create if _filter(elem)} with benchmark("Snapshot._create._get_revisions"): revision_id_cache = get_revisions(for_create, revisions) response_data["revisions"] = revision_id_cache with benchmark("Snapshot._create.create payload"): for pair in for_create: if pair in revision_id_cache: revision_id = revision_id_cache[pair] context_id = self.context_cache[pair.parent] data = create_snapshot_dict(pair, revision_id, user_id, context_id) data_payload += [data] else: missed_keys.add(pair) if missed_keys: logger.warning( "Tried to create snapshots for the following objects but " "found no revisions: %s", missed_keys) with benchmark("Snapshot._create.write to database"): self._execute(models.Snapshot.__table__.insert(), data_payload) with benchmark("Snapshot._create.retrieve inserted snapshots"): snapshots = get_snapshots(for_create) with benchmark("Snapshot._create.access control list"): acl_payload = get_acl_payload(snapshots) with benchmark("Snapshot._create.write acls to database"): self._execute(all_models.AccessControlList.__table__.insert(), acl_payload) with benchmark( "Snapshot._create.create parent object -> snapshot rels"): for snapshot in snapshots: parent = Stub(snapshot.parent_type, snapshot.parent_id) base = Stub(snapshot.child_type, snapshot.child_id) relationship = create_relationship_dict( parent, base, user_id, self.context_cache[parent]) relationship_payload += [relationship] with benchmark("Snapshot._create.write relationships to database"): self._execute(models.Relationship.__table__.insert(), relationship_payload) with benchmark("Snapshot._create.get created relationships"): created_relationships = { (rel["source_type"], rel["source_id"], rel["destination_type"], rel["destination_id"]) for rel in relationship_payload } relationships = get_relationships(created_relationships) with benchmark("Snapshot._create.create revision payload"): with benchmark( "Snapshot._create.create snapshots revision payload"): for snapshot in snapshots: parent = Stub(snapshot.parent_type, snapshot.parent_id) context_id = self.context_cache[parent] data = create_snapshot_revision_dict( "created", event_id, snapshot, user_id, context_id) revision_payload += [data] with benchmark("Snapshot._create.create rel revision payload"): for relationship in relationships: parent = Stub(relationship.source_type, relationship.source_id) context_id = self.context_cache[parent] data = create_relationship_revision_dict( "created", event_id, relationship, user_id, context_id) revision_payload += [data] with benchmark("Snapshot._create.write revisions to database"): self._execute(models.Revision.__table__.insert(), revision_payload) return OperationResponse("create", True, for_create, response_data)
def _update(self, for_update, event, revisions, _filter): """Update (or create) parent objects' snapshots and create revisions for them. Args: event: A ggrc.models.Event instance revisions: A set of tuples of pairs with revisions to which it should either create or update a snapshot of that particular audit _filter: Callable that should return True if it should be updated Returns: OperationResponse """ # pylint: disable=too-many-locals with benchmark("Snapshot._update"): user_id = get_current_user_id() missed_keys = set() snapshot_cache = dict() modified_snapshot_keys = set() data_payload_update = list() revision_payload = list() response_data = dict() if self.dry_run and event is None: event_id = 0 else: event_id = event.id with benchmark("Snapshot._update.filter"): if _filter: for_update = {elem for elem in for_update if _filter(elem)} with benchmark("Snapshot._update.get existing snapshots"): existing_snapshots = db.session.query( models.Snapshot.id, models.Snapshot.revision_id, models.Snapshot.parent_type, models.Snapshot.parent_id, models.Snapshot.child_type, models.Snapshot.child_id, ).filter( tuple_(models.Snapshot.parent_type, models.Snapshot.parent_id, models.Snapshot.child_type, models.Snapshot.child_id).in_( {pair.to_4tuple() for pair in for_update})) for esnap in existing_snapshots: sid, rev_id, pair_tuple = esnap[0], esnap[1], esnap[2:] pair = Pair.from_4tuple(pair_tuple) snapshot_cache[pair] = (sid, rev_id) with benchmark("Snapshot._update.retrieve latest revisions"): revision_id_cache = get_revisions( for_update, filters=[ models.Revision.action.in_(["created", "modified"]) ], revisions=revisions) response_data["revisions"] = { "old": {pair: values[1] for pair, values in snapshot_cache.items()}, "new": revision_id_cache } with benchmark("Snapshot._update.build snapshot payload"): for key in for_update: if key in revision_id_cache: sid, rev_id = snapshot_cache[key] latest_rev = revision_id_cache[key] if rev_id != latest_rev: modified_snapshot_keys.add(key) data_payload_update += [{ "_id": sid, "_revision_id": latest_rev, "_modified_by_id": user_id }] else: missed_keys.add(key) if missed_keys: logger.warning( "Tried to update snapshots for the following objects but " "found no revisions: %s", missed_keys) if not modified_snapshot_keys: return OperationResponse("update", True, set(), response_data) with benchmark("Snapshot._update.write snapshots to database"): update_sql = models.Snapshot.__table__.update().where( models.Snapshot.id == bindparam("_id")).values( revision_id=bindparam("_revision_id"), modified_by_id=bindparam("_modified_by_id")) self._execute(update_sql, data_payload_update) with benchmark("Snapshot._update.retrieve inserted snapshots"): snapshots = get_snapshots(modified_snapshot_keys) with benchmark( "Snapshot._update.create snapshots revision payload"): for snapshot in snapshots: parent = Stub(snapshot.parent_type, snapshot.parent_id) context_id = self.context_cache[parent] data = create_snapshot_revision_dict( "modified", event_id, snapshot, user_id, context_id) revision_payload += [data] with benchmark("Insert Snapshot entries into Revision"): self._execute(models.Revision.__table__.insert(), revision_payload) return OperationResponse("update", True, for_update, response_data)
def _create(self, for_create, event, revisions, _filter): """Create snapshots of parent objects neighhood and create revisions for snapshots. Args: event: A ggrc.models.Event instance revisions: A set of tuples of pairs with revisions to which it should either create or update a snapshot of that particular audit _filter: Callable that should return True if it should be updated Returns: OperationResponse """ # pylint: disable=too-many-locals,too-many-statements with benchmark("Snapshot._create"): with benchmark("Snapshot._create init"): user_id = get_current_user_id() missed_keys = set() data_payload = list() revision_payload = list() relationship_payload = list() response_data = dict() if self.dry_run and event is None: event_id = 0 else: event_id = event.id with benchmark("Snapshot._create.filter"): if _filter: for_create = {elem for elem in for_create if _filter(elem)} with benchmark("Snapshot._create._get_revisions"): revision_id_cache = get_revisions(for_create, revisions) response_data["revisions"] = revision_id_cache with benchmark("Snapshot._create.create payload"): for pair in for_create: if pair in revision_id_cache: revision_id = revision_id_cache[pair] context_id = self.context_cache[pair.parent] data = create_snapshot_dict(pair, revision_id, user_id, context_id) data_payload += [data] else: missed_keys.add(pair) with benchmark("Snapshot._create.write to database"): self._execute( models.Snapshot.__table__.insert(), data_payload) with benchmark("Snapshot._create.retrieve inserted snapshots"): snapshots = get_snapshots(for_create) with benchmark("Snapshot._create.create base object -> snapshot rels"): for snapshot in snapshots: base_object = Stub.from_tuple(snapshot, 6, 7) snapshot_object = Stub("Snapshot", snapshot[0]) relationship = create_relationship_dict(base_object, snapshot_object, user_id, snapshot[1]) relationship_payload += [relationship] with benchmark("Snapshot._create.write relationships to database"): self._execute(models.Relationship.__table__.insert(), relationship_payload) with benchmark("Snapshot._create.get created relationships"): created_relationships = { (rel["source_type"], rel["source_id"], rel["destination_type"], rel["destination_id"]) for rel in relationship_payload} relationships = get_relationships(created_relationships) with benchmark("Snapshot._create.create revision payload"): with benchmark("Snapshot._create.create snapshots revision payload"): for snapshot in snapshots: parent = Stub.from_tuple(snapshot, 4, 5) context_id = self.context_cache[parent] data = create_snapshot_revision_dict("created", event_id, snapshot, user_id, context_id) revision_payload += [data] with benchmark("Snapshot._create.create rel revision payload"): snapshot_parents = {pair.child: pair.parent for pair in for_create} for relationship in relationships: obj = Stub.from_tuple(relationship, 4, 5) parent = snapshot_parents[obj] context_id = self.context_cache[parent] data = create_relationship_revision_dict( "created", event_id, relationship, user_id, context_id) revision_payload += [data] with benchmark("Snapshot._create.write revisions to database"): self._execute(models.Revision.__table__.insert(), revision_payload) return OperationResponse("create", True, for_create, response_data)