def handle_acl_creation(session): """Create relations for mapped objects.""" base_objects = defaultdict(set) for obj in session.new: if isinstance(obj, all_models.AccessControlList): acr_id = obj.ac_role.id if obj.ac_role else obj.ac_role_id acr_name = get_custom_roles_for(obj.object_type).get(acr_id) if acr_name in Assignable.ASSIGNEE_TYPES: base_objects[Stub(obj.object_type, obj.object_id)].add(obj) if base_objects: related_objects = related(base_objects.keys(), RelationshipsCache()) snapshot_ids = collect_snapshot_ids(related_objects) if snapshot_ids: add_related_snapshots(snapshot_ids, related_objects) create_related_roles(base_objects, related_objects)
def after_flush(self, session, _): """Handle legacy audit captain -> program editor role propagation""" self.caches = { "relationship_cache": RelationshipsCache(), "snapshots_cache": {}, "access_control_list_cache": AccessControlListCache() } handlers = { all_models.AccessControlList: self.handle_access_control_list, all_models.Snapshot: self.handle_snapshot, all_models.Relationship: self.handle_relationship, } for obj in session.new: handler = handlers.get(type(obj)) if callable(handler): handler(obj)
def after_flush(self, session, _): """Handle program related acl""" self.access_control_list_manager = ACLManager() self.relationship_cache = RelationshipsCache() self.program_roles = _get_program_roles() handlers = { all_models.AccessControlList: self.handle_access_control_list, all_models.Snapshot: self.handle_snapshot, all_models.Relationship: self.handle_relationship_creation, all_models.Audit: self.handle_audit_creation } for obj in session.new: handler = handlers.get(type(obj)) if callable(handler): if obj not in self.done: handler(obj) self.done.add(obj)
def __init__(self): self.processed = set() self.queue = set() self.auto_mappings = set() self.related_cache = RelationshipsCache()
class AutomapperGenerator(object): """Generator for automappings. Consumes automapping rules and newly created Relationships, creates autogenerated Relationships registering them in Automappings table. Note: we can rely on the order of src/dst pairs of queued and inserted mappings since we only queue ordered pairs (see `order`). """ COUNT_LIMIT = 10000 def __init__(self): self.processed = set() self.queue = set() self.auto_mappings = set() self.related_cache = RelationshipsCache() def related(self, obj): if obj in self.related_cache.cache: return self.related_cache.cache[obj] # Pre-fetch neighborhood for enqueued objects since we're gonna need these # results in a few steps. This drastically reduces number of queries. stubs = {s for rel in self.queue for s in rel} stubs.add(obj) self.related_cache.populate_cache(stubs) return self.related_cache.cache[obj] @staticmethod def order(src, dst): return (src, dst) if src < dst else (dst, src) def generate_automappings(self, relationship): self.auto_mappings = set() with benchmark("Automapping generate_automappings"): # initial relationship is special since it is already created and # processing it would abort the loop so we manually enqueue the # neighborhood src = Stub.from_source(relationship) dst = Stub.from_destination(relationship) self._step(src, dst) self._step(dst, src) while self.queue: if len(self.auto_mappings) > self.COUNT_LIMIT: break src, dst = entry = self.queue.pop() if {src.type, dst.type} != {"Audit", "Issue"}: # Auditor doesn't have edit (+map) permission on the Audit, # but the Auditor should be allowed to Raise an Issue. # Since Issue-Assessment-Audit is the only rule that # triggers Issue to Audit mapping, we should skip the # permission check for it if not (self._can_map_to(src, relationship) and self._can_map_to(dst, relationship)): continue created = self._ensure_relationship(src, dst) self.processed.add(entry) if not created: # If the edge already exists it means that auto mappings for it have # already been processed and it is safe to cut here. continue self._step(src, dst) self._step(dst, src) if len(self.auto_mappings) <= self.COUNT_LIMIT: self._flush(relationship) else: relationship._json_extras = { 'automapping_limit_exceeded': True } @staticmethod def _can_map_to(obj, parent_relationship): """True if the current user can edit obj in parent_relationship.context.""" context_id = None if parent_relationship.context: context_id = parent_relationship.context.id elif parent_relationship.context_id: logger.warning( "context is unset but context_id is set on a " "relationship %r: context=%r, context_id=%r", parent_relationship, parent_relationship.context, parent_relationship.context_id) context_id = parent_relationship.context_id return is_allowed_update(obj.type, obj.id, context_id) def _flush(self, parent_relationship): """Manually INSERT generated automappings.""" if not self.auto_mappings: return with benchmark("Automapping flush"): current_user_id = login.get_current_user_id() automapping_result = db.session.execute( Automapping.__table__.insert().values( relationship_id=parent_relationship.id, source_id=parent_relationship.source_id, source_type=parent_relationship.source_type, destination_id=parent_relationship.destination_id, destination_type=parent_relationship.destination_type, )) automapping_id = automapping_result.inserted_primary_key now = datetime.now() # We are doing an INSERT IGNORE INTO here to mitigate a race condition # that happens when multiple simultaneous requests create the same # automapping. If a relationship object fails our unique constraint # it means that the mapping was already created by another request # and we can safely ignore it. inserter = Relationship.__table__.insert().prefix_with("IGNORE") original = self.order(Stub.from_source(parent_relationship), Stub.from_destination(parent_relationship)) db.session.execute( inserter.values([{ "id": None, "modified_by_id": current_user_id, "created_at": now, "updated_at": now, "source_id": src.id, "source_type": src.type, "destination_id": dst.id, "destination_type": dst.type, "context_id": None, "status": None, "parent_id": parent_relationship.id, "automapping_id": automapping_id } for src, dst in self.auto_mappings if (src, dst) != original ])) # (src, dst) is sorted self._set_audit_id_for_issues(automapping_id) cache = Cache.get_cache(create=True) if cache: # Add inserted relationships into new objects collection of the cache, # so that they will be logged within event and appropriate revisions # will be created. cache.new.update( (relationship, relationship.log_json()) for relationship in Relationship.query.filter_by( automapping_id=automapping_id, )) @staticmethod def _set_audit_id_for_issues(automapping_id): """Set audit_id and context_id in automapped Issues.""" iss, rel, aud = Issue.__table__, Relationship.__table__, Audit.__table__ db.session.execute( sa.sql.expression.update(iss).values({ iss.c.audit_id: aud.c.id, iss.c.context_id: aud.c.context_id, }).where( sa.and_( rel.c.automapping_id == automapping_id, rel.c.source_type == Audit.__name__, rel.c.source_id == aud.c.id, rel.c.destination_type == Issue.__name__, rel.c.destination_id == iss.c.id, ), )) def _step(self, src, dst): mappings = rules.rules[src.type, dst.type] if mappings: dst_related = (o for o in self.related(dst) if o.type in mappings and o != src) for r in dst_related: entry = self.order(r, src) if entry not in self.processed: self.queue.add(entry) def _ensure_relationship(self, src, dst): """Create the relationship if not exists already. Returns: True if a relationship was created; False otherwise. """ # even though self.cache is defaultdict, self.cache[key] adds key # to self.cache if it is not present, and we should avoid it if dst in self.related(src): return False if src in self.related(dst): return False self._check_single_audit_restriction(src, dst) self.auto_mappings.add((src, dst)) if src in self.related_cache.cache: self.related_cache.cache[src].add(dst) if dst in self.related_cache.cache: self.related_cache.cache[dst].add(src) return True def _check_single_audit_restriction(self, src, dst): """Fail if dst (Issue) is already mapped to an Audit.""" # src, dst are ordered since they come from self.queue if (src.type, dst.type) == ("Audit", "Issue"): if "Audit" in (r.type for r in self.related(dst)): raise exceptions.ValidationError( "This request will result in automapping that will map " "Issue#{issue.id} to multiple Audits.".format(issue=dst))
def __init__(self): self.processed = set() self.queue = set() self.auto_mappings = set() self.automapping_ids = set() self.related_cache = RelationshipsCache()
class AutomapperGenerator(object): """Generator for automappings. Consumes automapping rules and newly created Relationships, creates autogenerated Relationships registering them in Automappings table. Note: we can rely on the order of src/dst pairs of queued and inserted mappings since we only queue ordered pairs (see `order`). """ COUNT_LIMIT = 10000 def __init__(self): self.processed = set() self.queue = set() self.auto_mappings = set() self.automapping_ids = set() self.related_cache = RelationshipsCache() def related(self, obj): """Return obj's relationship stubs""" if obj in self.related_cache.cache: return self.related_cache.cache[obj] # Pre-fetch neighborhood for enqueued objects since we're gonna need these # results in a few steps. This drastically reduces number of queries. stubs = {s for rel in self.queue for s in rel} stubs.add(obj) self.related_cache.populate_cache(stubs) return self.related_cache.cache[obj] @staticmethod def order(src, dst): return (src, dst) if src < dst else (dst, src) def generate_automappings(self, relationship): """Generate Automappings for a given relationship""" # pylint: disable=protected-access self.auto_mappings = set() with benchmark("Automapping generate_automappings"): # initial relationship is special since it is already created and # processing it would abort the loop so we manually enqueue the # neighborhood src = Stub.from_source(relationship) dst = Stub.from_destination(relationship) self._step(src, dst) self._step(dst, src) while self.queue: if len(self.auto_mappings) > self.COUNT_LIMIT: break src, dst = entry = self.queue.pop() if {src.type, dst.type} != {"Audit", "Issue"}: # Auditor doesn't have edit (+map) permission on the Audit, # but the Auditor should be allowed to Raise an Issue. # Since Issue-Assessment-Audit is the only rule that # triggers Issue to Audit mapping, we should skip the # permission check for it if not (permissions.is_allowed_update(src.type, src.id, None) and permissions.is_allowed_update(dst.type, dst.id, None)): continue created = self._ensure_relationship(src, dst) self.processed.add(entry) if not created: # If the edge already exists it means that auto mappings for it have # already been processed and it is safe to cut here. continue self._step(src, dst) self._step(dst, src) if len(self.auto_mappings) <= self.COUNT_LIMIT: self._flush(relationship) else: logger.error("Automapping limit exceeded: limit=%s, count=%s", self.COUNT_LIMIT, len(self.auto_mappings)) def _flush(self, parent_relationship): """Manually INSERT generated automappings.""" if not self.auto_mappings: return with benchmark("Automapping flush"): current_user_id = login.get_current_user_id() automapping_result = db.session.execute( Automapping.__table__.insert().values( relationship_id=parent_relationship.id, source_id=parent_relationship.source_id, source_type=parent_relationship.source_type, destination_id=parent_relationship.destination_id, destination_type=parent_relationship.destination_type, ) ) automapping_id = automapping_result.inserted_primary_key[0] self.automapping_ids.add(automapping_id) now = datetime.utcnow() # We are doing an INSERT IGNORE INTO here to mitigate a race condition # that happens when multiple simultaneous requests create the same # automapping. If a relationship object fails our unique constraint # it means that the mapping was already created by another request # and we can safely ignore it. inserter = Relationship.__table__.insert().prefix_with("IGNORE") original = self.order(Stub.from_source(parent_relationship), Stub.from_destination(parent_relationship)) db.session.execute(inserter.values([{ "id": None, "modified_by_id": current_user_id, "created_at": now, "updated_at": now, "source_id": src.id, "source_type": src.type, "destination_id": dst.id, "destination_type": dst.type, "context_id": None, "status": None, "parent_id": parent_relationship.id, "automapping_id": automapping_id, "is_external": False} for src, dst in self.auto_mappings if (src, dst) != original])) # (src, dst) is sorted self._set_audit_id_for_issues(automapping_id) cache = Cache.get_cache(create=True) if cache: # Add inserted relationships into new objects collection of the cache, # so that they will be logged within event and appropriate revisions # will be created. cache.new.update( (relationship, relationship.log_json()) for relationship in Relationship.query.filter_by( automapping_id=automapping_id, ) ) def propagate_acl(self): """Propagate acl records for newly created automappings""" if not self.automapping_ids: return from ggrc.models.hooks import acl ids_query = db.session.query( Relationship.id ).filter( Relationship.automapping_id.in_(self.automapping_ids) ) relationship_ids = {rel.id for rel in ids_query} acl.add_relationships(relationship_ids) @staticmethod def _set_audit_id_for_issues(automapping_id): """Set audit_id and context_id in automapped Issues.""" iss, rel, aud = Issue.__table__, Relationship.__table__, Audit.__table__ db.session.execute( sa.sql.expression.update(iss) .values({ iss.c.audit_id: aud.c.id, iss.c.context_id: aud.c.context_id, }) .where( sa.and_( rel.c.automapping_id == automapping_id, rel.c.source_type == Audit.__name__, rel.c.source_id == aud.c.id, rel.c.destination_type == Issue.__name__, rel.c.destination_id == iss.c.id, ), ) ) def _step(self, src, dst): """Step through the automapping rules tree.""" mappings = rules.rules[src.type, dst.type] if mappings: dst_related = (o for o in self.related(dst) if o.type in mappings and o != src) for related in dst_related: entry = self.order(related, src) if entry not in self.processed: self.queue.add(entry) def _ensure_relationship(self, src, dst): """Create the relationship if not exists already. Returns: True if a relationship was created; False otherwise. """ # even though self.cache is defaultdict, self.cache[key] adds key # to self.cache if it is not present, and we should avoid it if dst in self.related(src): return False if src in self.related(dst): return False self._check_single_audit_restriction(src, dst) self.auto_mappings.add((src, dst)) if src in self.related_cache.cache: self.related_cache.cache[src].add(dst) if dst in self.related_cache.cache: self.related_cache.cache[dst].add(src) return True def _check_single_audit_restriction(self, src, dst): """Fail if dst (Issue) is already mapped to an Audit.""" # src, dst are ordered since they come from self.queue if (src.type, dst.type) == ("Audit", "Issue"): if "Audit" in (related.type for related in self.related(dst)): raise exceptions.ValidationError( "This request will result in automapping that will map " "Issue#{issue.id} to multiple Audits." .format(issue=dst) )
class AutomapperGenerator(object): """Generator for automappings. Consumes automapping rules and newly created Relationships, creates autogenerated Relationships registering them in Automappings table. Note: we can rely on the order of src/dst pairs of queued and inserted mappings since we only queue ordered pairs (see `order`). """ COUNT_LIMIT = 50000 _AUTOMAP_WITHOUT_PERMISSION = [ {"Audit", "Issue"}, {"Snapshot", "Issue"}, ] def __init__(self): self.processed = set() self.queue = set() self.auto_mappings = set() self.automapping_ids = set() self.related_cache = RelationshipsCache() def related(self, obj, of_types=None): # type: (models.relationship.Stub, Optional[List[str]]) -> set """Return obj's relationship stubs""" if obj in self.related_cache.cache: return self.related_cache.cache[obj] # Pre-fetch neighborhood for enqueued objects since we're gonna need these # results in a few steps. This drastically reduces number of queries. stubs = {s for rel in self.queue for s in rel} stubs.add(obj) self.related_cache.populate_cache(stubs, of_types=of_types) return self.related_cache.cache[obj] @staticmethod def order(src, dst): return (src, dst) if src < dst else (dst, src) def generate_automappings(self, relationship): """Generate Automappings for a given relationship""" self.auto_mappings = set() # initial relationship is special since it is already created and # processing it would abort the loop so we manually enqueue the # neighborhood src = Stub.from_source(relationship) dst = Stub.from_destination(relationship) self._step(src, dst) self._step(dst, src) while self.queue: if len(self.auto_mappings) > self.COUNT_LIMIT: break src, dst = entry = self.queue.pop() if {src.type, dst.type} not in self._AUTOMAP_WITHOUT_PERMISSION: # Mapping between some objects should be created even if there is no # permission to edit (+map) this objects. Thus permissions check for # them should be skipped. if not (permissions.is_allowed_update(src.type, src.id, None) and permissions.is_allowed_update(dst.type, dst.id, None)): continue created = self._ensure_relationship(src, dst) self.processed.add(entry) if not created: # If the edge already exists it means that auto mappings for it have # already been processed and it is safe to cut here. continue self._step(src, dst) self._step(dst, src) if len(self.auto_mappings) <= self.COUNT_LIMIT: if self.auto_mappings: logger.info("Automapping count: count=%s", len(self.auto_mappings)) self._flush(relationship) else: logger.error("Automapping limit exceeded: limit=%s, count=%s", self.COUNT_LIMIT, len(self.auto_mappings)) def _flush(self, parent_relationship): """Manually INSERT generated automappings.""" if not self.auto_mappings: return with benchmark("Automapping flush"): current_user_id = login.get_current_user_id() automapping_result = db.session.execute( Automapping.__table__.insert().values( relationship_id=parent_relationship.id, source_id=parent_relationship.source_id, source_type=parent_relationship.source_type, destination_id=parent_relationship.destination_id, destination_type=parent_relationship.destination_type, modified_by_id=current_user_id, ) ) automapping_id = automapping_result.inserted_primary_key[0] self.automapping_ids.add(automapping_id) now = datetime.utcnow() # We are doing an INSERT IGNORE INTO here to mitigate a race condition # that happens when multiple simultaneous requests create the same # automapping. If a relationship object fails our unique constraint # it means that the mapping was already created by another request # and we can safely ignore it. inserter = Relationship.__table__.insert().prefix_with("IGNORE") original = self.order(Stub.from_source(parent_relationship), Stub.from_destination(parent_relationship)) db.session.execute(inserter.values([{ "id": None, "modified_by_id": current_user_id, "created_at": now, "updated_at": now, "source_id": src.id, "source_type": src.type, "destination_id": dst.id, "destination_type": dst.type, "context_id": None, "status": None, "parent_id": parent_relationship.id, "automapping_id": automapping_id, "is_external": False} for src, dst in self.auto_mappings if (src, dst) != original])) # (src, dst) is sorted self._set_audit_id_for_issues(automapping_id) cache = Cache.get_cache(create=True) if cache: # Add inserted relationships into new objects collection of the cache, # so that they will be logged within event and appropriate revisions # will be created. cache.new.update( (relationship, relationship.log_json()) for relationship in Relationship.query.filter_by( automapping_id=automapping_id, ) ) def propagate_acl(self): """Propagate acl records for newly created automappings""" if not self.automapping_ids: return from ggrc.models.hooks import acl ids_query = db.session.query( Relationship.id ).filter( Relationship.automapping_id.in_(self.automapping_ids) ) relationship_ids = {rel.id for rel in ids_query} acl.add_relationships(relationship_ids) @staticmethod def _set_audit_id_for_issues(automapping_id): """Set audit_id and context_id in automapped Issues.""" iss, rel, aud = Issue.__table__, Relationship.__table__, Audit.__table__ db.session.execute( sa.sql.expression.update(iss) .values({ iss.c.audit_id: aud.c.id, iss.c.context_id: aud.c.context_id, }) .where( sa.and_( rel.c.automapping_id == automapping_id, rel.c.source_type == Audit.__name__, rel.c.source_id == aud.c.id, rel.c.destination_type == Issue.__name__, rel.c.destination_id == iss.c.id, ), ) ) def _step(self, src, dst): """Step through the automapping rules tree.""" mappings = rules.rules[src.type, dst.type] if mappings: dst_related = (o for o in self.related(dst, of_types=mappings) if o.type in mappings and o != src) dst_model = models.get_model(dst.type) for related in dst_related: if (issubclass(dst_model, mega.Mega) and dst_model.skip_automapping(src, dst, related)): # Mega objects mapping are directed and objects from child # object should be mapped to parent objects, but not vice versa continue entry = self.order(related, src) if entry not in self.processed: self.queue.add(entry) def _ensure_relationship(self, src, dst): """Create the relationship if not exists already. Returns: True if a relationship was created; False otherwise. """ # even though self.cache is defaultdict, self.cache[key] adds key # to self.cache if it is not present, and we should avoid it if dst in self.related(src): return False if src in self.related(dst): return False self._check_single_audit_restriction(src, dst) self.auto_mappings.add((src, dst)) if src in self.related_cache.cache: self.related_cache.cache[src].add(dst) if dst in self.related_cache.cache: self.related_cache.cache[dst].add(src) return True def _check_single_audit_restriction(self, src, dst): """Fail if dst (Issue) is already mapped to an Audit.""" # src, dst are ordered since they come from self.queue if (src.type, dst.type) == ("Audit", "Issue"): if "Audit" in (related.type for related in self.related(dst)): raise exceptions.ValidationError( "This request will result in automapping that will map " "Issue#{issue.id} to multiple Audits." .format(issue=dst) )