Exemplo n.º 1
0
def upsert_all(sender, obj=None, src=None, service=None, event=None):  # noqa  # pylint: disable=unused-argument
  """Update snapshots globally"""
  snapshot_settings = src.get("snapshots")
  if snapshot_settings:
    if snapshot_settings["operation"] == "upsert":
      revisions = {
          (Stub.from_dict(revision["parent"]),
           Stub.from_dict(revision["child"])): revision["revision_id"]
          for revision in snapshot_settings.get("revisions", {})}
      upsert_snapshots(obj, event, revisions=revisions)
Exemplo n.º 2
0
def upsert_all(sender, obj=None, src=None, service=None, event=None):  # noqa  # pylint: disable=unused-argument
    """Update snapshots globally"""
    snapshot_settings = src.get("snapshots")
    if snapshot_settings:
        if snapshot_settings["operation"] == "upsert":
            revisions = {
                (Stub.from_dict(revision["parent"]),
                 Stub.from_dict(revision["child"])): revision["revision_id"]
                for revision in snapshot_settings.get("revisions", {})
            }
            upsert_snapshots(obj, event, revisions=revisions)
Exemplo n.º 3
0
def upsert_all(
    sender, obj=None, src=None, service=None,
    event=None, initial_state=None):  # noqa
  """Updates snapshots globally."""
  del sender, service, initial_state  # Unused
  snapshot_settings = src.get("snapshots")
  if snapshot_settings:
    if snapshot_settings["operation"] == "upsert":
      revisions = {
          (Stub.from_dict(revision["parent"]),
           Stub.from_dict(revision["child"])): revision["revision_id"]
          for revision in snapshot_settings.get("revisions", {})}
      upsert_snapshots(obj, event, revisions=revisions)
Exemplo n.º 4
0
def upsert_all(
    sender, obj=None, src=None, service=None,
    event=None, initial_state=None):  # noqa
  """Updates snapshots globally."""
  del sender, service, initial_state  # Unused
  snapshot_settings = src.get("snapshots")
  if snapshot_settings:
    if snapshot_settings["operation"] == "upsert":
      revisions = {
          (Stub.from_dict(revision["parent"]),
           Stub.from_dict(revision["child"])): revision["revision_id"]
          for revision in snapshot_settings.get("revisions", {})}
      upsert_snapshots(obj, event, revisions=revisions)
Exemplo n.º 5
0
def clone_scope(base_parent, new_parent, event):
    """Create exact copy of parent object scope.

  Args:
    base_parent: Old parent object
    new_parent: New parent object
    event: Event that triggered scope cloning
  """

    with benchmark("clone_scope.clone audit scope"):
        source_snapshots = db.session.query(
            models.Snapshot.child_type, models.Snapshot.child_id,
            models.Snapshot.revision_id).filter(
                models.Snapshot.parent_type == base_parent.type,
                models.Snapshot.parent_id == base_parent.id)

        snapshot_revisions = {
            Pair.from_4tuple((new_parent.type, new_parent.id, ctype, cid)):
            revid
            for ctype, cid, revid in source_snapshots
        }

        parent = Stub(new_parent.type, new_parent.id)
        children = {pair.child for pair in snapshot_revisions}
        generator = SnapshotGenerator(dry_run=False)
        generator.add_family(parent, children)
        generator.create(event, snapshot_revisions)
Exemplo n.º 6
0
  def _get_snapshottable_objects(self, obj):
    """Get snapshottable objects from parent object's neighborhood."""
    with benchmark("Snapshot._get_snapshotable_objects"):
      related_mappings = set()
      object_rules = self.rules.rules[obj.type]

      with benchmark("Snapshot._get_snapshotable_objects.related_mappings"):
        relatable_rules = {
            rule for rule in object_rules["fst"]
            if isinstance(rule, basestring)
        }

        if relatable_rules:
          related_mappings = obj.related_objects({
              rule for rule in object_rules["fst"]
              if isinstance(rule, basestring)})

      with benchmark("Snapshot._get_snapshotable_objects.direct mappings"):
        direct_mappings = {getattr(obj, rule.name)
                           for rule in object_rules["fst"]
                           if isinstance(rule, Attr)}

      related_objects = {Stub.from_object(obj)
                         for obj in related_mappings | direct_mappings}

      with benchmark("Snapshot._get_snapshotable_objects.fetch neighborhood"):
        return self._fetch_neighborhood(obj, related_objects)
Exemplo n.º 7
0
  def _get_snapshottable_objects(self, obj):
    """Get snapshottable objects from parent object's neighborhood."""
    with benchmark("Snapshot._get_snapshotable_objects"):
      related_mappings = set()
      object_rules = self.rules.rules[obj.type]

      with benchmark("Snapshot._get_snapshotable_objects.related_mappings"):
        relatable_rules = {
            rule for rule in object_rules["fst"]
            if isinstance(rule, basestring)
        }

        if relatable_rules:
          related_mappings = obj.related_objects({
              rule for rule in object_rules["fst"]
              if isinstance(rule, basestring)})

      with benchmark("Snapshot._get_snapshotable_objects.direct mappings"):
        direct_mappings = {getattr(obj, rule.name)
                           for rule in object_rules["fst"]
                           if isinstance(rule, Attr)}

      related_objects = {Stub.from_object(obj)
                         for obj in related_mappings | direct_mappings}

      with benchmark("Snapshot._get_snapshotable_objects.fetch neighborhood"):
        return self._fetch_neighborhood(obj, related_objects)
Exemplo n.º 8
0
def get_revisions(pairs, revisions, filters=None):
    """Retrieve revision ids for pairs

  If revisions dictionary is provided it will validate that the selected
  revision exists in the objects revision history.

  Args:
    pairs: set([(parent_1, child_1), (parent_2, child_2), ...])
    revisions: dict({(parent, child): revision_id, ...})
    filters: predicate
  """
    with benchmark("snapshotter.helpers.get_revisions"):
        revision_id_cache = dict()

        if pairs:
            with benchmark("get_revisions.create caches"):
                child_stubs = {pair.child for pair in pairs}

                with benchmark("get_revisions.create child -> parents cache"):
                    parents_cache = collections.defaultdict(set)
                    for parent, child in pairs:
                        parents_cache[child].add(parent)

            with benchmark("get_revisions.retrieve revisions"):
                query = db.session.query(
                    models.Revision.id, models.Revision.resource_type,
                    models.Revision.resource_id).filter(
                        tuple_(models.Revision.resource_type,
                               models.Revision.resource_id).in_(
                                   child_stubs)).order_by(
                                       models.Revision.id.desc())
                if filters:
                    for _filter in filters:
                        query = query.filter(_filter)

            with benchmark("get_revisions.create revision_id cache"):
                for revid, restype, resid in query:
                    child = Stub(restype, resid)
                    for parent in parents_cache[child]:
                        key = Pair(parent, child)
                        if key in revisions:
                            if revid == revisions[key]:
                                revision_id_cache[key] = revid
                            else:
                                logger.warning(
                                    "Specified revision for object %s but couldn't find the"
                                    "revision '%s' in object history", key,
                                    revisions[key])
                        else:
                            if key not in revision_id_cache:
                                revision_id_cache[key] = revid
        return revision_id_cache
Exemplo n.º 9
0
 def add_parent(self, obj):
     """Add parent object and automatically scan neighborhood for snapshottable
 objects."""
     with benchmark("Snapshot.add_parent_object"):
         key = Stub.from_object(obj)
         if key not in self.parents:
             with benchmark("Snapshot.add_parent_object.add object"):
                 objs = self._get_snapshottable_objects(obj)
                 self.parents.add(key)
                 self.context_cache[key] = obj.context_id
                 self.children = self.children | objs
                 self.snapshots[key] = objs
         return self.parents
Exemplo n.º 10
0
 def add_parent(self, obj):
   """Add parent object and automatically scan neighborhood for snapshottable
   objects."""
   with benchmark("Snapshot.add_parent_object"):
     key = Stub.from_object(obj)
     if key not in self.parents:
       with benchmark("Snapshot.add_parent_object.add object"):
         objs = self._get_snapshottable_objects(obj)
         self.parents.add(key)
         self.context_cache[key] = obj.context_id
         self.children = self.children | objs
         self.snapshots[key] = objs
     return self.parents
Exemplo n.º 11
0
    def _fetch_neighborhood(self, parent_object, objects):
        """Fetch relationships for objects and parent."""
        with benchmark("Snapshot._fetch_object_neighborhood"):
            query_pairs = set()

            for obj in objects:
                for snd_obj in self.rules.rules[parent_object.type]["snd"]:
                    query_pairs.add((obj.type, obj.id, snd_obj))

            columns = db.session.query(models.Relationship.source_type,
                                       models.Relationship.source_id,
                                       models.Relationship.destination_type,
                                       models.Relationship.destination_id)

            relationships = columns.filter(
                tuple_(
                    models.Relationship.destination_type,
                    models.Relationship.destination_id,
                    models.Relationship.source_type,
                ).in_(query_pairs)).union(
                    columns.filter(
                        tuple_(
                            models.Relationship.source_type,
                            models.Relationship.source_id,
                            models.Relationship.destination_type,
                        ).in_(query_pairs)))

            neighborhood = set()
            for (stype, sid, dtype, did) in relationships:
                source = Stub(stype, sid)
                destination = Stub(dtype, did)

                if source in objects:
                    neighborhood.add(destination)
                else:
                    neighborhood.add(source)
            return neighborhood
Exemplo n.º 12
0
def get_revisions(pairs, revisions, filters=None):
    """Retrieve revision ids for pairs

  Args:
    pairs: set([(parent_1, child_1), (parent_2, child_2), ...])
    revisions: dict({(parent, child): revision_id, ...})
    filters: predicate
  """
    with benchmark("snapshotter.helpers.get_revisions"):
        revision_id_cache = dict()

        if pairs:
            with benchmark("get_revisions.create caches"):
                child_stubs = {pair.child for pair in pairs}

                with benchmark("get_revisions.create child -> parents cache"):
                    parents_cache = collections.defaultdict(set)
                    for parent, child in pairs:
                        parents_cache[child].add(parent)

            with benchmark("get_revisions.retrieve revisions"):
                query = db.session.query(
                    models.Revision.id, models.Revision.resource_type,
                    models.Revision.resource_id).filter(
                        tuple_(models.Revision.resource_type,
                               models.Revision.resource_id).in_(
                                   child_stubs)).order_by(
                                       models.Revision.id.desc())
                if filters:
                    for _filter in filters:
                        query = query.filter(_filter)

            with benchmark("get_revisions.create revision_id cache"):
                for revid, restype, resid in query:
                    child = Stub(restype, resid)
                    for parent in parents_cache[child]:
                        key = Pair(parent, child)
                        if key in revisions:
                            if revid == revisions[key]:
                                revision_id_cache[key] = revid
                        else:
                            if key not in revision_id_cache:
                                revision_id_cache[key] = revid
        return revision_id_cache
Exemplo n.º 13
0
def get_revisions(pairs, revisions, filters=None):
    """Retrieve revision ids for pairs

  If revisions dictionary is provided it will validate that the selected
  revision exists in the objects revision history.

  Args:
    pairs: set([(parent_1, child_1), (parent_2, child_2), ...])
    revisions: dict({(parent, child): revision_id, ...})
    filters: predicate
  """
    with benchmark("snapshotter.helpers.get_revisions"):
        if not pairs:
            return {}

        with benchmark("get_revisions.create child -> parents cache"):
            parents_cache = collections.defaultdict(set)
            child_stubs = set()
            for parent, child in pairs:
                parents_cache[child].add(parent)
                child_stubs.add(child)

        with benchmark("get_revisions.retrieve revisions"):
            query = get_revisions_query(child_stubs, revisions, filters)

        revision_id_cache = {}
        with benchmark("get_revisions.create revision_id cache"):
            for revid, restype, resid in query:
                child = Stub(restype, resid)
                for parent in parents_cache[child]:
                    key = Pair(parent, child)
                    if key in revisions and revisions[key] != revid:
                        logger.warning(
                            "Specified revision for object %s but couldn't find the"
                            "revision '%s' in object history",
                            key,
                            revisions[key],
                        )
                    else:
                        revision_id_cache[key] = revid
        return revision_id_cache
Exemplo n.º 14
0
    def _create(self, for_create, event, revisions, _filter):
        """Create snapshots of parent objects neighhood and create revisions for
    snapshots.

    Args:
      event: A ggrc.models.Event instance
      revisions: A set of tuples of pairs with revisions to which it should
        either create or update a snapshot of that particular audit
      _filter: Callable that should return True if it should be updated
    Returns:
      OperationResponse
    """
        # pylint: disable=too-many-locals,too-many-statements
        with benchmark("Snapshot._create"):
            with benchmark("Snapshot._create init"):
                user_id = get_current_user_id()
                missed_keys = set()
                data_payload = list()
                revision_payload = list()
                relationship_payload = list()
                response_data = dict()

                if self.dry_run and event is None:
                    event_id = 0
                else:
                    event_id = event.id

            with benchmark("Snapshot._create.filter"):
                if _filter:
                    for_create = {elem for elem in for_create if _filter(elem)}

            with benchmark("Snapshot._create._get_revisions"):
                revision_id_cache = get_revisions(for_create, revisions)

            response_data["revisions"] = revision_id_cache

            with benchmark("Snapshot._create.create payload"):
                for pair in for_create:
                    if pair in revision_id_cache:
                        revision_id = revision_id_cache[pair]
                        context_id = self.context_cache[pair.parent]
                        data = create_snapshot_dict(pair, revision_id, user_id,
                                                    context_id)
                        data_payload += [data]
                    else:
                        missed_keys.add(pair)

            if missed_keys:
                logger.warning(
                    "Tried to create snapshots for the following objects but "
                    "found no revisions: %s", missed_keys)

            with benchmark("Snapshot._create.write to database"):
                self._execute(models.Snapshot.__table__.insert(), data_payload)

            with benchmark("Snapshot._create.retrieve inserted snapshots"):
                snapshots = get_snapshots(for_create)

            with benchmark("Snapshot._create.access control list"):
                acl_payload = get_acl_payload(snapshots)

            with benchmark("Snapshot._create.write acls to database"):
                self._execute(all_models.AccessControlList.__table__.insert(),
                              acl_payload)

            with benchmark(
                    "Snapshot._create.create parent object -> snapshot rels"):
                for snapshot in snapshots:
                    parent = Stub(snapshot.parent_type, snapshot.parent_id)
                    base = Stub(snapshot.child_type, snapshot.child_id)
                    relationship = create_relationship_dict(
                        parent, base, user_id, self.context_cache[parent])
                    relationship_payload += [relationship]

            with benchmark("Snapshot._create.write relationships to database"):
                self._execute(models.Relationship.__table__.insert(),
                              relationship_payload)

            with benchmark("Snapshot._create.get created relationships"):
                created_relationships = {
                    (rel["source_type"], rel["source_id"],
                     rel["destination_type"], rel["destination_id"])
                    for rel in relationship_payload
                }
                relationships = get_relationships(created_relationships)

            with benchmark("Snapshot._create.create revision payload"):
                with benchmark(
                        "Snapshot._create.create snapshots revision payload"):
                    for snapshot in snapshots:
                        parent = Stub(snapshot.parent_type, snapshot.parent_id)
                        context_id = self.context_cache[parent]
                        data = create_snapshot_revision_dict(
                            "created", event_id, snapshot, user_id, context_id)
                        revision_payload += [data]

                with benchmark("Snapshot._create.create rel revision payload"):
                    for relationship in relationships:
                        parent = Stub(relationship.source_type,
                                      relationship.source_id)
                        context_id = self.context_cache[parent]
                        data = create_relationship_revision_dict(
                            "created", event_id, relationship, user_id,
                            context_id)
                        revision_payload += [data]

            with benchmark("Snapshot._create.write revisions to database"):
                self._execute(models.Revision.__table__.insert(),
                              revision_payload)
            return OperationResponse("create", True, for_create, response_data)
Exemplo n.º 15
0
    def _update(self, for_update, event, revisions, _filter):
        """Update (or create) parent objects' snapshots and create revisions for
    them.

    Args:
      event: A ggrc.models.Event instance
      revisions: A set of tuples of pairs with revisions to which it should
        either create or update a snapshot of that particular audit
      _filter: Callable that should return True if it should be updated
    Returns:
      OperationResponse
    """
        # pylint: disable=too-many-locals
        with benchmark("Snapshot._update"):
            user_id = get_current_user_id()
            missed_keys = set()
            snapshot_cache = dict()
            modified_snapshot_keys = set()
            data_payload_update = list()
            revision_payload = list()
            response_data = dict()

            if self.dry_run and event is None:
                event_id = 0
            else:
                event_id = event.id

            with benchmark("Snapshot._update.filter"):
                if _filter:
                    for_update = {elem for elem in for_update if _filter(elem)}

            with benchmark("Snapshot._update.get existing snapshots"):
                existing_snapshots = db.session.query(
                    models.Snapshot.id,
                    models.Snapshot.revision_id,
                    models.Snapshot.parent_type,
                    models.Snapshot.parent_id,
                    models.Snapshot.child_type,
                    models.Snapshot.child_id,
                ).filter(
                    tuple_(models.Snapshot.parent_type,
                           models.Snapshot.parent_id,
                           models.Snapshot.child_type,
                           models.Snapshot.child_id).in_(
                               {pair.to_4tuple()
                                for pair in for_update}))

                for esnap in existing_snapshots:
                    sid, rev_id, pair_tuple = esnap[0], esnap[1], esnap[2:]
                    pair = Pair.from_4tuple(pair_tuple)
                    snapshot_cache[pair] = (sid, rev_id)

            with benchmark("Snapshot._update.retrieve latest revisions"):
                revision_id_cache = get_revisions(
                    for_update,
                    filters=[
                        models.Revision.action.in_(["created", "modified"])
                    ],
                    revisions=revisions)

            response_data["revisions"] = {
                "old":
                {pair: values[1]
                 for pair, values in snapshot_cache.items()},
                "new": revision_id_cache
            }

            with benchmark("Snapshot._update.build snapshot payload"):
                for key in for_update:
                    if key in revision_id_cache:
                        sid, rev_id = snapshot_cache[key]
                        latest_rev = revision_id_cache[key]
                        if rev_id != latest_rev:
                            modified_snapshot_keys.add(key)
                            data_payload_update += [{
                                "_id": sid,
                                "_revision_id": latest_rev,
                                "_modified_by_id": user_id
                            }]
                    else:
                        missed_keys.add(key)

            if missed_keys:
                logger.warning(
                    "Tried to update snapshots for the following objects but "
                    "found no revisions: %s", missed_keys)

            if not modified_snapshot_keys:
                return OperationResponse("update", True, set(), response_data)

            with benchmark("Snapshot._update.write snapshots to database"):
                update_sql = models.Snapshot.__table__.update().where(
                    models.Snapshot.id == bindparam("_id")).values(
                        revision_id=bindparam("_revision_id"),
                        modified_by_id=bindparam("_modified_by_id"))
                self._execute(update_sql, data_payload_update)

            with benchmark("Snapshot._update.retrieve inserted snapshots"):
                snapshots = get_snapshots(modified_snapshot_keys)

            with benchmark(
                    "Snapshot._update.create snapshots revision payload"):
                for snapshot in snapshots:
                    parent = Stub(snapshot.parent_type, snapshot.parent_id)
                    context_id = self.context_cache[parent]
                    data = create_snapshot_revision_dict(
                        "modified", event_id, snapshot, user_id, context_id)
                    revision_payload += [data]

            with benchmark("Insert Snapshot entries into Revision"):
                self._execute(models.Revision.__table__.insert(),
                              revision_payload)
            return OperationResponse("update", True, for_update, response_data)
Exemplo n.º 16
0
  def _create(self, for_create, event, revisions, _filter):
    """Create snapshots of parent objects neighhood and create revisions for
    snapshots.

    Args:
      event: A ggrc.models.Event instance
      revisions: A set of tuples of pairs with revisions to which it should
        either create or update a snapshot of that particular audit
      _filter: Callable that should return True if it should be updated
    Returns:
      OperationResponse
    """
    # pylint: disable=too-many-locals,too-many-statements
    with benchmark("Snapshot._create"):
      with benchmark("Snapshot._create init"):
        user_id = get_current_user_id()
        missed_keys = set()
        data_payload = list()
        revision_payload = list()
        relationship_payload = list()
        response_data = dict()

        if self.dry_run and event is None:
          event_id = 0
        else:
          event_id = event.id

      with benchmark("Snapshot._create.filter"):
        if _filter:
          for_create = {elem for elem in for_create if _filter(elem)}

      with benchmark("Snapshot._create._get_revisions"):
        revision_id_cache = get_revisions(for_create, revisions)

      response_data["revisions"] = revision_id_cache

      with benchmark("Snapshot._create.create payload"):
        for pair in for_create:
          if pair in revision_id_cache:
            revision_id = revision_id_cache[pair]
            context_id = self.context_cache[pair.parent]
            data = create_snapshot_dict(pair, revision_id, user_id, context_id)
            data_payload += [data]
          else:
            missed_keys.add(pair)

      with benchmark("Snapshot._create.write to database"):
        self._execute(
            models.Snapshot.__table__.insert(),
            data_payload)

      with benchmark("Snapshot._create.retrieve inserted snapshots"):
        snapshots = get_snapshots(for_create)

      with benchmark("Snapshot._create.create base object -> snapshot rels"):
        for snapshot in snapshots:
          base_object = Stub.from_tuple(snapshot, 6, 7)
          snapshot_object = Stub("Snapshot", snapshot[0])
          relationship = create_relationship_dict(base_object, snapshot_object,
                                                  user_id, snapshot[1])
          relationship_payload += [relationship]

      with benchmark("Snapshot._create.write relationships to database"):
        self._execute(models.Relationship.__table__.insert(),
                      relationship_payload)

      with benchmark("Snapshot._create.get created relationships"):
        created_relationships = {
            (rel["source_type"], rel["source_id"],
             rel["destination_type"], rel["destination_id"])
            for rel in relationship_payload}
        relationships = get_relationships(created_relationships)

      with benchmark("Snapshot._create.create revision payload"):
        with benchmark("Snapshot._create.create snapshots revision payload"):
          for snapshot in snapshots:
            parent = Stub.from_tuple(snapshot, 4, 5)
            context_id = self.context_cache[parent]
            data = create_snapshot_revision_dict("created", event_id, snapshot,
                                                 user_id, context_id)
            revision_payload += [data]

        with benchmark("Snapshot._create.create rel revision payload"):
          snapshot_parents = {pair.child: pair.parent for pair in for_create}
          for relationship in relationships:
            obj = Stub.from_tuple(relationship, 4, 5)
            parent = snapshot_parents[obj]
            context_id = self.context_cache[parent]
            data = create_relationship_revision_dict(
                "created", event_id, relationship, user_id, context_id)
            revision_payload += [data]

      with benchmark("Snapshot._create.write revisions to database"):
        self._execute(models.Revision.__table__.insert(), revision_payload)
      return OperationResponse("create", True, for_create, response_data)