Example #1
0
    def test_stack_can_produce_snapshot_of_future_revision_of_update_type(self):
        """Test that the stack can create a future state of a document"""
        master_id = yield self.collection.insert(self.test_fixture)

        stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings, master_id=master_id)

        update = {
            test_attr: test_val
        }

        yield stack.push(update, self.three_min_past_now)

        update["new_persistent"] = True
        yield stack.push(update, self.two_min_past_now)

        update["foo"] = "baz"
        id = yield stack.push(update, self.one_min_from_now)

        response = yield stack.preview(id)
        snapshot = response.get("snapshot")
        self.assertIsInstance(snapshot, dict)

        self.assertEqual(snapshot.get("bool_val"), True)
        self.assertEqual(snapshot.get("new_persistent"), True)
        self.assertEqual(snapshot.get("foo"), "baz")
Example #2
0
    def __lazy_migration(self, master_id):
        """
        Creates a revision for a master id that didn't previously have a
        revision, this allows you to easily turn on revisioning for a
        collection that didn't previously allow for it.

        :param master_id:
        :returns: list of objects
        """
        collection_name = self.request.headers.get("collection")

        if collection_name:
            stack = AsyncSchedulableDocumentRevisionStack(
                collection_name,
                self.settings,
                master_id=master_id,
            )
            objects = yield stack._lazy_migration(meta=self._get_meta_data())
            raise Return(objects)

        self.raise_error(
            500, "This object %s/%s didn't exist as a revision, "
            "we tried to create it but we failed... Sorry. "
            "Please check this object" % (collection_name, master_id))
        raise Return(None)
Example #3
0
    def test_stack_can_produce_snapshot_of_future_revision_of_insert_type(self):
        """Test that the stack can create a future state of a new yet to be created document"""

        stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings)

        fixture = self.test_fixture

        yield stack.push(self.test_fixture, self.three_min_past_now)

        fixture["baz"] = "bop"
        fixture["new_persistent"] = True
        yield stack.push(fixture, self.two_min_past_now)

        del fixture["new_persistent"]
        fixture["baz"] = "bit"
        id = yield stack.push(fixture, self.one_min_past_now)


        response = yield stack.preview(id)
        snapshot = response.get("snapshot")

        self.assertIsInstance(snapshot, dict)
        self.assertEqual(snapshot.get("bool_val"), True)
        self.assertEqual(snapshot.get("new_persistent"), True)
        self.assertEqual(snapshot.get("baz"), "bit")
Example #4
0
    def test_stack_can_migrate_a_legacy_object_automatically(self):
        """Test the stack can migrate a legacy object automatically for the user"""
        client = BaseAsyncMotorDocument("test_fixture", settings)
        revisions = BaseAsyncMotorDocument("test_fixture_revisions", settings)

        fixture = self.test_fixture

        master_id = yield client.insert(fixture)

        stack = AsyncSchedulableDocumentRevisionStack("test_fixture",
                                                      settings,
                                                      master_id=master_id)

        fixture["baz"] = "bop"
        yield stack.push(self.test_fixture,
                         self.three_min_past_now,
                         meta={
                             "author": "UnitTest",
                             "comment": "Just a test, BRO."
                         })

        fixture["new_persistent"] = True
        yield stack.push(fixture, self.two_min_past_now)

        del fixture["new_persistent"]
        fixture["baz"] = "bit"
        id = yield stack.push(fixture, self.two_min_past_now)

        #response = yield stack.preview(id)
        list = yield revisions.find({"master_id": master_id})
        self.assertEqual(len(list), 4)
Example #5
0
    def test_stack_push_can_be_invalid_based_on_object_type(self):
        """Test that if we push the wrong type into a patch attribute, we fail correctly"""
        master_id = yield self.collection.insert(self.mini_doc)
        manager = AsyncRevisionStackManager(settings)
        stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings, master_id=master_id)

        yield stack.push("foo", toa=self.three_min_past_now)
        yield stack.push(False, toa=self.three_min_past_now)
Example #6
0
    def put(self, id):
        """
        Update a resource by bson ObjectId

        :returns: json string representation
        :rtype: JSON
        """
        try:
            #Async update flow
            object_ = json_util.loads(self.request.body)

            toa = self.request.headers.get("Caesium-TOA", None)

            obj_check = yield self.client.find_one_by_id(id)
            if not obj_check:
                self.raise_error(404, "Resource not found: %s" % id)
                self.finish()
                return

            if toa:

                stack = AsyncSchedulableDocumentRevisionStack(self.client.collection_name, self.settings, master_id=id)
                revision_id = yield stack.push(object_, int(toa), meta=self._get_meta_data())

                if isinstance(revision_id, str):
                    self.set_header("Caesium-TOA", toa)

                    #We add the id of the original request, because we don't want to infer this
                    #On the client side, as the state of the client code could change easily
                    #We want this request to return with the originating ID as well.
                    object_["id"] = id
                    self.return_resource(object_)
                else:
                    self.raise_error(404, "Revision not scheduled for object: %s" % id)

            else:
                if object_.get("_id"):
                    del object_["_id"]

                response = yield self.client.update(id, object_)

                if response.get("updatedExisting"):
                    object_ = yield self.client.find_one_by_id(id)
                    self.return_resource(object_)
                else:
                    self.raise_error(404, "Resource not found: %s" % id)

        except ValidationError as vex:
            self.logger.error("%s validation error" % self.object_name, vex)
            self.raise_error(400, "Your %s cannot be updated because it is missing required fields, see docs" % self.object_name)
        except ValueError as ex:
            self.raise_error(400, "Invalid JSON Body, check formatting. %s" % ex[0])
        except InvalidId as ex:
            self.raise_error(message="Your ID is malformed: %s" % id)
        except Exception as ex:
            self.logger.error(ex)
            self.raise_error()
Example #7
0
    def post(self, id=None):
        """
        Create a new object resource

        :json: Object to create
        :returns: json string representation
        :rtype: JSON

        """
        try:

            try:
                base_object = json_util.loads(self.request.body)
            except TypeError:
                base_object = json_util.loads(self.request.body.decode())

            #assert not hasattr(base_object, "_id")

            toa = self.request.headers.get("Caesium-TOA", None)

            if toa:
                # Async create flow
                stack = AsyncSchedulableDocumentRevisionStack(
                    self.client.collection_name, self.settings)

                revision_id = yield stack.push(base_object,
                                               toa=int(toa),
                                               meta=self._get_meta_data())
                resource = yield stack.preview(revision_id)

                if isinstance(revision_id, str):
                    self.set_header("Caesium-TOA", toa)
                    self.return_resource(resource.get("snapshot"))
                else:
                    self.raise_error(
                        404, "Revision not scheduled for object: %s" % id)

            else:

                id = yield self.client.insert(base_object)
                base_object = yield self.client.find_one_by_id(id)

                self.return_resource(base_object)

        except ValidationError as vex:
            self.logger.error("%s validation error" % self.object_name, vex)
            self.raise_error(
                400,
                "Your %s cannot be created because it is missing required fields, see docs"
                % self.object_name)
        except ValueError as ex:
            self.raise_error(400,
                             "Invalid JSON Body, check formatting. %s" % ex[0])
        except Exception as ex:
            self.logger.error(ex)
            self.raise_error()
Example #8
0
    def test_list_of_revisions(self):
        """Test that we can create a list of revisions from a given document in a collection"""
        master_id = yield self.collection.insert(self.test_fixture)
        self.stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings, master_id=master_id)
        id1 = yield self.stack.push(self.test_fixture, self.three_min_past_now)
        id2 = yield self.stack.push(self.test_fixture, self.three_min_ahead_of_now)

        revisions = yield self.stack.list()
        self.assertEqual(len(revisions), 1,msg="Did not receive the correct number of revisions")
        self.assertEqual(revisions[0].get("id"), id1, msg="The order doesn't appear to be correct")
Example #9
0
 def test_publish_with_insert_action(self):
     """Test that we can schedule a new collection object"""
     stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings)
     meta = {"comment": "foo"}
     bson_id = yield stack.push(self.mini_doc,
                                toa=self.three_min_past_now,
                                meta=meta)
     revisions = yield stack.list()
     self.assertEqual(len(revisions), 1)
     revision = revisions[0]
     self.assertEqual(revision.get("action"), "insert")
Example #10
0
 def test_publish_with_insert_action(self):
     """Test that we can schedule a new collection object"""
     stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings)
     meta = {
         "comment" : "foo"
     }
     bson_id = yield stack.push(self.mini_doc, toa=self.three_min_past_now, meta=meta)
     revisions = yield stack.list()
     self.assertEqual(len(revisions), 1)
     revision = revisions[0]
     self.assertEqual(revision.get("action"), "insert")
Example #11
0
    def test_peek_on_stack(self):
        """Test that we get a single object off the top of the stack"""
        master_id = yield self.collection.insert(self.test_fixture)
        self.stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings, master_id=master_id)
        id1 = yield self.stack.push(self.test_fixture, time.mktime(datetime.datetime.strptime('24052010', "%d%m%Y").timetuple()))
        id2 = yield self.stack.push(self.test_fixture, time.mktime(datetime.datetime.now().timetuple()))
        peeked_obj = yield self.stack.peek()

        self.assertIsNotNone(peeked_obj)
        self.assertIsInstance(peeked_obj, dict)
        self.assertEqual(id1, peeked_obj.get("id"))
        self.assertEqual(peeked_obj.get("action"), "update")
Example #12
0
    def test_patch_is_converted_and_storeable(self):
        """Test that a patch can be set with dot namespace safely and applied asynchronously via pop"""
        master_id = yield self.collection.insert(self.test_fixture)
        patch = {"patch.baz": True}
        self.stack = AsyncSchedulableDocumentRevisionStack("test_fixture",
                                                           settings,
                                                           master_id=master_id)
        yield self.stack.push(patch, self.three_min_past_now)
        yield self.stack.pop()
        obj = yield self.collection.find_one_by_id(master_id)

        self.assertEqual(obj.get("patch").get("foo"), "bar")
        self.assertEqual(obj.get("patch").get("baz"), True)
Example #13
0
    def test_pop_off_stack(self):
        """Test that our stack.pop method updates all relevant data correctly"""
        master_id = yield self.collection.insert(self.mini_doc)
        self.stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings,master_id=master_id)
        self.mini_doc[test_attr] = test_val
        id1 = yield self.stack.push(self.mini_doc, self.three_min_past_now)
        id2 = yield self.stack.push(self.mini_doc, self.now)
        obj = yield self.stack.pop()
        self.assertIsInstance(obj, dict)
        self.assertEqual(obj["processed"], True)
        self.assertEqual(obj["snapshot"][test_attr], test_val)

        obj_check = yield self.collection.find_one_by_id(master_id)
        self.assertEqual(test_val, obj_check.get(test_attr))
Example #14
0
    def test_publish_scheduled_pop_off_stack_w_manager(self):
        """Test that we can schedule a revision for 3 min ago and that the revision is applied through the manager"""
        master_id = yield self.collection.insert(self.mini_doc)
        manager = AsyncRevisionStackManager(settings)
        stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings, master_id=master_id)

        self.mini_doc[test_attr] = test_val
        yield stack.push(self.mini_doc, self.three_min_past_now)

        # Run the publish method manually
        yield manager.publish()

        #Make sure the live document contains our test values
        obj_check = yield self.collection.find_one_by_id(master_id)
        self.assertEqual(test_val, obj_check.get(test_attr))
Example #15
0
    def post(self, id=None):
        """
        Create a new object resource

        :json: Object to create
        :returns: json string representation
        :rtype: JSON

        """
        try:

            try:
                base_object = json_util.loads(self.request.body)
            except TypeError:
                base_object = json_util.loads(self.request.body.decode())

            #assert not hasattr(base_object, "_id")

            toa = self.request.headers.get("Caesium-TOA", None)

            if toa:
                # Async create flow
                stack = AsyncSchedulableDocumentRevisionStack(self.client.collection_name, self.settings)

                revision_id = yield stack.push(base_object, toa=int(toa), meta=self._get_meta_data())
                resource = yield stack.preview(revision_id)

                if isinstance(revision_id, str):
                    self.set_header("Caesium-TOA", toa)
                    self.return_resource(resource.get("snapshot"))
                else:
                    self.raise_error(404, "Revision not scheduled for object: %s" % id)

            else:

                id = yield self.client.insert(base_object)
                base_object = yield self.client.find_one_by_id(id)

                self.return_resource(base_object)

        except ValidationError as vex:
            self.logger.error("%s validation error" % self.object_name, vex)
            self.raise_error(400, "Your %s cannot be created because it is missing required fields, see docs" % self.object_name)
        except ValueError as ex:
            self.raise_error(400, "Invalid JSON Body, check formatting. %s" % ex[0])
        except Exception as ex:
            self.logger.error(ex)
            self.raise_error()
Example #16
0
    def test_publish_with_delete_action(self):
        """Test that we can delete a document on a schedule"""
        master_id = yield self.collection.insert(self.mini_doc)
        manager = AsyncRevisionStackManager(settings)
        stack = AsyncSchedulableDocumentRevisionStack("test_fixture",
                                                      settings,
                                                      master_id=master_id)
        meta = {"comment": "foo"}
        yield stack.push(None, toa=self.three_min_past_now, meta=meta)

        # Run the publish method manually
        yield stack.pop()

        #Make sure the live document contains our test values
        obj_check = yield self.collection.find_one_by_id(master_id)
        self.assertIsNone(obj_check, "Object did not delete")
Example #17
0
    def test_publish_with_delete_action(self):
        """Test that we can delete a document on a schedule"""
        master_id = yield self.collection.insert(self.mini_doc)
        manager = AsyncRevisionStackManager(settings)
        stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings, master_id=master_id)
        meta = {
            "comment" : "foo"
        }
        yield stack.push(None, toa=self.three_min_past_now, meta=meta)

        # Run the publish method manually
        yield stack.pop()

        #Make sure the live document contains our test values
        obj_check = yield self.collection.find_one_by_id(master_id)
        self.assertIsNone(obj_check, "Object did not delete")
Example #18
0
    def get(self, id):
        """
        Get revision based on the stack preview algorithm

        :param id: BSON id
        :return: JSON
        """
        collection_name = self.request.headers.get("collection")

        if not collection_name:
            self.raise_error(400, "Missing a collection name for stack")

        self.stack = AsyncSchedulableDocumentRevisionStack(
            collection_name, self.settings)

        revision = yield self.stack.preview(id)
        self.write(revision)
Example #19
0
    def test_stack_can_produce_snapshot_of_future_revision_of_update_type(
            self):
        """Test that the stack can create a future state of a document"""
        master_id = yield self.collection.insert(self.test_fixture)

        stack = AsyncSchedulableDocumentRevisionStack("test_fixture",
                                                      settings,
                                                      master_id=master_id)

        update = {test_attr: test_val}

        yield stack.push(update, self.three_min_past_now)

        update["new_persistent"] = True
        yield stack.push(update, self.two_min_past_now)

        update["foo"] = "baz"
        id = yield stack.push(update, self.one_min_from_now)

        response = yield stack.preview(id)
        snapshot = response.get("snapshot")
        self.assertIsInstance(snapshot, dict)

        self.assertEqual(snapshot.get("bool_val"), True)
        self.assertEqual(snapshot.get("new_persistent"), True)
        self.assertEqual(snapshot.get("foo"), "baz")
Example #20
0
    def put(self, id=None):
        """Update many objects with a single PUT.

        Example Request::

            {
                "ids": ["52b0ede98ac752b358b1bd69", "52b0ede98ac752b358b1bd70"],
                "patch": {
                    "foo": "bar"
                }
            }

        """

        toa = self.request.headers.get("Caesium-TOA")

        if not toa:
            self.raise_error(400, "Caesium-TOA header is required, none found")
            self.finish(self.request.headers.get("Caesium-TOA"))

        meta = self._get_meta_data()
        meta["bulk_id"] = uuid.uuid4().get_hex()
        ids = self.get_json_argument("ids")
        patch = self.get_json_argument("patch")

        self.get_json_argument("ids", [])
        for id in ids:
            stack = AsyncSchedulableDocumentRevisionStack(
                self.client.collection_name, self.settings, master_id=id)
            stack.push(patch, toa=toa, meta=meta)

        self.write({
            "count": len(ids),
            "result": {
                "ids": ids,
                "toa": toa,
                "patch": patch,
            }
        })
        self.finish()
Example #21
0
    def test_stack_can_produce_snapshot_of_future_revision_of_insert_type(
            self):
        """Test that the stack can create a future state of a new yet to be created document"""

        stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings)

        fixture = self.test_fixture

        yield stack.push(self.test_fixture, self.three_min_past_now)

        fixture["baz"] = "bop"
        fixture["new_persistent"] = True
        yield stack.push(fixture, self.two_min_past_now)

        del fixture["new_persistent"]
        fixture["baz"] = "bit"
        id = yield stack.push(fixture, self.one_min_past_now)

        response = yield stack.preview(id)
        snapshot = response.get("snapshot")

        self.assertIsInstance(snapshot, dict)
        self.assertEqual(snapshot.get("bool_val"), True)
        self.assertEqual(snapshot.get("new_persistent"), True)
        self.assertEqual(snapshot.get("baz"), "bit")
Example #22
0
    def put(self, id=None):
        """Update many objects with a single PUT.

        Example Request::

            {
                "ids": ["52b0ede98ac752b358b1bd69", "52b0ede98ac752b358b1bd70"],
                "patch": {
                    "foo": "bar"
                }
            }

        """

        toa = self.request.headers.get("Caesium-TOA")

        if not toa:
            self.raise_error(400, "Caesium-TOA header is required, none found")
            self.finish(self.request.headers.get("Caesium-TOA"))

        meta = self._get_meta_data()
        meta["bulk_id"] = uuid.uuid4().get_hex()
        ids = self.get_json_argument("ids")
        patch = self.get_json_argument("patch")

        self.get_json_argument("ids", [])
        for id in ids:
            stack = AsyncSchedulableDocumentRevisionStack(self.client.collection_name, self.settings, master_id=id)
            stack.push(patch, toa=toa, meta=meta)

        self.write({
            "count": len(ids),
            "result": {
                "ids": ids,
                "toa": toa,
                "patch": patch,
            }
        })
        self.finish()
Example #23
0
    def test_patch_is_converted_and_storeable(self):
        """Test that a patch can be set with dot namespace safely and applied asynchronously via pop"""
        master_id = yield self.collection.insert(self.test_fixture)
        patch = {
            "patch.baz" : True
        }
        self.stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings, master_id=master_id)
        yield self.stack.push(patch, self.three_min_past_now)
        yield self.stack.pop()
        obj = yield self.collection.find_one_by_id(master_id)

        self.assertEqual(obj.get("patch").get("foo"), "bar")
        self.assertEqual(obj.get("patch").get("baz"), True)
Example #24
0
    def test_stack_can_migrate_a_legacy_object_automatically(self):
        """Test the stack can migrate a legacy object automatically for the user"""
        client = BaseAsyncMotorDocument("test_fixture", settings)
        revisions = BaseAsyncMotorDocument("test_fixture_revisions", settings)

        fixture = self.test_fixture

        master_id = yield client.insert(fixture)

        stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings, master_id=master_id)

        fixture["baz"] = "bop"
        yield stack.push(self.test_fixture, self.three_min_past_now, meta={"author": "UnitTest", "comment": "Just a test, BRO."})

        fixture["new_persistent"] = True
        yield stack.push(fixture, self.two_min_past_now)

        del fixture["new_persistent"]
        fixture["baz"] = "bit"
        id = yield stack.push(fixture, self.two_min_past_now)

        #response = yield stack.preview(id)
        list = yield revisions.find({"master_id": master_id})
        self.assertEqual(len(list), 4)
Example #25
0
    def __lazy_migration(self, master_id):
        """
        Creates a revision for a master id that didn't previously have a
        revision, this allows you to easily turn on revisioning for a
        collection that didn't previously allow for it.

        :param master_id:
        :returns: list of objects
        """
        collection_name = self.request.headers.get("collection")

        if collection_name:
            stack = AsyncSchedulableDocumentRevisionStack(collection_name,
                                                          self.settings,
                                                          master_id=master_id,
                                                          )
            objects = yield stack._lazy_migration(meta=self._get_meta_data())
            raise Return(objects)

        self.raise_error(500, "This object %s/%s didn't exist as a revision, "
                         "we tried to create it but we failed... Sorry. "
                         "Please check this object" % (collection_name,
                                                       master_id))
        raise Return(None)
Example #26
0
    def get(self, id):
        """
        Get revision based on the stack preview algorithm

        :param id: BSON id
        :return: JSON
        """
        collection_name = self.request.headers.get("collection")

        if not collection_name:
            self.raise_error(400, "Missing a collection name for stack")

        self.stack = AsyncSchedulableDocumentRevisionStack(collection_name, self.settings)

        revision = yield self.stack.preview(id)
        self.write(revision)
Example #27
0
 def setUp(self):
     super(self.__class__, self).setUp()
     self.collection = BaseAsyncMotorDocument("test_fixture", settings)
     self.stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings)
     self.setup_database()
Example #28
0
 def setUp(self):
     super(self.__class__, self).setUp()
     self.collection = BaseAsyncMotorDocument("test_fixture", settings)
     self.stack = AsyncSchedulableDocumentRevisionStack(
         "test_fixture", settings)
     self.setup_database()
Example #29
0
class TestAsyncRevisionStackAndManagerFunctions(BaseAsyncTest):
    """ Test the Mongo Client funcitons here"""
    mini_doc = {"my doc": "little bitty doc"}

    test_fixture = {
        "attr1": "attr1_val",
        "date1": time.mktime(datetime.datetime.now().timetuple()),
        "bool_val": True,
        "list_val": ["My Item", "Item 2", 1],
        "loc": [-75.22, 39.25],
        "sub_document": mini_doc,
        "patch": {
            "foo": "bar"
        }
    }

    three_min_past_now = time.mktime(
        (datetime.datetime.now() - datetime.timedelta(minutes=3)).timetuple())
    two_min_past_now = time.mktime(
        (datetime.datetime.now() - datetime.timedelta(minutes=2)).timetuple())
    one_min_past_now = time.mktime(
        (datetime.datetime.now() - datetime.timedelta(minutes=1)).timetuple())
    one_min_from_now = time.mktime(
        (datetime.datetime.now() + datetime.timedelta(minutes=1)).timetuple())

    three_min_ahead_of_now = time.mktime(
        (datetime.datetime.now() + datetime.timedelta(minutes=3)).timetuple())
    now = time.mktime(datetime.datetime.now().timetuple())

    def setUp(self):
        super(self.__class__, self).setUp()
        self.collection = BaseAsyncMotorDocument("test_fixture", settings)
        self.stack = AsyncSchedulableDocumentRevisionStack(
            "test_fixture", settings)
        self.setup_database()

    @tornado.gen.coroutine
    def setup_database(self):
        self.collection.collection.drop()
        self.stack.revisions.collection.drop()
        #self.stack.previews.collection.drop()

    @tornado.testing.gen_test
    def test_push_on_stack(self):
        """Test that a revision is pushed onto the stack and stored into mongo"""
        id = yield self.stack.push(self.test_fixture, self.three_min_past_now)
        self.assertIsInstance(id, str)
        obj_id = ObjectId(id)
        self.assertIsInstance(obj_id, ObjectId)

    @tornado.testing.gen_test
    def test_patch_is_converted_and_storeable(self):
        """Test that a patch can be set with dot namespace safely and applied asynchronously via pop"""
        master_id = yield self.collection.insert(self.test_fixture)
        patch = {"patch.baz": True}
        self.stack = AsyncSchedulableDocumentRevisionStack("test_fixture",
                                                           settings,
                                                           master_id=master_id)
        yield self.stack.push(patch, self.three_min_past_now)
        yield self.stack.pop()
        obj = yield self.collection.find_one_by_id(master_id)

        self.assertEqual(obj.get("patch").get("foo"), "bar")
        self.assertEqual(obj.get("patch").get("baz"), True)

    @tornado.testing.gen_test
    def test_list_of_revisions(self):
        """Test that we can create a list of revisions from a given document in a collection"""
        master_id = yield self.collection.insert(self.test_fixture)
        self.stack = AsyncSchedulableDocumentRevisionStack("test_fixture",
                                                           settings,
                                                           master_id=master_id)
        id1 = yield self.stack.push(self.test_fixture, self.three_min_past_now)
        id2 = yield self.stack.push(self.test_fixture,
                                    self.three_min_ahead_of_now)

        revisions = yield self.stack.list()
        self.assertEqual(len(revisions),
                         1,
                         msg="Did not receive the correct number of revisions")
        self.assertEqual(revisions[0].get("id"),
                         id1,
                         msg="The order doesn't appear to be correct")

    @tornado.testing.gen_test
    def test_peek_on_stack(self):
        """Test that we get a single object off the top of the stack"""
        master_id = yield self.collection.insert(self.test_fixture)
        self.stack = AsyncSchedulableDocumentRevisionStack("test_fixture",
                                                           settings,
                                                           master_id=master_id)
        id1 = yield self.stack.push(
            self.test_fixture,
            time.mktime(
                datetime.datetime.strptime('24052010', "%d%m%Y").timetuple()))
        id2 = yield self.stack.push(
            self.test_fixture,
            time.mktime(datetime.datetime.now().timetuple()))
        peeked_obj = yield self.stack.peek()

        self.assertIsNotNone(peeked_obj)
        self.assertIsInstance(peeked_obj, dict)
        self.assertEqual(id1, peeked_obj.get("id"))
        self.assertEqual(peeked_obj.get("action"), "update")

    @gen_test
    def test_pop_off_stack(self):
        """Test that our stack.pop method updates all relevant data correctly"""
        master_id = yield self.collection.insert(self.mini_doc)
        self.stack = AsyncSchedulableDocumentRevisionStack("test_fixture",
                                                           settings,
                                                           master_id=master_id)
        self.mini_doc[test_attr] = test_val
        id1 = yield self.stack.push(self.mini_doc, self.three_min_past_now)
        id2 = yield self.stack.push(self.mini_doc, self.now)
        obj = yield self.stack.pop()
        self.assertIsInstance(obj, dict)
        self.assertEqual(obj["processed"], True)
        self.assertEqual(obj["snapshot"][test_attr], test_val)

        obj_check = yield self.collection.find_one_by_id(master_id)
        self.assertEqual(test_val, obj_check.get(test_attr))

    @gen_test
    def test_publish_scheduled_pop_off_stack_w_manager(self):
        """Test that we can schedule a revision for 3 min ago and that the revision is applied through the manager"""
        master_id = yield self.collection.insert(self.mini_doc)
        manager = AsyncRevisionStackManager(settings)
        stack = AsyncSchedulableDocumentRevisionStack("test_fixture",
                                                      settings,
                                                      master_id=master_id)

        self.mini_doc[test_attr] = test_val
        yield stack.push(self.mini_doc, self.three_min_past_now)

        # Run the publish method manually
        yield manager.publish()

        #Make sure the live document contains our test values
        obj_check = yield self.collection.find_one_by_id(master_id)
        self.assertEqual(test_val, obj_check.get(test_attr))

    @gen_test
    def test_publish_with_insert_action(self):
        """Test that we can schedule a new collection object"""
        stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings)
        meta = {"comment": "foo"}
        bson_id = yield stack.push(self.mini_doc,
                                   toa=self.three_min_past_now,
                                   meta=meta)
        revisions = yield stack.list()
        self.assertEqual(len(revisions), 1)
        revision = revisions[0]
        self.assertEqual(revision.get("action"), "insert")
        # # Run the publish method manually
        # yield stack.pop()
        #
        # #Make sure the live document contains our test values
        # obj_check = yield self.collection.find_one_by_id(master_id)
        # self.assertIsNone(obj_check, "Object did not delete")

    @gen_test
    def test_publish_with_delete_action(self):
        """Test that we can delete a document on a schedule"""
        master_id = yield self.collection.insert(self.mini_doc)
        manager = AsyncRevisionStackManager(settings)
        stack = AsyncSchedulableDocumentRevisionStack("test_fixture",
                                                      settings,
                                                      master_id=master_id)
        meta = {"comment": "foo"}
        yield stack.push(None, toa=self.three_min_past_now, meta=meta)

        # Run the publish method manually
        yield stack.pop()

        #Make sure the live document contains our test values
        obj_check = yield self.collection.find_one_by_id(master_id)
        self.assertIsNone(obj_check, "Object did not delete")

    @raises(RevisionActionNotValid)
    @gen_test
    def test_stack_push_can_be_invalid_based_on_object_type(self):
        """Test that if we push the wrong type into a patch attribute, we fail correctly"""
        master_id = yield self.collection.insert(self.mini_doc)
        manager = AsyncRevisionStackManager(settings)
        stack = AsyncSchedulableDocumentRevisionStack("test_fixture",
                                                      settings,
                                                      master_id=master_id)

        yield stack.push("foo", toa=self.three_min_past_now)
        yield stack.push(False, toa=self.three_min_past_now)

    @gen_test
    def test_stack_can_produce_snapshot_of_future_revision_of_update_type(
            self):
        """Test that the stack can create a future state of a document"""
        master_id = yield self.collection.insert(self.test_fixture)

        stack = AsyncSchedulableDocumentRevisionStack("test_fixture",
                                                      settings,
                                                      master_id=master_id)

        update = {test_attr: test_val}

        yield stack.push(update, self.three_min_past_now)

        update["new_persistent"] = True
        yield stack.push(update, self.two_min_past_now)

        update["foo"] = "baz"
        id = yield stack.push(update, self.one_min_from_now)

        response = yield stack.preview(id)
        snapshot = response.get("snapshot")
        self.assertIsInstance(snapshot, dict)

        self.assertEqual(snapshot.get("bool_val"), True)
        self.assertEqual(snapshot.get("new_persistent"), True)
        self.assertEqual(snapshot.get("foo"), "baz")

    @gen_test(timeout=50)
    def test_stack_can_produce_snapshot_of_future_revision_of_insert_type(
            self):
        """Test that the stack can create a future state of a new yet to be created document"""

        stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings)

        fixture = self.test_fixture

        yield stack.push(self.test_fixture, self.three_min_past_now)

        fixture["baz"] = "bop"
        fixture["new_persistent"] = True
        yield stack.push(fixture, self.two_min_past_now)

        del fixture["new_persistent"]
        fixture["baz"] = "bit"
        id = yield stack.push(fixture, self.one_min_past_now)

        response = yield stack.preview(id)
        snapshot = response.get("snapshot")

        self.assertIsInstance(snapshot, dict)
        self.assertEqual(snapshot.get("bool_val"), True)
        self.assertEqual(snapshot.get("new_persistent"), True)
        self.assertEqual(snapshot.get("baz"), "bit")

    @gen_test(timeout=50)
    def test_stack_can_migrate_a_legacy_object_automatically(self):
        """Test the stack can migrate a legacy object automatically for the user"""
        client = BaseAsyncMotorDocument("test_fixture", settings)
        revisions = BaseAsyncMotorDocument("test_fixture_revisions", settings)

        fixture = self.test_fixture

        master_id = yield client.insert(fixture)

        stack = AsyncSchedulableDocumentRevisionStack("test_fixture",
                                                      settings,
                                                      master_id=master_id)

        fixture["baz"] = "bop"
        yield stack.push(self.test_fixture,
                         self.three_min_past_now,
                         meta={
                             "author": "UnitTest",
                             "comment": "Just a test, BRO."
                         })

        fixture["new_persistent"] = True
        yield stack.push(fixture, self.two_min_past_now)

        del fixture["new_persistent"]
        fixture["baz"] = "bit"
        id = yield stack.push(fixture, self.two_min_past_now)

        #response = yield stack.preview(id)
        list = yield revisions.find({"master_id": master_id})
        self.assertEqual(len(list), 4)
Example #30
0
class TestAsyncRevisionStackAndManagerFunctions(BaseAsyncTest):
    """ Test the Mongo Client funcitons here"""
    mini_doc = {
        "my doc" : "little bitty doc"
    }

    test_fixture = {
        "attr1": "attr1_val",
        "date1": time.mktime(datetime.datetime.now().timetuple()),
        "bool_val": True,
        "list_val": ["My Item", "Item 2", 1],
        "loc": [-75.22, 39.25],
        "sub_document" : mini_doc,
        "patch" :  {
            "foo": "bar"
        }
    }

    three_min_past_now = time.mktime((datetime.datetime.now() - datetime.timedelta(minutes=3)).timetuple())
    two_min_past_now = time.mktime((datetime.datetime.now() - datetime.timedelta(minutes=2)).timetuple())
    one_min_past_now = time.mktime((datetime.datetime.now() - datetime.timedelta(minutes=1)).timetuple())
    one_min_from_now = time.mktime((datetime.datetime.now() + datetime.timedelta(minutes=1)).timetuple())

    three_min_ahead_of_now = time.mktime((datetime.datetime.now() + datetime.timedelta(minutes=3)).timetuple())
    now = time.mktime(datetime.datetime.now().timetuple())

    def setUp(self):
        super(self.__class__, self).setUp()
        self.collection = BaseAsyncMotorDocument("test_fixture", settings)
        self.stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings)
        self.setup_database()

    @tornado.gen.coroutine
    def setup_database(self):
        self.collection.collection.drop()
        self.stack.revisions.collection.drop()
        #self.stack.previews.collection.drop()

    @tornado.testing.gen_test
    def test_push_on_stack(self):
        """Test that a revision is pushed onto the stack and stored into mongo"""
        id = yield self.stack.push(self.test_fixture, self.three_min_past_now)
        self.assertIsInstance(id, str)
        obj_id = ObjectId(id)
        self.assertIsInstance(obj_id, ObjectId)

    @tornado.testing.gen_test
    def test_patch_is_converted_and_storeable(self):
        """Test that a patch can be set with dot namespace safely and applied asynchronously via pop"""
        master_id = yield self.collection.insert(self.test_fixture)
        patch = {
            "patch.baz" : True
        }
        self.stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings, master_id=master_id)
        yield self.stack.push(patch, self.three_min_past_now)
        yield self.stack.pop()
        obj = yield self.collection.find_one_by_id(master_id)

        self.assertEqual(obj.get("patch").get("foo"), "bar")
        self.assertEqual(obj.get("patch").get("baz"), True)


    @tornado.testing.gen_test
    def test_list_of_revisions(self):
        """Test that we can create a list of revisions from a given document in a collection"""
        master_id = yield self.collection.insert(self.test_fixture)
        self.stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings, master_id=master_id)
        id1 = yield self.stack.push(self.test_fixture, self.three_min_past_now)
        id2 = yield self.stack.push(self.test_fixture, self.three_min_ahead_of_now)

        revisions = yield self.stack.list()
        self.assertEqual(len(revisions), 1,msg="Did not receive the correct number of revisions")
        self.assertEqual(revisions[0].get("id"), id1, msg="The order doesn't appear to be correct")

    @tornado.testing.gen_test
    def test_peek_on_stack(self):
        """Test that we get a single object off the top of the stack"""
        master_id = yield self.collection.insert(self.test_fixture)
        self.stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings, master_id=master_id)
        id1 = yield self.stack.push(self.test_fixture, time.mktime(datetime.datetime.strptime('24052010', "%d%m%Y").timetuple()))
        id2 = yield self.stack.push(self.test_fixture, time.mktime(datetime.datetime.now().timetuple()))
        peeked_obj = yield self.stack.peek()

        self.assertIsNotNone(peeked_obj)
        self.assertIsInstance(peeked_obj, dict)
        self.assertEqual(id1, peeked_obj.get("id"))
        self.assertEqual(peeked_obj.get("action"), "update")

    @gen_test
    def test_pop_off_stack(self):
        """Test that our stack.pop method updates all relevant data correctly"""
        master_id = yield self.collection.insert(self.mini_doc)
        self.stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings,master_id=master_id)
        self.mini_doc[test_attr] = test_val
        id1 = yield self.stack.push(self.mini_doc, self.three_min_past_now)
        id2 = yield self.stack.push(self.mini_doc, self.now)
        obj = yield self.stack.pop()
        self.assertIsInstance(obj, dict)
        self.assertEqual(obj["processed"], True)
        self.assertEqual(obj["snapshot"][test_attr], test_val)

        obj_check = yield self.collection.find_one_by_id(master_id)
        self.assertEqual(test_val, obj_check.get(test_attr))

    @gen_test
    def test_publish_scheduled_pop_off_stack_w_manager(self):
        """Test that we can schedule a revision for 3 min ago and that the revision is applied through the manager"""
        master_id = yield self.collection.insert(self.mini_doc)
        manager = AsyncRevisionStackManager(settings)
        stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings, master_id=master_id)

        self.mini_doc[test_attr] = test_val
        yield stack.push(self.mini_doc, self.three_min_past_now)

        # Run the publish method manually
        yield manager.publish()

        #Make sure the live document contains our test values
        obj_check = yield self.collection.find_one_by_id(master_id)
        self.assertEqual(test_val, obj_check.get(test_attr))

    @gen_test
    def test_publish_with_insert_action(self):
        """Test that we can schedule a new collection object"""
        stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings)
        meta = {
            "comment" : "foo"
        }
        bson_id = yield stack.push(self.mini_doc, toa=self.three_min_past_now, meta=meta)
        revisions = yield stack.list()
        self.assertEqual(len(revisions), 1)
        revision = revisions[0]
        self.assertEqual(revision.get("action"), "insert")
        # # Run the publish method manually
        # yield stack.pop()
        #
        # #Make sure the live document contains our test values
        # obj_check = yield self.collection.find_one_by_id(master_id)
        # self.assertIsNone(obj_check, "Object did not delete")


    @gen_test
    def test_publish_with_delete_action(self):
        """Test that we can delete a document on a schedule"""
        master_id = yield self.collection.insert(self.mini_doc)
        manager = AsyncRevisionStackManager(settings)
        stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings, master_id=master_id)
        meta = {
            "comment" : "foo"
        }
        yield stack.push(None, toa=self.three_min_past_now, meta=meta)

        # Run the publish method manually
        yield stack.pop()

        #Make sure the live document contains our test values
        obj_check = yield self.collection.find_one_by_id(master_id)
        self.assertIsNone(obj_check, "Object did not delete")

    @raises(RevisionActionNotValid)
    @gen_test
    def test_stack_push_can_be_invalid_based_on_object_type(self):
        """Test that if we push the wrong type into a patch attribute, we fail correctly"""
        master_id = yield self.collection.insert(self.mini_doc)
        manager = AsyncRevisionStackManager(settings)
        stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings, master_id=master_id)

        yield stack.push("foo", toa=self.three_min_past_now)
        yield stack.push(False, toa=self.three_min_past_now)

    @gen_test
    def test_stack_can_produce_snapshot_of_future_revision_of_update_type(self):
        """Test that the stack can create a future state of a document"""
        master_id = yield self.collection.insert(self.test_fixture)

        stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings, master_id=master_id)

        update = {
            test_attr: test_val
        }

        yield stack.push(update, self.three_min_past_now)

        update["new_persistent"] = True
        yield stack.push(update, self.two_min_past_now)

        update["foo"] = "baz"
        id = yield stack.push(update, self.one_min_from_now)

        response = yield stack.preview(id)
        snapshot = response.get("snapshot")
        self.assertIsInstance(snapshot, dict)

        self.assertEqual(snapshot.get("bool_val"), True)
        self.assertEqual(snapshot.get("new_persistent"), True)
        self.assertEqual(snapshot.get("foo"), "baz")

    @gen_test(timeout=50)
    def test_stack_can_produce_snapshot_of_future_revision_of_insert_type(self):
        """Test that the stack can create a future state of a new yet to be created document"""

        stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings)

        fixture = self.test_fixture

        yield stack.push(self.test_fixture, self.three_min_past_now)

        fixture["baz"] = "bop"
        fixture["new_persistent"] = True
        yield stack.push(fixture, self.two_min_past_now)

        del fixture["new_persistent"]
        fixture["baz"] = "bit"
        id = yield stack.push(fixture, self.one_min_past_now)


        response = yield stack.preview(id)
        snapshot = response.get("snapshot")

        self.assertIsInstance(snapshot, dict)
        self.assertEqual(snapshot.get("bool_val"), True)
        self.assertEqual(snapshot.get("new_persistent"), True)
        self.assertEqual(snapshot.get("baz"), "bit")

    @gen_test(timeout=50)
    def test_stack_can_migrate_a_legacy_object_automatically(self):
        """Test the stack can migrate a legacy object automatically for the user"""
        client = BaseAsyncMotorDocument("test_fixture", settings)
        revisions = BaseAsyncMotorDocument("test_fixture_revisions", settings)

        fixture = self.test_fixture

        master_id = yield client.insert(fixture)

        stack = AsyncSchedulableDocumentRevisionStack("test_fixture", settings, master_id=master_id)

        fixture["baz"] = "bop"
        yield stack.push(self.test_fixture, self.three_min_past_now, meta={"author": "UnitTest", "comment": "Just a test, BRO."})

        fixture["new_persistent"] = True
        yield stack.push(fixture, self.two_min_past_now)

        del fixture["new_persistent"]
        fixture["baz"] = "bit"
        id = yield stack.push(fixture, self.two_min_past_now)

        #response = yield stack.preview(id)
        list = yield revisions.find({"master_id": master_id})
        self.assertEqual(len(list), 4)
Example #31
0
class RevisionHandler(BaseRestfulMotorHandler):
    def initialize(self):
        """Initializer for the Search Handler"""
        super(self.__class__, self).initialize()
        self.client = None

    @coroutine
    def put(self, id):
        """
        Update a revision by ID

        :param id: BSON id
        :return:
        """

        collection_name = self.request.headers.get("collection")

        if not collection_name:
            self.raise_error(400, "Missing a collection name header")

        self.client = BaseAsyncMotorDocument("%s_revisions" % collection_name)

        super(self.__class__, self).put(id)

    @coroutine
    def delete(self, id):
        """
        Delete a revision by ID

        :param id: BSON id
        :return:
        """

        collection_name = self.request.headers.get("collection")

        if not collection_name:
            self.raise_error(400, "Missing a collection name header")

        self.client = BaseAsyncMotorDocument("%s_revisions" % collection_name)

        super(self.__class__, self).delete(id)

    @coroutine
    def post(self, id=None):
        """
        Create a revision manually without the stack

        :param id: BSON id
        :return: JSON
        """
        collection_name = self.request.headers.get("collection")

        if not collection_name:
            self.raise_error(400, "Missing a collection name header")

        self.client = BaseAsyncMotorDocument("%s_revisions" % collection_name)

        super(self.__class__, self).post(id)

    @coroutine
    def get(self, id):
        """
        Get revision based on the stack preview algorithm

        :param id: BSON id
        :return: JSON
        """
        collection_name = self.request.headers.get("collection")

        if not collection_name:
            self.raise_error(400, "Missing a collection name for stack")

        self.stack = AsyncSchedulableDocumentRevisionStack(
            collection_name, self.settings)

        revision = yield self.stack.preview(id)
        self.write(revision)
Example #32
0
    def put(self, id):
        """
        Update a resource by bson ObjectId

        :returns: json string representation
        :rtype: JSON
        """
        try:
            #Async update flow
            object_ = json_util.loads(self.request.body)

            toa = self.request.headers.get("Caesium-TOA", None)

            obj_check = yield self.client.find_one_by_id(id)
            if not obj_check:
                self.raise_error(404, "Resource not found: %s" % id)
                self.finish()
                return

            if toa:

                stack = AsyncSchedulableDocumentRevisionStack(
                    self.client.collection_name, self.settings, master_id=id)
                revision_id = yield stack.push(object_,
                                               int(toa),
                                               meta=self._get_meta_data())

                if isinstance(revision_id, str):
                    self.set_header("Caesium-TOA", toa)

                    #We add the id of the original request, because we don't want to infer this
                    #On the client side, as the state of the client code could change easily
                    #We want this request to return with the originating ID as well.
                    object_["id"] = id
                    self.return_resource(object_)
                else:
                    self.raise_error(
                        404, "Revision not scheduled for object: %s" % id)

            else:
                if object_.get("_id"):
                    del object_["_id"]

                response = yield self.client.update(id, object_)

                if response.get("updatedExisting"):
                    object_ = yield self.client.find_one_by_id(id)
                    self.return_resource(object_)
                else:
                    self.raise_error(404, "Resource not found: %s" % id)

        except ValidationError as vex:
            self.logger.error("%s validation error" % self.object_name, vex)
            self.raise_error(
                400,
                "Your %s cannot be updated because it is missing required fields, see docs"
                % self.object_name)
        except ValueError as ex:
            self.raise_error(400,
                             "Invalid JSON Body, check formatting. %s" % ex[0])
        except InvalidId as ex:
            self.raise_error(message="Your ID is malformed: %s" % id)
        except Exception as ex:
            self.logger.error(ex)
            self.raise_error()
Example #33
0
class RevisionHandler(BaseRestfulMotorHandler):

    def initialize(self):
        """Initializer for the Search Handler"""
        super(self.__class__, self).initialize()
        self.client = None

    @coroutine
    def put(self, id):
        """
        Update a revision by ID

        :param id: BSON id
        :return:
        """

        collection_name = self.request.headers.get("collection")

        if not collection_name:
            self.raise_error(400, "Missing a collection name header")

        self.client = BaseAsyncMotorDocument("%s_revisions" % collection_name)

        super(self.__class__, self).put(id)

    @coroutine
    def delete(self, id):
        """
        Delete a revision by ID

        :param id: BSON id
        :return:
        """

        collection_name = self.request.headers.get("collection")

        if not collection_name:
            self.raise_error(400, "Missing a collection name header")

        self.client = BaseAsyncMotorDocument("%s_revisions" % collection_name)

        super(self.__class__, self).delete(id)

    @coroutine
    def post(self, id=None):
        """
        Create a revision manually without the stack

        :param id: BSON id
        :return: JSON
        """
        collection_name = self.request.headers.get("collection")

        if not collection_name:
            self.raise_error(400, "Missing a collection name header")

        self.client = BaseAsyncMotorDocument("%s_revisions" % collection_name)

        super(self.__class__, self).post(id)


    @coroutine
    def get(self, id):
        """
        Get revision based on the stack preview algorithm

        :param id: BSON id
        :return: JSON
        """
        collection_name = self.request.headers.get("collection")

        if not collection_name:
            self.raise_error(400, "Missing a collection name for stack")

        self.stack = AsyncSchedulableDocumentRevisionStack(collection_name, self.settings)

        revision = yield self.stack.preview(id)
        self.write(revision)