Exemple #1
0
    def create_initial_message_tables(self):
        """Initializes a dict of the initial rotating messages tables.

        An entry for last months table, and an entry for this months table.

        """
        last_month = get_rotating_message_table(self._message_prefix, -1)
        this_month = get_rotating_message_table(self._message_prefix)
        self.message_tables = {
            last_month.table_name: Message(last_month, self.metrics),
            this_month.table_name: Message(this_month, self.metrics),
        }
Exemple #2
0
    def test_message_storage(self):
        chid = str(uuid.uuid4())
        chid2 = str(uuid.uuid4())
        m = get_rotating_message_table()
        message = Message(m, SinkMetrics())
        message.register_channel(self.uaid, chid)
        message.register_channel(self.uaid, chid2)

        data1 = str(uuid.uuid4())
        data2 = str(uuid.uuid4())
        ttl = int(time.time()) + 100
        time1, time2, time3 = self._nstime(), self._nstime(
        ), self._nstime() + 1
        message.store_message(self.uaid, chid, time1, ttl, data1, {})
        message.store_message(self.uaid, chid2, time2, ttl, data2, {})
        message.store_message(self.uaid, chid2, time3, ttl, data1, {})

        all_messages = list(message.fetch_messages(self.uaid))
        eq_(len(all_messages), 3)

        message.delete_messages_for_channel(self.uaid, chid2)
        all_messages = list(message.fetch_messages(self.uaid))
        eq_(len(all_messages), 1)

        message.delete_message(self.uaid, chid, time1)
        all_messages = list(message.fetch_messages(self.uaid))
        eq_(len(all_messages), 0)
Exemple #3
0
    def test_unregister(self):
        chid = str(uuid.uuid4())
        m = get_rotating_message_table()
        message = Message(m, SinkMetrics())
        message.register_channel(self.uaid, chid)

        # Verify its in the db
        rows = m.query_2(uaid__eq=self.uaid, chidmessageid__eq=" ")
        results = list(rows)
        assert (len(results) == 1)
        eq_(results[0]["chids"], set([chid]))

        message.unregister_channel(self.uaid, chid)

        # Verify its not in the db
        rows = m.query_2(uaid__eq=self.uaid, chidmessageid__eq=" ")
        results = list(rows)
        assert (len(results) == 1)
        eq_(results[0]["chids"], set([]))

        # Test for the very unlikely case that there's no 'chid'
        m.connection.update_item = Mock()
        m.connection.update_item.return_value = {
            'Attributes': {
                'uaid': {
                    'S': self.uaid
                }
            },
            'ConsumedCapacityUnits': 0.5
        }
        r = message.unregister_channel(self.uaid, dummy_chid)
        eq_(r, False)
Exemple #4
0
    def test_message_delete_pagination(self):
        def make_messages(channel_id, count):
            m = []
            t = self._nstime()
            ttl = int(time.time()) + 200
            for i in range(count):
                m.append(
                    (self.uaid, channel_id, str(uuid.uuid4()), ttl, {}, t + i))
            return m

        chid = str(uuid.uuid4())
        m = get_rotating_message_table()
        message = Message(m, SinkMetrics())
        message.register_channel(self.uaid, chid)

        # Shove 80 messages in
        for message_args in make_messages(chid, 80):
            message.store_message(*message_args)

        # Verify we can see them all
        all_messages = list(message.fetch_messages(self.uaid, limit=100))
        eq_(len(all_messages), 80)

        # Delete them all
        message.delete_messages_for_channel(self.uaid, chid)

        # Verify they're gone
        all_messages = list(message.fetch_messages(self.uaid, limit=100))
        eq_(len(all_messages), 0)
Exemple #5
0
    def update_rotating_tables(self):
        """This method is intended to be tasked to run periodically off the
        twisted event hub to rotate tables.

        When today is a new month from yesterday, then we swap out all the
        table objects on the settings object.

        """
        today = datetime.date.today()
        if today.month == self.current_month:
            # No change in month, we're fine.
            returnValue(False)

        # Get tables for the new month, and verify they exist before we try to
        # switch over
        message_table = get_rotating_message_table(self._message_prefix)

        try:
            yield deferToThread(message_table.describe)
        except Exception:
            tblname = make_rotating_tablename(self._message_prefix)
            log.err("Unable to locate new message table: %s" % tblname)
            returnValue(False)

        # Both tables found, safe to switch-over
        self.current_month = today.month
        self.current_msg_month = message_table.table_name
        self.message_tables[self.current_msg_month] = \
            Message(message_table, self.metrics)
        returnValue(True)
Exemple #6
0
    def test_message_rotate_table_with_date(self):
        prefix = "message" + uuid.uuid4().hex
        future = datetime.today() + timedelta(days=32)
        tbl_name = make_rotating_tablename(prefix, date=future)

        m = get_rotating_message_table(prefix=prefix, date=future)
        eq_(m.table_name, tbl_name)
Exemple #7
0
    def test_unregister(self):
        chid = str(uuid.uuid4())
        m = get_rotating_message_table()
        message = Message(m, SinkMetrics())
        message.register_channel(self.uaid, chid)

        # Verify its in the db
        rows = m.query_2(uaid__eq=self.uaid, chidmessageid__eq=" ")
        results = list(rows)
        assert len(results) == 1
        eq_(results[0]["chids"], set([chid]))

        message.unregister_channel(self.uaid, chid)

        # Verify its not in the db
        rows = m.query_2(uaid__eq=self.uaid, chidmessageid__eq=" ")
        results = list(rows)
        assert len(results) == 1
        eq_(results[0]["chids"], set([]))

        # Test for the very unlikely case that there's no 'chid'
        m.connection.update_item = Mock()
        m.connection.update_item.return_value = {"Attributes": {"uaid": {"S": self.uaid}}, "ConsumedCapacityUnits": 0.5}
        r = message.unregister_channel(self.uaid, "test")
        eq_(r, False)
Exemple #8
0
    def test_message_storage(self):
        chid = str(uuid.uuid4())
        chid2 = str(uuid.uuid4())
        m = get_rotating_message_table()
        message = Message(m, SinkMetrics())
        message.register_channel(self.uaid, chid)
        message.register_channel(self.uaid, chid2)

        data1 = str(uuid.uuid4())
        data2 = str(uuid.uuid4())
        ttl = int(time.time()) + 100
        time1, time2, time3 = self._nstime(), self._nstime(), self._nstime() + 1
        message.store_message(self.uaid, chid, time1, ttl, data1, {})
        message.store_message(self.uaid, chid2, time2, ttl, data2, {})
        message.store_message(self.uaid, chid2, time3, ttl, data1, {})

        all_messages = list(message.fetch_messages(self.uaid))
        eq_(len(all_messages), 3)

        message.delete_messages_for_channel(self.uaid, chid2)
        all_messages = list(message.fetch_messages(self.uaid))
        eq_(len(all_messages), 1)

        message.delete_message(self.uaid, chid, time1)
        all_messages = list(message.fetch_messages(self.uaid))
        eq_(len(all_messages), 0)
Exemple #9
0
    def test_message_delete_pagination(self):
        def make_messages(channel_id, count):
            m = []
            t = self._nstime()
            ttl = int(time.time()) + 200
            for i in range(count):
                m.append((self.uaid, channel_id, str(uuid.uuid4()), ttl, {}, t + i))
            return m

        chid = str(uuid.uuid4())
        m = get_rotating_message_table()
        message = Message(m, SinkMetrics())
        message.register_channel(self.uaid, chid)

        # Shove 80 messages in
        for message_args in make_messages(chid, 80):
            message.store_message(*message_args)

        # Verify we can see them all
        all_messages = list(message.fetch_messages(self.uaid, limit=100))
        eq_(len(all_messages), 80)

        # Delete them all
        message.delete_messages_for_channel(self.uaid, chid)

        # Verify they're gone
        all_messages = list(message.fetch_messages(self.uaid, limit=100))
        eq_(len(all_messages), 0)
Exemple #10
0
    def update_rotating_tables(self):
        """This method is intended to be tasked to run periodically off the
        twisted event hub to rotate tables.

        When today is a new month from yesterday, then we swap out all the
        table objects on the settings object.

        """
        today = datetime.date.today()
        tomorrow = self._tomorrow()
        if ((tomorrow.month != today.month)
                and sorted(self.message_tables.keys())[-1] != tomorrow.month):
            next_month = get_rotating_message_table(self._message_prefix, 0,
                                                    tomorrow)
            self.message_tables[next_month.table_name] = Message(
                next_month, self.metrics)

        if today.month == self.current_month:
            # No change in month, we're fine.
            returnValue(False)

        # Get tables for the new month, and verify they exist before we try to
        # switch over
        message_table = yield deferToThread(get_rotating_message_table,
                                            self._message_prefix)

        # Both tables found, safe to switch-over
        self.current_month = today.month
        self.current_msg_month = message_table.table_name
        self.message_tables[self.current_msg_month] = \
            Message(message_table, self.metrics)
        returnValue(True)
Exemple #11
0
    def update_rotating_tables(self):
        """This method is intended to be tasked to run periodically off the
        twisted event hub to rotate tables.

        When today is a new month from yesterday, then we swap out all the
        table objects on the settings object.

        """
        today = datetime.date.today()
        tomorrow = self._tomorrow()
        if ((tomorrow.month != today.month) and
                sorted(self.message_tables.keys())[-1] !=
                tomorrow.month):
            next_month = get_rotating_message_table(
                self._message_prefix, 0, tomorrow)
            self.message_tables[next_month.table_name] = Message(
                next_month, self.metrics)

        if today.month == self.current_month:
            # No change in month, we're fine.
            returnValue(False)

        # Get tables for the new month, and verify they exist before we try to
        # switch over
        message_table = yield deferToThread(get_rotating_message_table,
                                            self._message_prefix)

        # Both tables found, safe to switch-over
        self.current_month = today.month
        self.current_msg_month = message_table.table_name
        self.message_tables[self.current_msg_month] = \
            Message(message_table, self.metrics)
        returnValue(True)
Exemple #12
0
    def test_message_rotate_table_with_date(self):
        prefix = "message" + uuid.uuid4().hex
        future = (datetime.today() + timedelta(days=32)).date()
        tbl_name = make_rotating_tablename(prefix, date=future)

        m = get_rotating_message_table(prefix=prefix, date=future)
        assert m.table_name == tbl_name
        # Clean up the temp table.
        _drop_table(tbl_name)
Exemple #13
0
    def test_register(self):
        chid = str(uuid.uuid4())
        m = get_rotating_message_table()
        message = Message(m, SinkMetrics())
        message.register_channel(self.uaid, chid)

        # Verify its in the db
        rows = m.query_2(uaid__eq=self.uaid, chidmessageid__eq=" ")
        results = list(rows)
        assert len(results) == 1
Exemple #14
0
    def create_initial_message_tables(self):
        """Initializes a dict of the initial rotating messages tables.

        An entry for last months table, and an entry for this months table.

        """
        last_month = get_rotating_message_table(self._message_prefix, -1)
        this_month = get_rotating_message_table(self._message_prefix)
        # Verify the table exists
        try:
            this_month.describe()
        except Exception:
            # Create the message table
            create_rotating_message_table(prefix=self._message_prefix)
            this_month = get_rotating_message_table(self._message_prefix)
        self.message_tables = {
            last_month.table_name: Message(last_month, self.metrics),
            this_month.table_name: Message(this_month, self.metrics),
        }
Exemple #15
0
    def test_register(self):
        chid = str(uuid.uuid4())
        m = get_rotating_message_table()
        message = Message(m, SinkMetrics())
        message.register_channel(self.uaid, chid)

        # Verify its in the db
        rows = m.query_2(uaid__eq=self.uaid, chidmessageid__eq=" ")
        results = list(rows)
        assert (len(results) == 1)
Exemple #16
0
    def test_update_message_fail(self):
        message = Message(get_rotating_message_table(), SinkMetrics)
        message.store_message(self.uaid, uuid.uuid4().hex, self._nstime(), str(uuid.uuid4()), {})
        u = message.table.connection.update_item = Mock()

        def raise_condition(*args, **kwargs):
            raise ConditionalCheckFailedException(None, None)

        u.side_effect = raise_condition
        b = message.update_message(self.uaid, uuid.uuid4().hex, self._nstime(), str(uuid.uuid4()), {})
        eq_(b, False)
Exemple #17
0
    def test_preflight_check(self):
        router = Router(get_router_table(), SinkMetrics())
        message = Message(get_rotating_message_table(), SinkMetrics())

        pf_uaid = "deadbeef00000000deadbeef01010101"
        preflight_check(message, router, pf_uaid)
        # now check that the database reports no entries.
        _, notifs = message.fetch_messages(uuid.UUID(pf_uaid))
        assert len(notifs) == 0
        with pytest.raises(ItemNotFound):
            router.get_uaid(pf_uaid)
Exemple #18
0
    def create_initial_message_tables(self):
        """Initializes a dict of the initial rotating messages tables.

        An entry for last months table, an entry for this months table,
        an entry for tomorrow, if tomorrow is a new month.

        """
        today = datetime.date.today()
        last_month = get_rotating_message_table(self._message_prefix, -1)
        this_month = get_rotating_message_table(self._message_prefix)
        self.current_month = today.month
        self.current_msg_month = this_month.table_name
        self.message_tables = {
            last_month.table_name: Message(last_month, self.metrics),
            this_month.table_name: Message(this_month, self.metrics)
        }
        if self._tomorrow().month != today.month:
            next_month = get_rotating_message_table(delta=1)
            self.message_tables[next_month.table_name] = Message(
                next_month, self.metrics)
Exemple #19
0
    def test_message_delete_fail_condition(self):
        m = get_rotating_message_table()
        message = Message(m, SinkMetrics())

        def raise_condition(*args, **kwargs):
            raise ConditionalCheckFailedException(None, None)

        message.table = Mock()
        message.table.delete_item.side_effect = raise_condition
        result = message.delete_message(uaid="asdf", channel_id="asdf", message_id="asdf", updateid="asdf")
        eq_(result, False)
Exemple #20
0
    def create_initial_message_tables(self):
        """Initializes a dict of the initial rotating messages tables.

        An entry for last months table, an entry for this months table,
        an entry for tomorrow, if tomorrow is a new month.

        """
        today = datetime.date.today()
        last_month = get_rotating_message_table(self._message_prefix, -1)
        this_month = get_rotating_message_table(self._message_prefix)
        self.current_month = today.month
        self.current_msg_month = this_month.table_name
        self.message_tables = {
            last_month.table_name: Message(last_month, self.metrics),
            this_month.table_name: Message(this_month, self.metrics)
        }
        if self._tomorrow().month != today.month:
            next_month = get_rotating_message_table(delta=1)
            self.message_tables[next_month.table_name] = Message(
                next_month, self.metrics)
Exemple #21
0
    def test_unregister(self):
        chid = str(uuid.uuid4())
        m = get_rotating_message_table()
        message = Message(m, SinkMetrics())
        message.register_channel(self.uaid, chid)

        # Verify its in the db
        response = m.query(
            KeyConditions={
                'uaid': {
                    'AttributeValueList': [self.uaid],
                    'ComparisonOperator': 'EQ'
                },
                'chidmessageid': {
                    'AttributeValueList': [" "],
                    'ComparisonOperator': 'EQ'
                },
            },
            ConsistentRead=True,
        )
        results = list(response.get('Items'))
        assert len(results) == 1
        assert results[0]["chids"] == {chid}

        message.unregister_channel(self.uaid, chid)

        # Verify its not in the db
        response = m.query(
            KeyConditions={
                'uaid': {
                    'AttributeValueList': [self.uaid],
                    'ComparisonOperator': 'EQ'
                },
                'chidmessageid': {
                    'AttributeValueList': [" "],
                    'ComparisonOperator': 'EQ'
                },
            },
            ConsistentRead=True,
        )
        results = list(response.get('Items'))
        assert len(results) == 1
        assert results[0].get("chids") is None

        # Test for the very unlikely case that there's no 'chid'
        m.update_item = Mock(return_value={
            'Attributes': {
                'uaid': self.uaid
            },
            'ResponseMetaData': {}
        })
        r = message.unregister_channel(self.uaid, dummy_chid)
        assert r is False
Exemple #22
0
    def test_preflight_check_fail(self):
        router = Router(get_router_table(), SinkMetrics())
        message = Message(get_rotating_message_table(), SinkMetrics())

        def raise_exc(*args, **kwargs):  # pragma: no cover
            raise Exception("Oops")

        router.clear_node = Mock()
        router.clear_node.side_effect = raise_exc

        with pytest.raises(Exception):
            preflight_check(message, router)
Exemple #23
0
    def test_save_channels(self):
        chid = str(uuid.uuid4())
        chid2 = str(uuid.uuid4())
        m = get_rotating_message_table()
        message = Message(m, SinkMetrics())
        message.register_channel(self.uaid, chid)
        message.register_channel(self.uaid, chid2)

        exists, chans = message.all_channels(self.uaid)
        new_uaid = uuid.uuid4().hex
        message.save_channels(new_uaid, chans)
        _, new_chans = message.all_channels(new_uaid)
        eq_(chans, new_chans)
Exemple #24
0
    def test_save_channels(self):
        chid = str(uuid.uuid4())
        chid2 = str(uuid.uuid4())
        m = get_rotating_message_table()
        message = Message(m, SinkMetrics())
        message.register_channel(self.uaid, chid)
        message.register_channel(self.uaid, chid2)

        exists, chans = message.all_channels(self.uaid)
        new_uaid = uuid.uuid4().hex
        message.save_channels(new_uaid, chans)
        _, new_chans = message.all_channels(new_uaid)
        eq_(chans, new_chans)
Exemple #25
0
 def test_update_message(self):
     chid = uuid.uuid4().hex
     m = get_rotating_message_table()
     message = Message(m, SinkMetrics())
     data1 = str(uuid.uuid4())
     data2 = str(uuid.uuid4())
     time1 = self._nstime()
     time2 = self._nstime() + 100
     ttl = self._nstime() + 1000
     message.store_message(self.uaid, chid, time1, ttl, data1, {})
     message.update_message(self.uaid, chid, time2, ttl, data2, {})
     messages = list(message.fetch_messages(self.uaid))
     eq_(data2, messages[0]["#dd"])
Exemple #26
0
    def test_message_delete_fail_condition(self):
        notif = make_webpush_notification(dummy_uaid, dummy_chid)
        notif.message_id = notif.update_id = dummy_uaid
        m = get_rotating_message_table()
        message = Message(m, SinkMetrics())

        def raise_condition(*args, **kwargs):
            raise ConditionalCheckFailedException(None, None)

        message.table = Mock()
        message.table.delete_item.side_effect = raise_condition
        result = message.delete_message(notif)
        eq_(result, False)
Exemple #27
0
 def test_update_message(self):
     chid = uuid.uuid4().hex
     m = get_rotating_message_table()
     message = Message(m, SinkMetrics())
     data1 = str(uuid.uuid4())
     data2 = str(uuid.uuid4())
     time1 = self._nstime()
     time2 = self._nstime() + 100
     ttl = self._nstime() + 1000
     message.store_message(self.uaid, chid, time1, ttl, data1, {})
     message.update_message(self.uaid, chid, time2, ttl, data2, {})
     messages = list(message.fetch_messages(self.uaid))
     eq_(data2, messages[0]['#dd'])
Exemple #28
0
    def test_message_delete_fail_condition(self):
        notif = make_webpush_notification(dummy_uaid, dummy_chid)
        notif.message_id = notif.update_id = dummy_uaid
        m = get_rotating_message_table()
        message = Message(m, SinkMetrics())

        def raise_condition(*args, **kwargs):
            raise ClientError({}, 'delete_item')

        message.table = Mock()
        message.table.delete_item.side_effect = raise_condition
        result = message.delete_message(notif)
        assert result is False
Exemple #29
0
    def test_all_channels_fail(self):
        m = get_rotating_message_table()
        message = Message(m, SinkMetrics())

        message.table.get_item = Mock()
        message.table.get_item.return_value = {
            "ResponseMetadata": {
                "HTTPStatusCode": 400
            },
        }

        res = message.all_channels(self.uaid)
        assert res == (False, set([]))
Exemple #30
0
    def test_preflight_check_wait(self):
        router = Router(get_router_table(), SinkMetrics())
        message = Message(get_rotating_message_table(), SinkMetrics())

        values = ["PENDING", "ACTIVE"]
        message.table_status = Mock(side_effect=values)

        pf_uaid = "deadbeef00000000deadbeef01010101"
        preflight_check(message, router, pf_uaid)
        # now check that the database reports no entries.
        _, notifs = message.fetch_messages(uuid.UUID(pf_uaid))
        assert len(notifs) == 0
        with pytest.raises(ItemNotFound):
            router.get_uaid(pf_uaid)
Exemple #31
0
    def test_message_delete_fail_condition(self):
        m = get_rotating_message_table()
        message = Message(m, SinkMetrics())

        def raise_condition(*args, **kwargs):
            raise ConditionalCheckFailedException(None, None)

        message.table = Mock()
        message.table.delete_item.side_effect = raise_condition
        result = message.delete_message(uaid=dummy_uaid,
                                        channel_id=dummy_chid,
                                        message_id="asdf",
                                        updateid="asdf")
        eq_(result, False)
Exemple #32
0
    def test_update_message_fail(self):
        message = Message(get_rotating_message_table(), SinkMetrics)
        message.store_message(self.uaid,
                              uuid.uuid4().hex, self._nstime(),
                              str(uuid.uuid4()), {})
        u = message.table.connection.update_item = Mock()

        def raise_condition(*args, **kwargs):
            raise ConditionalCheckFailedException(None, None)

        u.side_effect = raise_condition
        b = message.update_message(self.uaid,
                                   uuid.uuid4().hex, self._nstime(),
                                   str(uuid.uuid4()), {})
        eq_(b, False)
Exemple #33
0
    def test_message_storage(self):
        chid = str(uuid.uuid4())
        chid2 = str(uuid.uuid4())
        m = get_rotating_message_table()
        message = Message(m, SinkMetrics())
        message.register_channel(self.uaid, chid)
        message.register_channel(self.uaid, chid2)

        message.store_message(make_webpush_notification(self.uaid, chid))
        message.store_message(make_webpush_notification(self.uaid, chid))
        message.store_message(make_webpush_notification(self.uaid, chid))

        _, all_messages = message.fetch_timestamp_messages(
            uuid.UUID(self.uaid), " ")
        eq_(len(all_messages), 3)
Exemple #34
0
    def test_all_channels(self):
        chid = str(uuid.uuid4())
        chid2 = str(uuid.uuid4())
        m = get_rotating_message_table()
        message = Message(m, SinkMetrics())
        message.register_channel(self.uaid, chid)
        message.register_channel(self.uaid, chid2)

        _, chans = message.all_channels(self.uaid)
        assert chid in chans
        assert chid2 in chans

        message.unregister_channel(self.uaid, chid2)
        _, chans = message.all_channels(self.uaid)
        assert chid2 not in chans
        assert chid in chans
Exemple #35
0
    def test_all_channels(self):
        chid = str(uuid.uuid4())
        chid2 = str(uuid.uuid4())
        m = get_rotating_message_table()
        message = Message(m, SinkMetrics())
        message.register_channel(self.uaid, chid)
        message.register_channel(self.uaid, chid2)

        _, chans = message.all_channels(self.uaid)
        assert (chid in chans)
        assert (chid2 in chans)

        message.unregister_channel(self.uaid, chid2)
        _, chans = message.all_channels(self.uaid)
        assert (chid2 not in chans)
        assert (chid in chans)
Exemple #36
0
    def test_update_rotating_tables_month_end(self):
        """Test that rotating adds next months table

        This test is intended to ensure that if the next day is a new
        month, then update_rotating_tables realizes this and add's
        the new table to the message_tables.

        A pre-requisite is that today cannot be the last day of
        the current month. Therefore, we first sub in _tomorrow to
        ensure it always appears as next month, and then remove
        the new table create_initial_tables made so we can observe
        update_rotating_tables add the new one.

        Note that sorting message table keys to find the last month
        does *not work* since the month digit is not zero-padded.

        """
        today = datetime.date.today()
        next_month = today.month + 1
        next_year = today.year
        if next_month > 12:  # pragma: nocover
            next_month = 1
            next_year += 1
        tomorrow = datetime.datetime(year=next_year, month=next_month, day=1)

        conf = AutopushConfig(hostname="example.com", resolve_hostname=True)
        db = DatabaseManager.from_config(conf)
        db._tomorrow = Mock(return_value=tomorrow)
        db.create_initial_message_tables()

        # We should have 3 tables, one for next/this/last month
        assert len(db.message_tables) == 3

        # Grab next month's table name and remove it
        next_month = get_rotating_message_table(conf.message_table.tablename,
                                                delta=1)
        db.message_tables.pop(next_month.table_name)

        # Get the deferred back
        d = db.update_rotating_tables()

        def check_tables(result):
            assert len(db.message_tables) == 3
            assert next_month.table_name in db.message_tables

        d.addCallback(check_tables)
        return d
Exemple #37
0
    def test_message_storage_overwrite(self):
        """Test that store_message can overwrite existing messages which
        can occur in some reconnect cases but shouldn't error"""
        chid = str(uuid.uuid4())
        chid2 = str(uuid.uuid4())
        notif1 = make_webpush_notification(self.uaid, chid)
        notif2 = make_webpush_notification(self.uaid, chid)
        notif3 = make_webpush_notification(self.uaid, chid2)
        notif2.message_id = notif1.message_id
        m = get_rotating_message_table()
        message = Message(m, SinkMetrics())
        message.register_channel(self.uaid, chid)
        message.register_channel(self.uaid, chid2)

        message.store_message(notif1)
        message.store_message(notif2)
        message.store_message(notif3)

        all_messages = list(message.fetch_messages(uuid.UUID(self.uaid)))
        eq_(len(all_messages), 2)
Exemple #38
0
    def test_register(self):
        chid = str(uuid.uuid4())
        m = get_rotating_message_table()
        message = Message(m, SinkMetrics())
        message.register_channel(self.uaid, chid)

        # Verify it's in the db
        response = m.query(
            KeyConditions={
                'uaid': {
                    'AttributeValueList': [self.uaid],
                    'ComparisonOperator': 'EQ'
                },
                'chidmessageid': {
                    'AttributeValueList': ['02'],
                    'ComparisonOperator': 'LT'
                }
            },
            ConsistentRead=True,
        )
        assert len(response.get('Items'))
Exemple #39
0
 def test_all_channels_no_uaid(self):
     m = get_rotating_message_table()
     message = Message(m, SinkMetrics())
     exists, chans = message.all_channels("asdf")
     assert chans == set([])
Exemple #40
0
    def __init__(self,
                 crypto_key=None,
                 datadog_api_key=None,
                 datadog_app_key=None,
                 datadog_flush_interval=None,
                 hostname=None,
                 port=None,
                 router_scheme=None,
                 router_hostname=None,
                 router_port=None,
                 endpoint_scheme=None,
                 endpoint_hostname=None,
                 endpoint_port=None,
                 router_conf={},
                 router_tablename="router",
                 router_read_throughput=5,
                 router_write_throughput=5,
                 storage_tablename="storage",
                 storage_read_throughput=5,
                 storage_write_throughput=5,
                 message_tablename="message",
                 message_read_throughput=5,
                 message_write_throughput=5,
                 statsd_host="localhost",
                 statsd_port=8125,
                 resolve_hostname=False,
                 max_data=4096,
                 # Reflected up from UDP Router
                 wake_timeout=0,
                 env='development',
                 enable_cors=False,
                 s3_bucket=DEFAULT_BUCKET,
                 senderid_expry=SENDERID_EXPRY,
                 senderid_list={},
                 hello_timeout=0,
                 auth_key=None,
                 ):
        """Initialize the Settings object

        Upon creation, the HTTP agent will initialize, all configured routers
        will be setup and started, logging will be started, and the database
        will have a preflight check done.

        """
        # Use a persistent connection pool for HTTP requests.
        pool = HTTPConnectionPool(reactor)
        self.agent = Agent(reactor, connectTimeout=5, pool=pool)

        # Metrics setup
        if datadog_api_key:
            self.metrics = DatadogMetrics(
                api_key=datadog_api_key,
                app_key=datadog_app_key,
                flush_interval=datadog_flush_interval
            )
        elif statsd_host:
            self.metrics = TwistedMetrics(statsd_host, statsd_port)
        else:
            self.metrics = SinkMetrics()
        if not crypto_key:
            crypto_key = [Fernet.generate_key()]
        if not isinstance(crypto_key, list):
            crypto_key = [crypto_key]
        self.update(crypto_key=crypto_key)
        self.crypto_key = crypto_key

        if auth_key is None:
            auth_key = []
        if not isinstance(auth_key, list):
            auth_key = [auth_key]
        self.auth_key = auth_key

        self.max_data = max_data
        self.clients = {}

        # Setup hosts/ports/urls
        default_hostname = socket.gethostname()
        self.hostname = hostname or default_hostname
        if resolve_hostname:
            self.hostname = resolve_ip(self.hostname)

        self.port = port
        self.endpoint_hostname = endpoint_hostname or self.hostname
        self.router_hostname = router_hostname or self.hostname

        self.router_conf = router_conf
        self.router_url = canonical_url(
            router_scheme or 'http',
            self.router_hostname,
            router_port
        )

        self.endpoint_url = canonical_url(
            endpoint_scheme or 'http',
            self.endpoint_hostname,
            endpoint_port
        )

        # Database objects
        self.router_table = get_router_table(router_tablename,
                                             router_read_throughput,
                                             router_write_throughput)
        self.storage_table = get_storage_table(
            storage_tablename,
            storage_read_throughput,
            storage_write_throughput)
        self.message_table = get_rotating_message_table(
            message_tablename)
        self._message_prefix = message_tablename
        self.storage = Storage(self.storage_table, self.metrics)
        self.router = Router(self.router_table, self.metrics)

        # Used to determine whether a connection is out of date with current
        # db objects
        self.current_msg_month = make_rotating_tablename(self._message_prefix)
        self.current_month = datetime.date.today().month
        self.create_initial_message_tables()

        # Run preflight check
        preflight_check(self.storage, self.router)

        # CORS
        self.cors = enable_cors

        # Force timeout in idle seconds
        self.wake_timeout = wake_timeout

        # Setup the routers
        self.routers = {}
        self.routers["simplepush"] = SimpleRouter(
            self,
            router_conf.get("simplepush")
        )
        self.routers["webpush"] = WebPushRouter(self, None)
        if 'apns' in router_conf:
            self.routers["apns"] = APNSRouter(self, router_conf["apns"])
        if 'gcm' in router_conf:
            self.routers["gcm"] = GCMRouter(self, router_conf["gcm"])

        # Env
        self.env = env

        self.hello_timeout = hello_timeout
Exemple #41
0
    def __init__(
        self,
        crypto_key=None,
        datadog_api_key=None,
        datadog_app_key=None,
        datadog_flush_interval=None,
        hostname=None,
        port=None,
        router_scheme=None,
        router_hostname=None,
        router_port=None,
        endpoint_scheme=None,
        endpoint_hostname=None,
        endpoint_port=None,
        router_conf=None,
        router_tablename="router",
        router_read_throughput=5,
        router_write_throughput=5,
        storage_tablename="storage",
        storage_read_throughput=5,
        storage_write_throughput=5,
        message_tablename="message",
        message_read_throughput=5,
        message_write_throughput=5,
        statsd_host="localhost",
        statsd_port=8125,
        resolve_hostname=False,
        max_data=4096,
        # Reflected up from UDP Router
        wake_timeout=0,
        env='development',
        enable_cors=False,
        hello_timeout=0,
        bear_hash_key=None,
        preflight_uaid="deadbeef00000000deadbeef00000000",
        ami_id=None,
        client_certs=None,
        msg_limit=100,
        debug=False,
        connect_timeout=0.5,
    ):
        """Initialize the Settings object

        Upon creation, the HTTP agent will initialize, all configured routers
        will be setup and started, logging will be started, and the database
        will have a preflight check done.

        """
        # Use a persistent connection pool for HTTP requests.
        pool = HTTPConnectionPool(reactor)
        if not debug:
            pool._factory = QuietClientFactory

        self.agent = Agent(reactor, connectTimeout=connect_timeout, pool=pool)

        if not crypto_key:
            crypto_key = [Fernet.generate_key()]
        if not isinstance(crypto_key, list):
            crypto_key = [crypto_key]
        self.update(crypto_key=crypto_key)
        self.crypto_key = crypto_key

        if bear_hash_key is None:
            bear_hash_key = []
        if not isinstance(bear_hash_key, list):
            bear_hash_key = [bear_hash_key]
        self.bear_hash_key = bear_hash_key

        self.max_data = max_data
        self.clients = {}

        # Setup hosts/ports/urls
        default_hostname = socket.gethostname()
        self.hostname = hostname or default_hostname
        if resolve_hostname:
            self.hostname = resolve_ip(self.hostname)

        # Metrics setup
        if datadog_api_key:
            self.metrics = DatadogMetrics(
                hostname=self.hostname,
                api_key=datadog_api_key,
                app_key=datadog_app_key,
                flush_interval=datadog_flush_interval,
            )
        elif statsd_host:
            self.metrics = TwistedMetrics(statsd_host, statsd_port)
        else:
            self.metrics = SinkMetrics()

        self.port = port
        self.endpoint_hostname = endpoint_hostname or self.hostname
        self.router_hostname = router_hostname or self.hostname

        if router_conf is None:
            router_conf = {}
        self.router_conf = router_conf
        self.router_url = canonical_url(router_scheme or 'http',
                                        self.router_hostname, router_port)

        self.endpoint_url = canonical_url(endpoint_scheme or 'http',
                                          self.endpoint_hostname,
                                          endpoint_port)
        self.enable_tls_auth = client_certs is not None
        self.client_certs = client_certs

        # Database objects
        self.router_table = get_router_table(router_tablename,
                                             router_read_throughput,
                                             router_write_throughput)
        self.storage_table = get_storage_table(storage_tablename,
                                               storage_read_throughput,
                                               storage_write_throughput)
        self.message_table = get_rotating_message_table(
            message_tablename,
            message_read_throughput=message_read_throughput,
            message_write_throughput=message_write_throughput)
        self._message_prefix = message_tablename
        self.message_limit = msg_limit
        self.storage = Storage(self.storage_table, self.metrics)
        self.router = Router(self.router_table, self.metrics)

        # Used to determine whether a connection is out of date with current
        # db objects. There are three noteworty cases:
        # 1 "Last Month" the table requires a rollover.
        # 2 "This Month" the most common case.
        # 3 "Next Month" where the system will soon be rolling over, but with
        #   timing, some nodes may roll over sooner. Ensuring the next month's
        #   table is present before the switchover is the main reason for this,
        #   just in case some nodes do switch sooner.
        self.create_initial_message_tables()

        # Run preflight check
        preflight_check(self.storage, self.router, preflight_uaid)

        # CORS
        self.cors = enable_cors

        # Force timeout in idle seconds
        self.wake_timeout = wake_timeout

        # Setup the routers
        self.routers = dict()
        self.routers["simplepush"] = SimpleRouter(
            self, router_conf.get("simplepush"))
        self.routers["webpush"] = WebPushRouter(self, None)
        if 'apns' in router_conf:
            self.routers["apns"] = APNSRouter(self, router_conf["apns"])
        if 'gcm' in router_conf:
            self.routers["gcm"] = GCMRouter(self, router_conf["gcm"])

        # Env
        self.env = env

        self.hello_timeout = hello_timeout

        self.ami_id = ami_id

        # Generate messages per legacy rules, only used for testing to
        # generate legacy data.
        self._notification_legacy = False
Exemple #42
0
    def __init__(self,
                 crypto_key=None,
                 datadog_api_key=None,
                 datadog_app_key=None,
                 datadog_flush_interval=None,
                 hostname=None,
                 port=None,
                 router_scheme=None,
                 router_hostname=None,
                 router_port=None,
                 endpoint_scheme=None,
                 endpoint_hostname=None,
                 endpoint_port=None,
                 router_conf={},
                 router_tablename="router",
                 router_read_throughput=5,
                 router_write_throughput=5,
                 storage_tablename="storage",
                 storage_read_throughput=5,
                 storage_write_throughput=5,
                 message_tablename="message",
                 message_read_throughput=5,
                 message_write_throughput=5,
                 statsd_host="localhost",
                 statsd_port=8125,
                 resolve_hostname=False,
                 max_data=4096,
                 # Reflected up from UDP Router
                 wake_timeout=0,
                 env='development',
                 enable_cors=False,
                 s3_bucket=DEFAULT_BUCKET,
                 senderid_expry=SENDERID_EXPRY,
                 senderid_list={},
                 hello_timeout=0,
                 bear_hash_key=None,
                 preflight_uaid="deadbeef00000000deadbeef000000000",
                 ):
        """Initialize the Settings object

        Upon creation, the HTTP agent will initialize, all configured routers
        will be setup and started, logging will be started, and the database
        will have a preflight check done.

        """
        # Use a persistent connection pool for HTTP requests.
        pool = HTTPConnectionPool(reactor)
        self.agent = Agent(reactor, connectTimeout=5, pool=pool)

        # Metrics setup
        if datadog_api_key:
            self.metrics = DatadogMetrics(
                api_key=datadog_api_key,
                app_key=datadog_app_key,
                flush_interval=datadog_flush_interval
            )
        elif statsd_host:
            self.metrics = TwistedMetrics(statsd_host, statsd_port)
        else:
            self.metrics = SinkMetrics()
        if not crypto_key:
            crypto_key = [Fernet.generate_key()]
        if not isinstance(crypto_key, list):
            crypto_key = [crypto_key]
        self.update(crypto_key=crypto_key)
        self.crypto_key = crypto_key

        if bear_hash_key is None:
            bear_hash_key = []
        if not isinstance(bear_hash_key, list):
            bear_hash_key = [bear_hash_key]
        self.bear_hash_key = bear_hash_key

        self.max_data = max_data
        self.clients = {}

        # Setup hosts/ports/urls
        default_hostname = socket.gethostname()
        self.hostname = hostname or default_hostname
        if resolve_hostname:
            self.hostname = resolve_ip(self.hostname)

        self.port = port
        self.endpoint_hostname = endpoint_hostname or self.hostname
        self.router_hostname = router_hostname or self.hostname

        self.router_conf = router_conf
        self.router_url = canonical_url(
            router_scheme or 'http',
            self.router_hostname,
            router_port
        )

        self.endpoint_url = canonical_url(
            endpoint_scheme or 'http',
            self.endpoint_hostname,
            endpoint_port
        )

        # Database objects
        self.router_table = get_router_table(router_tablename,
                                             router_read_throughput,
                                             router_write_throughput)
        self.storage_table = get_storage_table(
            storage_tablename,
            storage_read_throughput,
            storage_write_throughput)
        self.message_table = get_rotating_message_table(
            message_tablename)
        self._message_prefix = message_tablename
        self.storage = Storage(self.storage_table, self.metrics)
        self.router = Router(self.router_table, self.metrics)

        # Used to determine whether a connection is out of date with current
        # db objects. There are three noteworty cases:
        # 1 "Last Month" the table requires a rollover.
        # 2 "This Month" the most common case.
        # 3 "Next Month" where the system will soon be rolling over, but with
        #   timing, some nodes may roll over sooner. Ensuring the next month's
        #   table is present before the switchover is the main reason for this,
        #   just in case some nodes do switch sooner.
        self.create_initial_message_tables()

        # Run preflight check
        preflight_check(self.storage, self.router, preflight_uaid)

        # CORS
        self.cors = enable_cors

        # Force timeout in idle seconds
        self.wake_timeout = wake_timeout

        # Setup the routers
        self.routers = {}
        self.routers["simplepush"] = SimpleRouter(
            self,
            router_conf.get("simplepush")
        )
        self.routers["webpush"] = WebPushRouter(self, None)
        if 'apns' in router_conf:
            self.routers["apns"] = APNSRouter(self, router_conf["apns"])
        if 'gcm' in router_conf:
            self.routers["gcm"] = GCMRouter(self, router_conf["gcm"])

        # Env
        self.env = env

        self.hello_timeout = hello_timeout
Exemple #43
0
 def setUp(self):
     table = get_rotating_message_table()
     self.real_table = table
     self.real_connection = table.connection
     self.uaid = str(uuid.uuid4())
Exemple #44
0
 def setUp(self):
     table = get_rotating_message_table()
     self.real_table = table
     self.real_connection = table.connection
     self.uaid = str(uuid.uuid4())
Exemple #45
0
 def test_all_channels_no_uaid(self):
     m = get_rotating_message_table()
     message = Message(m, SinkMetrics())
     exists, chans = message.all_channels(dummy_uaid)
     assert (chans == set([]))