Exemple #1
0
    def get_or_create(self, log_name, bucket_size):
        """
        Gets or creates a log.

        :rtype: Log
        """
        try:
            return self[log_name]
        except RepositoryKeyError:
            return start_new_log(log_name, bucket_size=bucket_size)
Exemple #2
0
    def test_buckets(self):
        # Start new log.
        log = start_new_log(name='log1', bucket_size='second')

        # Write messages across the time interval
        start = datetime.datetime.now()
        logger = get_logger(log)
        number_of_messages = 300
        events = []
        for i in range(number_of_messages):
            message_logged = logger.info(str(i))
            events.append(message_logged)
            sleep(0.01)
        self.assertGreater(datetime.datetime.now() - start,
                           datetime.timedelta(seconds=1))

        # Get the messages in descending order.
        reader = get_log_reader(log, self.event_store)
        messages = list(reader.get_messages(is_ascending=False, page_size=10))
        self.assertEqual(len(messages), number_of_messages)

        # Expect the order of the messages is the reverse of the created order.
        self.assertEqual(
            messages,
            list(reversed([str(i) for i in range(number_of_messages)])))

        # Get the messages in ascending order.
        messages = list(reader.get_messages(is_ascending=True, page_size=10))
        self.assertEqual(len(messages), number_of_messages)

        # Expect the order of the messages is the same as the created order.
        self.assertEqual(messages, [str(i) for i in range(number_of_messages)])

        # Get a limited number of messages in descending order.
        limit = 150
        messages = list(
            reader.get_messages(is_ascending=False, page_size=10, limit=limit))
        self.assertLess(limit, number_of_messages)
        self.assertEqual(len(messages), limit)

        # Expect the order of the messages is the reverse of the created order.
        self.assertEqual(
            messages,
            list(reversed([str(i)
                           for i in range(number_of_messages)]))[:limit])

        # Get a limited number of messages in ascending order.
        limit = 150
        messages = list(
            reader.get_messages(is_ascending=True, page_size=10, limit=limit))
        self.assertLess(limit, number_of_messages)
        self.assertEqual(len(messages), limit)

        # Expect the order of the messages is the same as the created order.
        self.assertEqual(messages, [str(i) for i in range(limit)])

        # Get a limited number of messages in descending order from the midpoint down.
        limit = 110
        midpoint = events[150].domain_event_id
        messages = list(
            reader.get_messages(is_ascending=False,
                                page_size=10,
                                limit=limit,
                                until=midpoint))
        self.assertLess(limit, number_of_messages)
        self.assertEqual(len(messages), limit)

        # Expect the order of the messages is the reverse of the created order.
        self.assertEqual(
            messages,
            list(reversed([str(i) for i in range(150 - limit, 150)])))

        # Get a limited number of messages in ascending order from the midpoint up.
        limit = 110
        midpoint = events[149].domain_event_id
        messages = list(
            reader.get_messages(is_ascending=True,
                                page_size=10,
                                limit=limit,
                                after=midpoint))
        self.assertLess(limit, number_of_messages)
        self.assertEqual(len(messages), limit)

        # Expect the order of the messages is the same as the created order.
        self.assertEqual(messages, [str(i) for i in range(150, 150 + limit)])

        # Get a limited number of messages in descending order above the midpoint down.
        limit = 200
        midpoint = events[150].domain_event_id
        messages = list(
            reader.get_messages(is_ascending=False,
                                page_size=10,
                                limit=limit,
                                after=midpoint))
        self.assertLess(limit, number_of_messages)
        self.assertEqual(len(messages), 150)

        # Expect the order of the messages is the reverse of the created order.
        self.assertEqual(messages,
                         list(reversed([str(i) for i in range(150, 300)])))

        # Get a limited number of messages in ascending order below the midpoint up.
        limit = 200
        midpoint = events[149].domain_event_id
        messages = list(
            reader.get_messages(is_ascending=True,
                                page_size=10,
                                limit=limit,
                                until=midpoint))
        self.assertLess(limit, number_of_messages)
        self.assertEqual(len(messages), 150)

        # Expect the order of the messages is the same as the created order.
        self.assertEqual(messages, [str(i) for i in range(150)])

        #
        # Use the last position to start part way through.
        limit = 20
        last_position = reader.position
        messages = reader.get_messages(is_ascending=True,
                                       page_size=10,
                                       limit=limit,
                                       after=last_position)
        messages = list(messages)
        self.assertEqual(len(messages), limit)

        # Expect the order of the messages is the same as the created order.
        self.assertEqual(messages, [str(i) for i in range(150, 150 + limit)])

        # Do it again.
        last_position = reader.position
        messages = reader.get_messages(is_ascending=True,
                                       page_size=10,
                                       limit=limit,
                                       after=last_position)
        messages = list(messages)
        self.assertEqual(len(messages), limit)

        # Expect the order of the messages is the same as the created order.
        self.assertEqual(messages,
                         [str(i) for i in range(150 + limit, 150 + limit * 2)])

        # Go back.
        last_position = reader.position
        messages = reader.get_messages(is_ascending=False,
                                       page_size=10,
                                       limit=limit,
                                       until=last_position)
        messages = list(messages)
        self.assertEqual(len(messages), limit)

        # Expect the order of the messages is the reverse of the created order.
        self.assertEqual(
            messages,
            [str(i) for i in range(148 + limit * 2, 148 + limit, -1)])

        # Go back.
        last_position = reader.position
        messages = reader.get_messages(is_ascending=False,
                                       page_size=10,
                                       limit=limit,
                                       until=last_position)
        messages = list(messages)
        self.assertEqual(len(messages), limit)

        # Expect the order of the messages is the reverse of the created order.
        self.assertEqual(
            messages,
            [str(i) for i in range(128 + limit * 2, 128 + limit, -1)])

        # Go back.
        last_position = reader.position
        messages = reader.get_messages(is_ascending=False,
                                       page_size=10,
                                       limit=limit,
                                       until=last_position)
        messages = list(messages)
        self.assertEqual(len(messages), limit)

        # Expect the order of the messages is the reverse of the created order.
        self.assertEqual(
            messages,
            [str(i) for i in range(108 + limit * 2, 108 + limit, -1)])

        # Repeat.
        messages = reader.get_messages(is_ascending=False,
                                       page_size=10,
                                       limit=limit,
                                       until=last_position)
        messages = list(messages)
        self.assertEqual(len(messages), limit)

        # Expect the order of the messages is the reverse of the created order.
        self.assertEqual(
            messages,
            [str(i) for i in range(108 + limit * 2, 108 + limit, -1)])
    def test_log_performance(self):
        log = start_new_log('example', bucket_size='year')
        logger = get_logger(log)
        log_reader = get_log_reader(log, self.app.event_store)

        # Write a load of messages.
        start_write = utc_now()
        number_of_messages = 111
        events = []
        for i in range(number_of_messages):
            event = logger.info('Logger message number {}'.format(i))
            events.append(event)
        time_to_write = (utc_now() - start_write)
        print("Time to log {} messages: {:.2f}s ({:.0f} messages/s, {:.6f}s each)"
              "".format(number_of_messages, time_to_write, number_of_messages/ time_to_write,
                        time_to_write / number_of_messages))

        # Read pages of messages in descending order.
        # - get a limited number until a time, then use the earliest in that list as the position
        position = events[-1].domain_event_id

        page_size = 10

        # Page back through the log in reverse chronological order.
        previous_position = None
        next_position = None
        count_pages = 0
        total_time_to_read = 0
        total_num_reads = 0
        while True:
            start_read = utc_now()
            page_of_events, next_position = self.get_message_logged_events_and_next_position(log_reader, position, page_size)
            time_to_read = (utc_now() - start_read)
            total_time_to_read += time_to_read
            total_num_reads += 1
            count_pages += 1
            if next_position is None:
                break
            else:
                previous_position, position = position, next_position

        # Check we got to the end of the line.
        self.assertEqual(count_pages, 11)
        self.assertIsNone(next_position)
        self.assertTrue(previous_position)

        # Page forward through the log in chronological order.
        count_pages = 0
        position = None
        while True:
            start_read = utc_now()
            page_of_events, next_position = self.get_message_logged_events_and_next_position(log_reader, position, page_size, is_ascending=True)
            time_to_read = (utc_now() - start_read)
            total_time_to_read += time_to_read
            total_num_reads += 1
            count_pages += 1
            if next_position is None:
                break
            else:
                position = next_position

        self.assertEqual(count_pages, 11)
        self.assertIsNone(next_position)
        self.assertTrue(previous_position)

        reads_per_second = total_num_reads / total_time_to_read
        messages_per_second = reads_per_second * number_of_messages
        print("Time to read {} pages of logged messages: {:.6f}s ({:.0f} pages/s, {:.0f} messages/s))"
              "".format(total_num_reads, total_time_to_read, reads_per_second, messages_per_second))
    def test_buckets(self):
        # Start new log.
        log = start_new_log(name='log1', bucket_size='second')

        # Write messages across the time interval
        start = datetime.datetime.now()
        logger = get_logger(log)
        number_of_messages = 300
        events = []
        for i in range(number_of_messages):
            message_logged = logger.info(str(i))
            events.append(message_logged)
            sleep(0.01)
        self.assertGreater(datetime.datetime.now() - start, datetime.timedelta(seconds=1))

        # Get the messages in descending order.
        reader = get_log_reader(log, self.event_store)
        messages = list(reader.get_messages(is_ascending=False, page_size=10))
        self.assertEqual(len(messages), number_of_messages)

        # Expect the order of the messages is the reverse of the created order.
        self.assertEqual(messages, list(reversed([str(i) for i in range(number_of_messages)])))

        # Get the messages in ascending order.
        messages = list(reader.get_messages(is_ascending=True, page_size=10))
        self.assertEqual(len(messages), number_of_messages)

        # Expect the order of the messages is the same as the created order.
        self.assertEqual(messages, [str(i) for i in range(number_of_messages)])

        # Get a limited number of messages in descending order.
        limit = 150
        messages = list(reader.get_messages(is_ascending=False, page_size=10, limit=limit))
        self.assertLess(limit, number_of_messages)
        self.assertEqual(len(messages), limit)

        # Expect the order of the messages is the reverse of the created order.
        self.assertEqual(messages, list(reversed([str(i) for i in range(number_of_messages)]))[:limit])

        # Get a limited number of messages in ascending order.
        limit = 150
        messages = list(reader.get_messages(is_ascending=True, page_size=10, limit=limit))
        self.assertLess(limit, number_of_messages)
        self.assertEqual(len(messages), limit)

        # Expect the order of the messages is the same as the created order.
        self.assertEqual(messages, [str(i) for i in range(limit)])

        # Get a limited number of messages in descending order from the midpoint down.
        limit = 110
        midpoint = events[150].domain_event_id
        messages = list(reader.get_messages(is_ascending=False, page_size=10, limit=limit, until=midpoint))
        self.assertLess(limit, number_of_messages)
        self.assertEqual(len(messages), limit)

        # Expect the order of the messages is the reverse of the created order.
        self.assertEqual(messages, list(reversed([str(i) for i in range(150 - limit, 150)])))

        # Get a limited number of messages in ascending order from the midpoint up.
        limit = 110
        midpoint = events[149].domain_event_id
        messages = list(reader.get_messages(is_ascending=True, page_size=10, limit=limit, after=midpoint))
        self.assertLess(limit, number_of_messages)
        self.assertEqual(len(messages), limit)

        # Expect the order of the messages is the same as the created order.
        self.assertEqual(messages, [str(i) for i in range(150, 150 + limit)])

        # Get a limited number of messages in descending order above the midpoint down.
        limit = 200
        midpoint = events[150].domain_event_id
        messages = list(reader.get_messages(is_ascending=False, page_size=10, limit=limit, after=midpoint))
        self.assertLess(limit, number_of_messages)
        self.assertEqual(len(messages), 150)

        # Expect the order of the messages is the reverse of the created order.
        self.assertEqual(messages, list(reversed([str(i) for i in range(150, 300)])))

        # Get a limited number of messages in ascending order below the midpoint up.
        limit = 200
        midpoint = events[149].domain_event_id
        messages = list(reader.get_messages(is_ascending=True, page_size=10, limit=limit, until=midpoint))
        self.assertLess(limit, number_of_messages)
        self.assertEqual(len(messages), 150)

        # Expect the order of the messages is the same as the created order.
        self.assertEqual(messages, [str(i) for i in range(150)])

        #
        # Use the last position to start part way through.
        limit = 20
        last_position = reader.position
        messages = reader.get_messages(is_ascending=True, page_size=10, limit=limit, after=last_position)
        messages = list(messages)
        self.assertEqual(len(messages), limit)

        # Expect the order of the messages is the same as the created order.
        self.assertEqual(messages, [str(i) for i in range(150, 150 + limit)])

        # Do it again.
        last_position = reader.position
        messages = reader.get_messages(is_ascending=True, page_size=10, limit=limit, after=last_position)
        messages = list(messages)
        self.assertEqual(len(messages), limit)

        # Expect the order of the messages is the same as the created order.
        self.assertEqual(messages, [str(i) for i in range(150 + limit, 150 + limit * 2)])

        # Go back.
        last_position = reader.position
        messages = reader.get_messages(is_ascending=False, page_size=10, limit=limit, until=last_position)
        messages = list(messages)
        self.assertEqual(len(messages), limit)

        # Expect the order of the messages is the reverse of the created order.
        self.assertEqual(messages, [str(i) for i in range(148 + limit * 2, 148 + limit, -1)])

        # Go back.
        last_position = reader.position
        messages = reader.get_messages(is_ascending=False, page_size=10, limit=limit, until=last_position)
        messages = list(messages)
        self.assertEqual(len(messages), limit)

        # Expect the order of the messages is the reverse of the created order.
        self.assertEqual(messages, [str(i) for i in range(128 + limit * 2, 128 + limit, -1)])

        # Go back.
        last_position = reader.position
        messages = reader.get_messages(is_ascending=False, page_size=10, limit=limit, until=last_position)
        messages = list(messages)
        self.assertEqual(len(messages), limit)

        # Expect the order of the messages is the reverse of the created order.
        self.assertEqual(messages, [str(i) for i in range(108 + limit * 2, 108 + limit, -1)])

        # Repeat.
        messages = reader.get_messages(is_ascending=False, page_size=10, limit=limit, until=last_position)
        messages = list(messages)
        self.assertEqual(len(messages), limit)

        # Expect the order of the messages is the reverse of the created order.
        self.assertEqual(messages, [str(i) for i in range(108 + limit * 2, 108 + limit, -1)])