def read_config(conf):

    files = []

    for f in getConfChildren(conf, "File"):

        instance_name = getConfFirstValue(f, 'Instance')
        filepath = f.values[0]
        groupby = getConfFirstValue(f, 'GroupBy')
        groupbygroup = getConfFirstValue(f, 'GroupName', None)
        maxgroups = int(getConfFirstValue(f, 'MaxGroups', 64))

        gt = GroupingTail(filepath, groupby, groupbygroup)

        files.append(dict(
                instance_name=instance_name,
                grouping_tail=gt
            ))

        for m in getConfChildren(f, 'Match'):

            minstance_name = getConfFirstValue(m, "Instance")
            valuetype = getConfFirstValue(m, "Type")

            # dstype determins the instrument used
            dstype = getConfFirstValue(m, "DSType")
            instrument = INSTRUMENTS[dstype](m)

            gt.add_match (minstance_name, valuetype, instrument)
    return files
Exemplo n.º 2
0
    def test_basic_counter_sum():
        """
            Test basic counter sum function for grouping tail class.
        """

        counter_sum = CounterSum('^(\d)')
        grouping_tail = GroupingTail(SIMPLE_LOG_FILE, '^(\d)')
        #print grouping_tail.offsetpath
        grouping_tail.add_match('simple', 'counter', counter_sum)
        grouping_tail.update()

        assert_equal(len(counter_sum.data), 4)
        assert_true('3' in counter_sum.data)
        assert_equal(counter_sum.data['3'], 9.0)
        assert_true('2' in counter_sum.data)
        assert_equal(counter_sum.data['2'], 4.0)
        assert_true('1' in counter_sum.data)
        assert_equal(counter_sum.data['1'], 1.0)
        assert_true('8' in counter_sum.data)
        assert_equal(counter_sum.data['8'], 8.0)

        read_metrics = grouping_tail.read_metrics()

        for key, key_value in counter_sum.data.items():
            metric_name, value_type, value = read_metrics.next()
            assert_equal(metric_name, str(key) + '.simple')
            assert_equal(value_type, 'counter')
            assert_equal(value, key_value)
    def test_basic_counter_sum():
        """
            Test basic counter sum function for grouping tail class.
        """

        counter_sum = CounterSum('^(\d)')
        grouping_tail = GroupingTail(SIMPLE_LOG_FILE, '^(\d)')
        #print grouping_tail.offsetpath
        grouping_tail.add_match('simple', 'counter', counter_sum)
        grouping_tail.update()

        assert_equal(len(counter_sum.data), 4)
        assert_true('3' in counter_sum.data)
        assert_equal(counter_sum.data['3'], 9.0)
        assert_true('2' in counter_sum.data)
        assert_equal(counter_sum.data['2'], 4.0)
        assert_true('1' in counter_sum.data)
        assert_equal(counter_sum.data['1'], 1.0)
        assert_true('8' in counter_sum.data)
        assert_equal(counter_sum.data['8'], 8.0)

        read_metrics = grouping_tail.read_metrics()

        for key, key_value in counter_sum.data.items():
            metric_name, value_type, value = read_metrics.next()
            assert_equal(metric_name, str(key) + '.simple')
            assert_equal(value_type, 'counter')
            assert_equal(value, key_value)
Exemplo n.º 4
0
    def test_syslog_connect():
        import logging
        import logging.handlers

        my_logger = logging.getLogger('MyLogger')
        my_logger.setLevel(logging.DEBUG)
        handler = logging.handlers.SysLogHandler(address=('localhost', 9514),
                                                 facility=19)
        my_logger.addHandler(handler)

        grouping_tail = GroupingTail('syslog://localhost:9514', "<159>-(.*)-")
        counter_inc = CounterInc('.')
        grouping_tail.add_match('requests', 'counter', counter_inc)
        grouping_tail.update()

        my_logger.debug('-domain- 1')
        # because its threads and sockets it needs a chance to catch up
        time.sleep(0.1)
        grouping_tail.update()
        assert_equal(counter_inc.data.get('domain'), 1)
        grouping_tail.server.shutdown()
        my_logger.removeHandler(handler)
Exemplo n.º 5
0
    def test_namedgroups():

        my_logger = logging.getLogger('MyLogger')
        my_logger.setLevel(logging.DEBUG)
        handler = logging.handlers.SysLogHandler(address=('localhost', 9514),
                                                 facility=19)
        my_logger.addHandler(handler)
        regex = "<159>-(?P<domain>.*)- (?P<num>.*)"
        grouping_tail = GroupingTail('syslog://localhost:9514',
                                     regex,
                                     groupname="domain")
        counter = CounterSum(regex, groupname="num", value_cast=int_cast)
        grouping_tail.add_match('requests', 'counter', counter)
        grouping_tail.update()

        my_logger.debug('-domain- 3')
        my_logger.debug('-domain- 2')
        # because its threads and sockets it needs a chance to catch up
        time.sleep(0.1)
        grouping_tail.update()
        assert_equal(counter.data.get('domain'), 5)
        grouping_tail.server.shutdown()
        my_logger.removeHandler(handler)
    def test_namedgroups():

        my_logger = logging.getLogger('MyLogger')
        my_logger.setLevel(logging.DEBUG)
        handler = logging.handlers.SysLogHandler(address=('localhost', 9514), facility=19)
        my_logger.addHandler(handler)
        regex = "<159>-(?P<domain>.*)- (?P<num>.*)"
        grouping_tail = GroupingTail('syslog://localhost:9514', regex, groupname="domain")
        counter = CounterSum(regex, groupname="num", value_cast=int_cast)
        grouping_tail.add_match('requests', 'counter', counter)
        grouping_tail.update()

        my_logger.debug('-domain- 3')
        my_logger.debug('-domain- 2')
        # because its threads and sockets it needs a chance to catch up
        time.sleep(0.1)
        grouping_tail.update()
        assert_equal(counter.data.get('domain'), 5)
        grouping_tail.server.shutdown()
        my_logger.removeHandler(handler)
    def test_syslog_connect():
        import logging
        import logging.handlers

        my_logger = logging.getLogger('MyLogger')
        my_logger.setLevel(logging.DEBUG)
        handler = logging.handlers.SysLogHandler(address=('localhost', 9514), facility=19)
        my_logger.addHandler(handler)

        grouping_tail = GroupingTail('syslog://localhost:9514', "<159>-(.*)-")
        counter_inc = CounterInc('.')
        grouping_tail.add_match('requests', 'counter', counter_inc)
        grouping_tail.update()

        my_logger.debug('-domain- 1')
        # because its threads and sockets it needs a chance to catch up
        time.sleep(0.1)
        grouping_tail.update()
        assert_equal(counter_inc.data.get('domain'), 1)
        grouping_tail.server.shutdown()
        my_logger.removeHandler(handler)
    def test_basic_multi_files():
        """
            Test basic tail multi files function for grouping tail class.
        """
        from pretaweb.collectd.groupingtail.groupingtail import GroupingTail
        from pretaweb.collectd.groupingtail.instruments import CounterInc

        counter_inc = CounterInc('.')
        grouping_tail = GroupingTail(SIMPLE_LOG_FILE, '^(\d)')

        second_counter_inc = CounterInc('.')
        second_grouping_tail = GroupingTail(second_simple_log_file, '^(\d)')

        grouping_tail.add_match('simple', 'counter', counter_inc)
        grouping_tail.update()

        third_counter_inc = CounterInc('.')
        third_grouping_tail = GroupingTail(SIMPLE_LOG_FILE, '^(\D)')

        second_grouping_tail.add_match('second_simple', 'counter', second_counter_inc)
        second_grouping_tail.update()

        third_grouping_tail.add_match('third_simple', 'counter', third_counter_inc)
        third_grouping_tail.update()

        assert_equal(len(counter_inc.data), 4)
        assert_true('3' in counter_inc.data)
        assert_equal(counter_inc.data['3'], 3)
        assert_true('2' in counter_inc.data)
        assert_equal(counter_inc.data['2'], 2)
        assert_true('1' in counter_inc.data)
        assert_equal(counter_inc.data['1'], 1)
        assert_true('8' in counter_inc.data)
        assert_equal(counter_inc.data['8'], 1)

        assert_equal(len(second_counter_inc.data), 2)
        assert_true('3' in second_counter_inc.data)
        assert_equal(second_counter_inc.data['3'], 4)
        assert_true('2' in second_counter_inc.data)
        assert_equal(second_counter_inc.data['2'], 1)
        assert_false('1' in second_counter_inc.data)

        assert_equal(len(third_counter_inc.data), 2)
        assert_true('w' in third_counter_inc.data)
        assert_equal(third_counter_inc.data['w'], 3)
        assert_true('y' in third_counter_inc.data)
        assert_equal(third_counter_inc.data['y'], 1)
        assert_false('z' in third_counter_inc.data)

        read_metrics = grouping_tail.read_metrics()

        for key, key_value in counter_inc.data.items():
            metric_name, value_type, value = read_metrics.next()
            assert_equal(metric_name, str(key) + '.simple')
            assert_equal(value_type, 'counter')
            assert_equal(value, key_value)

        second_read_metrics = second_grouping_tail.read_metrics()

        for key, key_value in second_counter_inc.data.items():
            metric_name, value_type, value = second_read_metrics.next()
            assert_equal(metric_name, str(key) + '.second_simple')
            assert_equal(value_type, 'counter')
            assert_equal(value, key_value)

        third_read_metrics = third_grouping_tail.read_metrics()

        for key, key_value in third_counter_inc.data.items():
            metric_name, value_type, value = third_read_metrics.next()
            assert_equal(metric_name, str(key) + '.third_simple')
            assert_equal(value_type, 'counter')
            assert_equal(value, key_value)
Exemplo n.º 9
0
    def test_basic_multi_files():
        """
            Test basic tail multi files function for grouping tail class.
        """
        from pretaweb.collectd.groupingtail.groupingtail import GroupingTail
        from pretaweb.collectd.groupingtail.instruments import CounterInc

        counter_inc = CounterInc('.')
        grouping_tail = GroupingTail(SIMPLE_LOG_FILE, '^(\d)')

        second_counter_inc = CounterInc('.')
        second_grouping_tail = GroupingTail(second_simple_log_file, '^(\d)')

        grouping_tail.add_match('simple', 'counter', counter_inc)
        grouping_tail.update()

        third_counter_inc = CounterInc('.')
        third_grouping_tail = GroupingTail(SIMPLE_LOG_FILE, '^(\D)')

        second_grouping_tail.add_match('second_simple', 'counter',
                                       second_counter_inc)
        second_grouping_tail.update()

        third_grouping_tail.add_match('third_simple', 'counter',
                                      third_counter_inc)
        third_grouping_tail.update()

        assert_equal(len(counter_inc.data), 4)
        assert_true('3' in counter_inc.data)
        assert_equal(counter_inc.data['3'], 3)
        assert_true('2' in counter_inc.data)
        assert_equal(counter_inc.data['2'], 2)
        assert_true('1' in counter_inc.data)
        assert_equal(counter_inc.data['1'], 1)
        assert_true('8' in counter_inc.data)
        assert_equal(counter_inc.data['8'], 1)

        assert_equal(len(second_counter_inc.data), 2)
        assert_true('3' in second_counter_inc.data)
        assert_equal(second_counter_inc.data['3'], 4)
        assert_true('2' in second_counter_inc.data)
        assert_equal(second_counter_inc.data['2'], 1)
        assert_false('1' in second_counter_inc.data)

        assert_equal(len(third_counter_inc.data), 2)
        assert_true('w' in third_counter_inc.data)
        assert_equal(third_counter_inc.data['w'], 3)
        assert_true('y' in third_counter_inc.data)
        assert_equal(third_counter_inc.data['y'], 1)
        assert_false('z' in third_counter_inc.data)

        read_metrics = grouping_tail.read_metrics()

        for key, key_value in counter_inc.data.items():
            metric_name, value_type, value = read_metrics.next()
            assert_equal(metric_name, str(key) + '.simple')
            assert_equal(value_type, 'counter')
            assert_equal(value, key_value)

        second_read_metrics = second_grouping_tail.read_metrics()

        for key, key_value in second_counter_inc.data.items():
            metric_name, value_type, value = second_read_metrics.next()
            assert_equal(metric_name, str(key) + '.second_simple')
            assert_equal(value_type, 'counter')
            assert_equal(value, key_value)

        third_read_metrics = third_grouping_tail.read_metrics()

        for key, key_value in third_counter_inc.data.items():
            metric_name, value_type, value = third_read_metrics.next()
            assert_equal(metric_name, str(key) + '.third_simple')
            assert_equal(value_type, 'counter')
            assert_equal(value, key_value)
Exemplo n.º 10
0
def new_grouping_tail(log_file, group_by):
    new_log = tempfile.NamedTemporaryFile(delete=False)
    grouping_tail = GroupingTail(new_log.name, group_by)
    copy_lines(log_file, new_log)
    return grouping_tail