def read_config(conf):
files = []
for f in getConfChildren(conf, "File"):
instance_name = getConfFirstValue(f, 'Instance')
filepath = f.values[0]
groupby = getConfFirstValue(f, 'GroupBy')
groupbygroup = getConfFirstValue(f, 'GroupName', None)
maxgroups = int(getConfFirstValue(f, 'MaxGroups', 64))
gt = GroupingTail(filepath, groupby, groupbygroup)
files.append(dict(
instance_name=instance_name,
grouping_tail=gt
))
for m in getConfChildren(f, 'Match'):
minstance_name = getConfFirstValue(m, "Instance")
valuetype = getConfFirstValue(m, "Type")
# dstype determins the instrument used
dstype = getConfFirstValue(m, "DSType")
instrument = INSTRUMENTS[dstype](m)
gt.add_match (minstance_name, valuetype, instrument)
return files
def test_basic_counter_sum():
"""
Test basic counter sum function for grouping tail class.
"""
counter_sum = CounterSum('^(\d)')
grouping_tail = GroupingTail(SIMPLE_LOG_FILE, '^(\d)')
#print grouping_tail.offsetpath
grouping_tail.add_match('simple', 'counter', counter_sum)
grouping_tail.update()
assert_equal(len(counter_sum.data), 4)
assert_true('3' in counter_sum.data)
assert_equal(counter_sum.data['3'], 9.0)
assert_true('2' in counter_sum.data)
assert_equal(counter_sum.data['2'], 4.0)
assert_true('1' in counter_sum.data)
assert_equal(counter_sum.data['1'], 1.0)
assert_true('8' in counter_sum.data)
assert_equal(counter_sum.data['8'], 8.0)
read_metrics = grouping_tail.read_metrics()
for key, key_value in counter_sum.data.items():
metric_name, value_type, value = read_metrics.next()
assert_equal(metric_name, str(key) + '.simple')
assert_equal(value_type, 'counter')
assert_equal(value, key_value)
def test_basic_counter_sum():
"""
Test basic counter sum function for grouping tail class.
"""
counter_sum = CounterSum('^(\d)')
grouping_tail = GroupingTail(SIMPLE_LOG_FILE, '^(\d)')
#print grouping_tail.offsetpath
grouping_tail.add_match('simple', 'counter', counter_sum)
grouping_tail.update()
assert_equal(len(counter_sum.data), 4)
assert_true('3' in counter_sum.data)
assert_equal(counter_sum.data['3'], 9.0)
assert_true('2' in counter_sum.data)
assert_equal(counter_sum.data['2'], 4.0)
assert_true('1' in counter_sum.data)
assert_equal(counter_sum.data['1'], 1.0)
assert_true('8' in counter_sum.data)
assert_equal(counter_sum.data['8'], 8.0)
read_metrics = grouping_tail.read_metrics()
for key, key_value in counter_sum.data.items():
metric_name, value_type, value = read_metrics.next()
assert_equal(metric_name, str(key) + '.simple')
assert_equal(value_type, 'counter')
assert_equal(value, key_value)
def test_syslog_connect():
import logging
import logging.handlers
my_logger = logging.getLogger('MyLogger')
my_logger.setLevel(logging.DEBUG)
handler = logging.handlers.SysLogHandler(address=('localhost', 9514),
facility=19)
my_logger.addHandler(handler)
grouping_tail = GroupingTail('syslog://localhost:9514', "<159>-(.*)-")
counter_inc = CounterInc('.')
grouping_tail.add_match('requests', 'counter', counter_inc)
grouping_tail.update()
my_logger.debug('-domain- 1')
# because its threads and sockets it needs a chance to catch up
time.sleep(0.1)
grouping_tail.update()
assert_equal(counter_inc.data.get('domain'), 1)
grouping_tail.server.shutdown()
my_logger.removeHandler(handler)
def test_namedgroups():
my_logger = logging.getLogger('MyLogger')
my_logger.setLevel(logging.DEBUG)
handler = logging.handlers.SysLogHandler(address=('localhost', 9514),
facility=19)
my_logger.addHandler(handler)
regex = "<159>-(?P<domain>.*)- (?P<num>.*)"
grouping_tail = GroupingTail('syslog://localhost:9514',
regex,
groupname="domain")
counter = CounterSum(regex, groupname="num", value_cast=int_cast)
grouping_tail.add_match('requests', 'counter', counter)
grouping_tail.update()
my_logger.debug('-domain- 3')
my_logger.debug('-domain- 2')
# because its threads and sockets it needs a chance to catch up
time.sleep(0.1)
grouping_tail.update()
assert_equal(counter.data.get('domain'), 5)
grouping_tail.server.shutdown()
my_logger.removeHandler(handler)
def test_namedgroups():
my_logger = logging.getLogger('MyLogger')
my_logger.setLevel(logging.DEBUG)
handler = logging.handlers.SysLogHandler(address=('localhost', 9514), facility=19)
my_logger.addHandler(handler)
regex = "<159>-(?P<domain>.*)- (?P<num>.*)"
grouping_tail = GroupingTail('syslog://localhost:9514', regex, groupname="domain")
counter = CounterSum(regex, groupname="num", value_cast=int_cast)
grouping_tail.add_match('requests', 'counter', counter)
grouping_tail.update()
my_logger.debug('-domain- 3')
my_logger.debug('-domain- 2')
# because its threads and sockets it needs a chance to catch up
time.sleep(0.1)
grouping_tail.update()
assert_equal(counter.data.get('domain'), 5)
grouping_tail.server.shutdown()
my_logger.removeHandler(handler)
def test_syslog_connect():
import logging
import logging.handlers
my_logger = logging.getLogger('MyLogger')
my_logger.setLevel(logging.DEBUG)
handler = logging.handlers.SysLogHandler(address=('localhost', 9514), facility=19)
my_logger.addHandler(handler)
grouping_tail = GroupingTail('syslog://localhost:9514', "<159>-(.*)-")
counter_inc = CounterInc('.')
grouping_tail.add_match('requests', 'counter', counter_inc)
grouping_tail.update()
my_logger.debug('-domain- 1')
# because its threads and sockets it needs a chance to catch up
time.sleep(0.1)
grouping_tail.update()
assert_equal(counter_inc.data.get('domain'), 1)
grouping_tail.server.shutdown()
my_logger.removeHandler(handler)
def test_basic_multi_files():
"""
Test basic tail multi files function for grouping tail class.
"""
from pretaweb.collectd.groupingtail.groupingtail import GroupingTail
from pretaweb.collectd.groupingtail.instruments import CounterInc
counter_inc = CounterInc('.')
grouping_tail = GroupingTail(SIMPLE_LOG_FILE, '^(\d)')
second_counter_inc = CounterInc('.')
second_grouping_tail = GroupingTail(second_simple_log_file, '^(\d)')
grouping_tail.add_match('simple', 'counter', counter_inc)
grouping_tail.update()
third_counter_inc = CounterInc('.')
third_grouping_tail = GroupingTail(SIMPLE_LOG_FILE, '^(\D)')
second_grouping_tail.add_match('second_simple', 'counter', second_counter_inc)
second_grouping_tail.update()
third_grouping_tail.add_match('third_simple', 'counter', third_counter_inc)
third_grouping_tail.update()
assert_equal(len(counter_inc.data), 4)
assert_true('3' in counter_inc.data)
assert_equal(counter_inc.data['3'], 3)
assert_true('2' in counter_inc.data)
assert_equal(counter_inc.data['2'], 2)
assert_true('1' in counter_inc.data)
assert_equal(counter_inc.data['1'], 1)
assert_true('8' in counter_inc.data)
assert_equal(counter_inc.data['8'], 1)
assert_equal(len(second_counter_inc.data), 2)
assert_true('3' in second_counter_inc.data)
assert_equal(second_counter_inc.data['3'], 4)
assert_true('2' in second_counter_inc.data)
assert_equal(second_counter_inc.data['2'], 1)
assert_false('1' in second_counter_inc.data)
assert_equal(len(third_counter_inc.data), 2)
assert_true('w' in third_counter_inc.data)
assert_equal(third_counter_inc.data['w'], 3)
assert_true('y' in third_counter_inc.data)
assert_equal(third_counter_inc.data['y'], 1)
assert_false('z' in third_counter_inc.data)
read_metrics = grouping_tail.read_metrics()
for key, key_value in counter_inc.data.items():
metric_name, value_type, value = read_metrics.next()
assert_equal(metric_name, str(key) + '.simple')
assert_equal(value_type, 'counter')
assert_equal(value, key_value)
second_read_metrics = second_grouping_tail.read_metrics()
for key, key_value in second_counter_inc.data.items():
metric_name, value_type, value = second_read_metrics.next()
assert_equal(metric_name, str(key) + '.second_simple')
assert_equal(value_type, 'counter')
assert_equal(value, key_value)
third_read_metrics = third_grouping_tail.read_metrics()
for key, key_value in third_counter_inc.data.items():
metric_name, value_type, value = third_read_metrics.next()
assert_equal(metric_name, str(key) + '.third_simple')
assert_equal(value_type, 'counter')
assert_equal(value, key_value)
def test_basic_multi_files():
"""
Test basic tail multi files function for grouping tail class.
"""
from pretaweb.collectd.groupingtail.groupingtail import GroupingTail
from pretaweb.collectd.groupingtail.instruments import CounterInc
counter_inc = CounterInc('.')
grouping_tail = GroupingTail(SIMPLE_LOG_FILE, '^(\d)')
second_counter_inc = CounterInc('.')
second_grouping_tail = GroupingTail(second_simple_log_file, '^(\d)')
grouping_tail.add_match('simple', 'counter', counter_inc)
grouping_tail.update()
third_counter_inc = CounterInc('.')
third_grouping_tail = GroupingTail(SIMPLE_LOG_FILE, '^(\D)')
second_grouping_tail.add_match('second_simple', 'counter',
second_counter_inc)
second_grouping_tail.update()
third_grouping_tail.add_match('third_simple', 'counter',
third_counter_inc)
third_grouping_tail.update()
assert_equal(len(counter_inc.data), 4)
assert_true('3' in counter_inc.data)
assert_equal(counter_inc.data['3'], 3)
assert_true('2' in counter_inc.data)
assert_equal(counter_inc.data['2'], 2)
assert_true('1' in counter_inc.data)
assert_equal(counter_inc.data['1'], 1)
assert_true('8' in counter_inc.data)
assert_equal(counter_inc.data['8'], 1)
assert_equal(len(second_counter_inc.data), 2)
assert_true('3' in second_counter_inc.data)
assert_equal(second_counter_inc.data['3'], 4)
assert_true('2' in second_counter_inc.data)
assert_equal(second_counter_inc.data['2'], 1)
assert_false('1' in second_counter_inc.data)
assert_equal(len(third_counter_inc.data), 2)
assert_true('w' in third_counter_inc.data)
assert_equal(third_counter_inc.data['w'], 3)
assert_true('y' in third_counter_inc.data)
assert_equal(third_counter_inc.data['y'], 1)
assert_false('z' in third_counter_inc.data)
read_metrics = grouping_tail.read_metrics()
for key, key_value in counter_inc.data.items():
metric_name, value_type, value = read_metrics.next()
assert_equal(metric_name, str(key) + '.simple')
assert_equal(value_type, 'counter')
assert_equal(value, key_value)
second_read_metrics = second_grouping_tail.read_metrics()
for key, key_value in second_counter_inc.data.items():
metric_name, value_type, value = second_read_metrics.next()
assert_equal(metric_name, str(key) + '.second_simple')
assert_equal(value_type, 'counter')
assert_equal(value, key_value)
third_read_metrics = third_grouping_tail.read_metrics()
for key, key_value in third_counter_inc.data.items():
metric_name, value_type, value = third_read_metrics.next()
assert_equal(metric_name, str(key) + '.third_simple')
assert_equal(value_type, 'counter')
assert_equal(value, key_value)
def new_grouping_tail(log_file, group_by):
new_log = tempfile.NamedTemporaryFile(delete=False)
grouping_tail = GroupingTail(new_log.name, group_by)
copy_lines(log_file, new_log)
return grouping_tail