def post(self, filter_id):
'''
Validates and updates the filter.
Redirects to the new filter if successful.
:param filter_id: int
'''
filter_ = Filter.get(id_=filter_id)
if not filter_:
self.error(404, 'Filter not found')
form = forms.Filter(self.request.arguments)
if form.validate():
Filter.update(
id_=filter_id,
name=form.name.data,
data_yaml=form.data.data,
)
self.redirect('/filter/{}'.format(filter_.id))
return
self.render(
'filter.html',
title='New filter',
form=form,
edit=True,
filter=filter_,
members=Filter_team.get_teams_by_filter_id(filter_id=filter_id),
teams=Teams.get(),
logs_per_minute=Times_seen_by_minute.get_logs_per_minute(filter_id=filter_.id),
)
def post(self, filter_id):
'''
Validates and updates the filter.
Redirects to the new filter if successful.
:param filter_id: int
'''
filter_ = Filter.get(id_=filter_id)
if not filter_:
self.error(404, 'Filter not found')
form = forms.Filter(self.request.arguments)
if form.validate():
Filter.update(
id_=filter_id,
name=form.name.data,
data_yaml=form.data.data,
)
self.redirect('/filter/{}'.format(filter_.id))
return
self.render(
'filter.html',
title='New filter',
form=form,
edit=True,
filter=filter_,
members=Filter_team.get_teams_by_filter_id(filter_id=filter_id),
teams=Teams.get(),
logs_per_minute=Times_seen_by_minute.get_logs_per_minute(
filter_id=filter_.id),
)
def test_send(self):
'''
Creates 3 users, 2 teams and 2 filters.
User 1 and 2 is a member of team 1 and user 2 is a member of team 2.
Team 1 has a relation to both filter 1 and 2.
Team 2 has a relation to filter 2.
User 1 and 3 should receive an email and User 2 should not.
'''
user = User.new(name=u'Test user 1',
email='*****@*****.**',
notification_types={
'send_email': {
'data': '*****@*****.**',
'enabled': True,
}
})
user2 = User.new(name=u'Test user 2',
email='*****@*****.**',
notification_types={
'send_email': {
'data': '*****@*****.**',
'enabled': False,
}
})
user3 = User.new(name=u'Test user 3',
email='*****@*****.**',
notification_types={
'send_email': {
'data': '*****@*****.**',
'enabled': True,
}
})
team = Team.new(name=u'Test team 1', )
team2 = Team.new(name=u'Test team 2', )
filter1 = Filter.new(
name=u'Test filter 1',
data_yaml='notify: true',
)
filter2 = Filter.new(
name=u'Test filter 2',
data_yaml='notify: false',
)
# create a relations
User_team.new(team_id=team.id, user_id=user.id)
User_team.new(team_id=team2.id, user_id=user2.id)
User_team.new(team_id=team.id, user_id=user3.id)
Filter_team.new(filter_id=filter1.id, team_id=team.id)
Filter_team.new(filter_id=filter2.id, team_id=team.id)
Filter_team.new(filter_id=filter2.id, team_id=team2.id)
Mock_notifier.email_send = False
Notifier.send(message='Test notification',
filters=[
filter1,
filter2,
])
self.assertTrue(Mock_notifier.email_send)
def test_check(self):
parsed = Parsed(
hostname='te-pc',
level=0,
data={
"message": "Some test message",
},
standard='Test standard',
)
data = [{
'name': 'test filter 1',
'match': {
'hostname': [
'te-pc',
],
'message': ['^[a-zA-Z ]+$'],
}
}, {
'name': 'test filter 2',
'match': {
'hostname': [
'te-pc',
],
'message': ['^[a-zA-Z ]+$'],
}
}]
filter1 = Filter(
id_=1,
version=1,
name='test',
data_yaml=yaml.safe_dump(data),
data=data,
)
data = {
'match': {
'hostname': [
'something wrong',
],
'message': ['^[a-zA-Z ]+$'],
}
}
filter2 = Filter(
id_=2,
version=1,
name='test',
data=data,
data_yaml=yaml.safe_dump(data),
)
filters = [filter1, filter2]
filter_matches = Filters_checker.check(filters, parsed)
self.assertEqual(len(filter_matches), 2)
self.assertEqual(filter_matches[0].id, filter1.id)
self.assertEqual(filter_matches[1].id, filter1.id)
def test_update(self):
filter_ = Filter.new(name=u'Test filter ø', data_yaml='')
Filter.update(
id_=filter_.id,
name=u'Test filter 2 ø',
data_yaml='{}'
)
filter2 = Filter.get(id_=filter_.id)
self.assertTrue(filter2.id>0)
self.assertEqual(filter2.name, u'Test filter 2 ø')
self.assertTrue(isinstance(filter2.data, dict))
self.check_filter_version(filter2.id, filter2.version, filter2.data)
def upgrade():
op.add_column('filters',
sa.Column('data_yaml', sa.Text)
)
op.add_column('filter_versions',
sa.Column('data_yaml', sa.Text)
)
for f in Filters.get():
Filter.update(
id_=f.id,
name=f.name,
data_yaml=yaml.safe_dump(f.data),
)
def test_get(self):
'''
Creates 2 filters and creates a relation between the filters a a log_group.
Tests that both strict and non strict version works.
'''
filter1 = Filter.new(name=u'Test filter 1', data_yaml='')
filter2 = Filter.new(name=u'Test filter 2', data_yaml='')
store = Store(Parse(u'<34>Oct 11 22:14:15 mymachine.example.com su - ID47 - ZwPpeQyUtrRKxw5'))
group = Log_group.add(store)
group = Log_group.get(message_hash=store.message_hash)
self.assertTrue(
Log_group_filters.add(
filters=[filter1, filter2],
log_group_id=group.id,
)
)
groups = Log_groups.get(
filters = [filter1, filter2],
)
self.assertEqual(len(groups), 1)
self.assertEqual(groups[0].id, group.id)
# test that the strict version also works.
filter2.update(id_=filter2.id, name='Test filter asd', data_yaml='')
filter2 = Filter.get(id_=filter2.id)
store = Store(Parse(u'<34>Oct 11 22:14:15 mymachine.example.com su - ID47 - NEW MESSAGE ZwPpeQyUtrRKxw5'))
group = Log_group.add(store)
group = Log_group.get(message_hash=store.message_hash)
self.assertTrue(
Log_group_filters.add(
filters=[filter1, filter2],
log_group_id=group.id,
)
)
groups = Log_groups.get(
filters = [filter2],
)
self.assertEqual(len(groups), 2)
groups = Log_groups.get(
filters = [filter2],
strict_version = True,
)
self.assertEqual(len(groups), 1)
def test_add(self):
filter1 = Filter.new(name=u'Test filter 1', data_yaml='')
filter2 = Filter.new(name=u'Test filter 2', data_yaml='')
store = Store(Parse(u'<34>Oct 11 22:14:15 mymachine.example.com su - ID47 - ZwPpeQyUtrRKxw5'))
group = Log_group.add(store)
group = Log_group.get(message_hash=store.message_hash)
self.assertTrue(
Log_group_filters.add(
filters=[filter1, filter2],
log_group_id=group.id,
)
)
# test get
filters = Log_group_filters.get(log_group_id=group.id)
self.assertEqual(len(filters), 2)
def test(self):
# new
filter_ = Filter.new(name=u'Test filter ø', data_yaml='')
team = Team.new(name=u'Test team ø')
self.assertTrue(
Filter_team.new(
filter_id=filter_.id,
team_id=team.id,
)
)
# get teams by filter_id
teams = Filter_team.get_teams_by_filter_id(filter_id=filter_.id)
self.assertEqual(len(teams), 1)
self.assertEqual(team.id, teams[0].id)
# delete
self.assertTrue(
Filter_team.delete(
filter_id=filter_.id,
team_id=team.id,
)
)
teams = Filter_team.get_teams_by_filter_id(filter_id=filter_.id)
self.assertEqual(len(teams), 0)
def test_new(self):
filter_ = Filter.new(name=u'Test filter ø', data_yaml='{}')
self.assertTrue(filter_.id>0)
self.assertEqual(filter_.name, u'Test filter ø')
self.assertTrue(isinstance(filter_.data, dict))
self.check_filter_version(filter_.id, filter_.version, filter_.data)
def _get_filters(self):
'''
If the filter_id argument is set, only log groups matching it will be shown.
Otherwise all the users filters will be used.
:returns list of tlog.base.filter.Filter
'''
filter_id = self.get_argument('filter_id', None)
if not filter_id:
return Filters_user.get(user_id=self.current_user.id)
else:
return [Filter.get(id_=filter_id)]
def test_update(self):
store = Store(Parse(u'<34>Oct 11 22:14:15 mymachine.example.com su - ID47 - ZwPpeQyUtrRKxw5'))
group = Log_group.add(store)
filter_ = Filter.new(name=u'Test filter ø', data_yaml='')
Times_seen_by_minute.update(
log_group_id=group.id,
filter_id=filter_.id,
)
minutes = Times_seen_by_minute.get_by_log_group_id(log_group_id=group.id)
self.assertEqual(len(minutes), 1)
self.assertEqual(minutes[0].times_seen, 1)
def test_set_events(self):
filter_ = Filter.new(u'Test filter ø', data_yaml='store: true')
store = Store(Parse(u'<34>Oct 11 22:14:15 mymachine.example.com su: BOM\'su root\' failed for lonvick on /dev/pts/8 æøå'), [filter_])
store.save()
Log_group.update_status(
id_=store.log_group.id,
status=constants.STATUS_RESOLVED,
reopened=None,
)
store = Store(Parse(u'<34>Oct 11 22:14:15 mymachine.example.com su: BOM\'su root\' failed for lonvick on /dev/pts/8 æøå'), [filter_])
store.save()
self.assertEqual(Mock_log_group_event.message, 'reopened this log group')
def get(cls, log_group_id):
'''
Returns a list of filters by `log_group_id`.
:param log_group_id: int
:returns: list of Filter
'''
with new_session() as session:
query = session.query(models.Filter, ).filter(
models.Filter_log_group.log_group_id == log_group_id,
models.Filter.id == models.Filter_log_group.filter_id,
).all()
filters = []
for filter_ in query:
filters.append(Filter._format_from_query(filter_))
return filters
def test_update(self):
store = Store(
Parse(
u'<34>Oct 11 22:14:15 mymachine.example.com su - ID47 - ZwPpeQyUtrRKxw5'
))
group = Log_group.add(store)
filter_ = Filter.new(name=u'Test filter ø', data_yaml='')
Times_seen_by_minute.update(
log_group_id=group.id,
filter_id=filter_.id,
)
minutes = Times_seen_by_minute.get_by_log_group_id(
log_group_id=group.id)
self.assertEqual(len(minutes), 1)
self.assertEqual(minutes[0].times_seen, 1)
def get(cls, log_group_id):
'''
Returns a list of filters by `log_group_id`.
:param log_group_id: int
:returns: list of Filter
'''
with new_session() as session:
query = session.query(
models.Filter,
).filter(
models.Filter_log_group.log_group_id == log_group_id,
models.Filter.id==models.Filter_log_group.filter_id,
).all()
filters = []
for filter_ in query:
filters.append(Filter._format_from_query(filter_))
return filters
def get_filters_to_check(cls):
'''
Retrieves a list of filters that has been active in the latest interval.
These filters should be check for early warning.
:returns: list of Filter_warning
'''
now = datetime.utcnow()
from_date = now - timedelta(minutes=MINUTE_NORMALIZATION)
with new_session() as session:
query = session.query(
models.Times_seen_by_minute,
models.Filter,
func.sum(models.Times_seen_by_minute.times_seen).label(
'times_seen'),
).filter(
models.Times_seen_by_minute.time >= from_date,
models.Filter.id == models.Times_seen_by_minute.filter_id,
).group_by(models.Times_seen_by_minute.filter_id, ).all()
check = []
for data in query:
filter_ = Filter._format_from_query(data.Filter)
if filter_.data.get('rate_warning', {}).get('enabled', False):
if data.times_seen == 0:
continue
minutes = (now -
data.Times_seen_by_minute.time).seconds / 60
if minutes == 0:
continue
normalized_count = int(
data.times_seen /
((now - data.Times_seen_by_minute.time).seconds / 60))
min_logs = filter_.data['rate_warning'].get(
'min_logs', 100)
if normalized_count >= min_logs:
check.append(
cls(
filter_=filter_,
from_date=from_date,
normalized_count=normalized_count,
threshold=filter_.data['rate_warning'].get(
'threshold', 500),
))
return check
def post(self):
'''
Validates and creates a new filter.
Redirects to the new filter if successful.
'''
form = forms.Filter(self.request.arguments)
if form.validate():
filter_ = Filter.new(
name=form.name.data,
data_yaml=form.data.data,
)
self.redirect('/filter/{}'.format(filter_.id))
return
self.render(
'filter.html',
title='New filter',
form=form,
edit=False,
)
def post(self):
'''
Validates and creates a new filter.
Redirects to the new filter if successful.
'''
form = forms.Filter(self.request.arguments)
if form.validate():
filter_ = Filter.new(
name=form.name.data,
data_yaml=form.data.data,
)
self.redirect('/filter/{}'.format(filter_.id))
return
self.render(
'filter.html',
title='New filter',
form=form,
edit=False,
)
def test_check_filter_warning(self):
with new_session() as session:
session.query(models.Filter).delete()
filter1 = Filter.new(
u'Test filter 1',
yaml.safe_dump({
'rate_warning': {
'enabled': True,
'min_logs': 100,
'threshold': 500,
}
}))
store = Store(
Parse(
u'<34>Oct 11 22:14:15 mymachine.example.com su - ID47 - ZwPpeQyUtrRKxw5123'
))
group1 = Log_group.add(store)
# create some intervals
intervals = 20
now = datetime.utcnow()
for i in xrange(1, intervals):
Times_seen_by_minute.update(
log_group_id=group1.id,
filter_id=filter1.id,
when=normalize_datetime(now - timedelta(
minutes=(i * MINUTE_NORMALIZATION))),
inc=1000 + 10 * i,
)
# create what would look like a lot of new messages in a short time. This should be enough to trigger then warning notification.
when = datetime.utcnow() - timedelta(minutes=1, seconds=30)
Times_seen_by_minute.update(
log_group_id=group1.id,
filter_id=filter1.id,
when=when,
inc=1000,
)
filters_to_check = Filter_warning.get_filters_to_check()
self.assertTrue(
Filter_warning.check_filter_warning(filters_to_check[0]), )
def test_check(self):
with new_session() as session:
session.query(models.Filter).delete()
filter1 = Filter.new(u'Test filter 1', yaml.safe_dump({
'inactivity': {
'enabled': True,
'minutes': 15,
}
}))
store = Store(Parse(u'<34>Oct 11 22:14:15 mymachine.example.com su - ID47 - ZwPpeQyUtrRKxw5'))
group1 = Log_group.add(store)
group1 = Log_group.get(message_hash=store.message_hash)
when = datetime.utcnow() - timedelta(minutes=16)
Times_seen_by_minute.update(
log_group_id=group1.id,
filter_id=filter1.id,
when=when,
)
self.assertTrue(Filter_inactivity.check())
def get_filters_to_check(cls):
'''
Retrieves a list of filters that has been active in the latest interval.
These filters should be check for early warning.
:returns: list of Filter_warning
'''
now = datetime.utcnow()
from_date = now - timedelta(minutes=MINUTE_NORMALIZATION)
with new_session() as session:
query = session.query(
models.Times_seen_by_minute,
models.Filter,
func.sum(models.Times_seen_by_minute.times_seen).label('times_seen'),
).filter(
models.Times_seen_by_minute.time >= from_date,
models.Filter.id == models.Times_seen_by_minute.filter_id,
).group_by(
models.Times_seen_by_minute.filter_id,
).all()
check = []
for data in query:
filter_ = Filter._format_from_query(data.Filter)
if filter_.data.get('rate_warning', {}).get('enabled', False):
if data.times_seen == 0:
continue
minutes = (now - data.Times_seen_by_minute.time).seconds / 60
if minutes == 0:
continue
normalized_count = int(data.times_seen / ((now - data.Times_seen_by_minute.time).seconds / 60))
min_logs = filter_.data['rate_warning'].get('min_logs', 100)
if normalized_count >= min_logs:
check.append(
cls(
filter_=filter_,
from_date=from_date,
normalized_count=normalized_count,
threshold=filter_.data['rate_warning'].get('threshold', 500),
)
)
return check
def test_set_events(self):
filter_ = Filter.new(u'Test filter ø', data_yaml='store: true')
store = Store(
Parse(
u'<34>Oct 11 22:14:15 mymachine.example.com su: BOM\'su root\' failed for lonvick on /dev/pts/8 æøå'
), [filter_])
store.save()
Log_group.update_status(
id_=store.log_group.id,
status=constants.STATUS_RESOLVED,
reopened=None,
)
store = Store(
Parse(
u'<34>Oct 11 22:14:15 mymachine.example.com su: BOM\'su root\' failed for lonvick on /dev/pts/8 æøå'
), [filter_])
store.save()
self.assertEqual(Mock_log_group_event.message,
'reopened this log group')
def get(self, filter_id):
'''
Renders the edit filter form.
:param filter_id: int
'''
filter_ = Filter.get(id_=filter_id)
if not filter_:
self.error(404, 'Filter not found')
form = forms.Filter()
form.name.data = filter_.name
form.data.data = filter_.data_yaml
self.render(
'filter.html',
title=u'Filter: {}'.format(filter_.name),
form=form,
edit=True,
filter=filter_,
members=Filter_team.get_teams_by_filter_id(filter_id=filter_id),
teams=Teams.get(),
logs_per_minute=Times_seen_by_minute.get_logs_per_minute(filter_id=filter_.id),
)
def test_check(self):
with new_session() as session:
session.query(models.Filter).delete()
filter1 = Filter.new(
u'Test filter 1',
yaml.safe_dump({'inactivity': {
'enabled': True,
'minutes': 15,
}}))
store = Store(
Parse(
u'<34>Oct 11 22:14:15 mymachine.example.com su - ID47 - ZwPpeQyUtrRKxw5'
))
group1 = Log_group.add(store)
group1 = Log_group.get(message_hash=store.message_hash)
when = datetime.utcnow() - timedelta(minutes=16)
Times_seen_by_minute.update(
log_group_id=group1.id,
filter_id=filter1.id,
when=when,
)
self.assertTrue(Filter_inactivity.check())
def get(self, filter_id):
'''
Renders the edit filter form.
:param filter_id: int
'''
filter_ = Filter.get(id_=filter_id)
if not filter_:
self.error(404, 'Filter not found')
form = forms.Filter()
form.name.data = filter_.name
form.data.data = filter_.data_yaml
self.render(
'filter.html',
title=u'Filter: {}'.format(filter_.name),
form=form,
edit=True,
filter=filter_,
members=Filter_team.get_teams_by_filter_id(filter_id=filter_id),
teams=Teams.get(),
logs_per_minute=Times_seen_by_minute.get_logs_per_minute(
filter_id=filter_.id),
)
def test_check_filter_warning(self):
with new_session() as session:
session.query(models.Filter).delete()
filter1 = Filter.new(u'Test filter 1', yaml.safe_dump({
'rate_warning': {
'enabled': True,
'min_logs': 100,
'threshold': 500,
}
}))
store = Store(Parse(u'<34>Oct 11 22:14:15 mymachine.example.com su - ID47 - ZwPpeQyUtrRKxw5123'))
group1 = Log_group.add(store)
# create some intervals
intervals = 20
now = datetime.utcnow()
for i in xrange(1, intervals):
Times_seen_by_minute.update(
log_group_id=group1.id,
filter_id=filter1.id,
when=normalize_datetime(now - timedelta(minutes=(i * MINUTE_NORMALIZATION))),
inc=1000 + 10 * i,
)
# create what would look like a lot of new messages in a short time. This should be enough to trigger then warning notification.
when = datetime.utcnow() - timedelta(minutes=1, seconds=30)
Times_seen_by_minute.update(
log_group_id=group1.id,
filter_id=filter1.id,
when=when,
inc=1000,
)
filters_to_check = Filter_warning.get_filters_to_check()
self.assertTrue(
Filter_warning.check_filter_warning(filters_to_check[0]),
)
def test_send(self):
'''
Creates 3 users, 2 teams and 2 filters.
User 1 and 2 is a member of team 1 and user 2 is a member of team 2.
Team 1 has a relation to both filter 1 and 2.
Team 2 has a relation to filter 2.
User 1 and 3 should receive an email and User 2 should not.
'''
user = User.new(
name=u'Test user 1',
email='*****@*****.**',
notification_types={
'send_email': {
'data': '*****@*****.**',
'enabled': True,
}
}
)
user2 = User.new(
name=u'Test user 2',
email='*****@*****.**',
notification_types={
'send_email': {
'data': '*****@*****.**',
'enabled': False,
}
}
)
user3 = User.new(
name=u'Test user 3',
email='*****@*****.**',
notification_types={
'send_email': {
'data': '*****@*****.**',
'enabled': True,
}
}
)
team = Team.new(
name=u'Test team 1',
)
team2 = Team.new(
name=u'Test team 2',
)
filter1 = Filter.new(
name=u'Test filter 1',
data_yaml='notify: true',
)
filter2 = Filter.new(
name=u'Test filter 2',
data_yaml='notify: false',
)
# create a relations
User_team.new(
team_id=team.id,
user_id=user.id
)
User_team.new(
team_id=team2.id,
user_id=user2.id
)
User_team.new(
team_id=team.id,
user_id=user3.id
)
Filter_team.new(
filter_id=filter1.id,
team_id=team.id
)
Filter_team.new(
filter_id=filter2.id,
team_id=team.id
)
Filter_team.new(
filter_id=filter2.id,
team_id=team2.id
)
Mock_notifier.email_send = False
Notifier.send(
message='Test notification',
filters=[
filter1,
filter2,
]
)
self.assertTrue(Mock_notifier.email_send)
def test_check(self):
parsed = Parsed(
hostname='te-pc',
level=0,
data={
"message": "Some test message",
},
standard='Test standard',
)
filter_ = Filter(
id_=0,
version=1,
name='test',
data={'store': True},
data_yaml='store: true',
)
filter_.data = {
'match': {
'message': '^[a-zA-Z ]+$',
}
}
self.assertTrue(Filter_checker.check(filter_, parsed))
# check that we do not accidentally remove the `data` field
# in th parsed message when validating it against a filter.
store = Store(parsed, [filter_])
self.assertTrue(isinstance(store, Store))
filter_.data = {
'match': {
'hostname': [
'something wrong',
],
'message': [
'^[a-zA-Z ]+$'
],
}
}
self.assertFalse(Filter_checker.check(filter_, parsed))
filter_.data = {
'match': {
'level': [
'[0-9]+'
]
}
}
self.assertTrue(Filter_checker.check(filter_, parsed))
filter_.data = {
'match': {
'level': [
'[5-6]+'
]
}
}
self.assertFalse(Filter_checker.check(filter_, parsed))
filter_.data = {
'match': {
'hostname': [
'te-pc',
],
'data': {
'message': [
'^[a-zA-Z ]+$'
],
}
},
'not_match': {
'hostname': [
'te-pc',
]
}
}
self.assertFalse(Filter_checker.check(filter_, parsed))
filter_.data = {
'match': {
'level': [
'[0-9]+'
]
},
'not_match': {
'hostname': [
'te-pc',
]
}
}
self.assertFalse(Filter_checker.check(filter_, parsed))
filter_.data = {
'match': {
'level': [
'[0-9]+'
]
},
'not_match': {
'hostname': [
'kurtkurtsen',
]
}
}
self.assertTrue(Filter_checker.check(filter_, parsed))
filter_.data = {
'not_match': {
'hostname': [
'kurtkurtsen',
]
}
}
self.assertTrue(Filter_checker.check(filter_, parsed))
filter_.data = {
'not_match': {
'level': [
'[0-9]+',
]
}
}
self.assertFalse(Filter_checker.check(filter_, parsed))
def test_get_filters_to_check(self):
with new_session() as session:
session.query(models.Filter).delete()
'''
Checks that `get_filters_to_check` only returns those filters that has been active in the latests interval.
'''
filter1 = Filter.new(
u'Test filter 1',
yaml.safe_dump({
'rate_warning': {
'enabled': True,
'min_logs': 100,
'threshold': 500,
}
}))
filter2 = Filter.new(
u'Test filter 1',
yaml.safe_dump({
'rate_warning': {
'enabled': True,
'min_logs': 100,
'threshold': 500,
}
}))
store = Store(
Parse(
u'<34>Oct 11 22:14:15 mymachine.example.com su - ID47 - ZwPpeQyUtrRKxw5'
))
group1 = Log_group.add(store)
group1 = Log_group.get(message_hash=store.message_hash)
store = Store(
Parse(
u'<34>Oct 11 22:14:15 mymachine.example.com su - ID47 - ZwPpeQyUtrRKxw5'
))
group2 = Log_group.add(store)
group2 = Log_group.get(message_hash=store.message_hash)
# add som info that should not show up in the list we have to check for warnings.
when = datetime.utcnow() - timedelta(minutes=MINUTE_NORMALIZATION)
Times_seen_by_minute.update(
log_group_id=group1.id,
filter_id=filter1.id,
when=when,
inc=25,
)
when = datetime.utcnow() - timedelta(minutes=10, seconds=30)
# checks that the filters groups correctly
Times_seen_by_minute.update(
log_group_id=group1.id,
filter_id=filter1.id,
when=when,
inc=1000,
)
Times_seen_by_minute.update(
log_group_id=group2.id,
filter_id=filter1.id,
when=when,
inc=150,
)
# there should not be enough messages received for this filter to be checked for alerts.
Times_seen_by_minute.update(
log_group_id=group1.id,
filter_id=filter2.id,
when=when,
inc=1000,
)
now = datetime.utcnow()
from_date = now - timedelta(minutes=MINUTE_NORMALIZATION)
filters_to_check = Filter_warning.get_filters_to_check()
self.assertTrue(len(filters_to_check) > 0)
self.assertEqual(filters_to_check[0].normalized_count, 100)
return filters_to_check
def test_get(self):
Filter.new(u'Test filter ø', data_yaml='')
filters = Filters.get()
self.assertTrue(len(filters)>0)
def get_filter(self):
return Filter.new(
name=u'Test filter ø',
data_yaml='store: true',
)
def test_check(self):
parsed = Parsed(
hostname='te-pc',
level=0,
data={
"message": "Some test message",
},
standard='Test standard',
)
filter_ = Filter(
id_=0,
version=1,
name='test',
data={'store': True},
data_yaml='store: true',
)
filter_.data = {
'match': {
'message': '^[a-zA-Z ]+$',
}
}
self.assertTrue(Filter_checker.check(filter_, parsed))
# check that we do not accidentally remove the `data` field
# in th parsed message when validating it against a filter.
store = Store(parsed, [filter_])
self.assertTrue(isinstance(store, Store))
filter_.data = {
'match': {
'hostname': [
'something wrong',
],
'message': ['^[a-zA-Z ]+$'],
}
}
self.assertFalse(Filter_checker.check(filter_, parsed))
filter_.data = {'match': {'level': ['[0-9]+']}}
self.assertTrue(Filter_checker.check(filter_, parsed))
filter_.data = {'match': {'level': ['[5-6]+']}}
self.assertFalse(Filter_checker.check(filter_, parsed))
filter_.data = {
'match': {
'hostname': [
'te-pc',
],
'data': {
'message': ['^[a-zA-Z ]+$'],
}
},
'not_match': {
'hostname': [
'te-pc',
]
}
}
self.assertFalse(Filter_checker.check(filter_, parsed))
filter_.data = {
'match': {
'level': ['[0-9]+']
},
'not_match': {
'hostname': [
'te-pc',
]
}
}
self.assertFalse(Filter_checker.check(filter_, parsed))
filter_.data = {
'match': {
'level': ['[0-9]+']
},
'not_match': {
'hostname': [
'kurtkurtsen',
]
}
}
self.assertTrue(Filter_checker.check(filter_, parsed))
filter_.data = {
'not_match': {
'hostname': [
'kurtkurtsen',
]
}
}
self.assertTrue(Filter_checker.check(filter_, parsed))
filter_.data = {
'not_match': {
'level': [
'[0-9]+',
]
}
}
self.assertFalse(Filter_checker.check(filter_, parsed))
def test_get_filters_to_check(self):
with new_session() as session:
session.query(models.Filter).delete()
'''
Checks that `get_filters_to_check` only returns those filters that has been active in the latests interval.
'''
filter1 = Filter.new(u'Test filter 1', yaml.safe_dump({
'rate_warning': {
'enabled': True,
'min_logs': 100,
'threshold': 500,
}
}))
filter2 = Filter.new(u'Test filter 1', yaml.safe_dump({
'rate_warning': {
'enabled': True,
'min_logs': 100,
'threshold': 500,
}
}))
store = Store(Parse(u'<34>Oct 11 22:14:15 mymachine.example.com su - ID47 - ZwPpeQyUtrRKxw5'))
group1 = Log_group.add(store)
group1 = Log_group.get(message_hash=store.message_hash)
store = Store(Parse(u'<34>Oct 11 22:14:15 mymachine.example.com su - ID47 - ZwPpeQyUtrRKxw5'))
group2 = Log_group.add(store)
group2 = Log_group.get(message_hash=store.message_hash)
# add som info that should not show up in the list we have to check for warnings.
when = datetime.utcnow() - timedelta(minutes=MINUTE_NORMALIZATION)
Times_seen_by_minute.update(
log_group_id=group1.id,
filter_id=filter1.id,
when=when,
inc=25,
)
when = datetime.utcnow() - timedelta(minutes=10, seconds=30)
# checks that the filters groups correctly
Times_seen_by_minute.update(
log_group_id=group1.id,
filter_id=filter1.id,
when=when,
inc=1000,
)
Times_seen_by_minute.update(
log_group_id=group2.id,
filter_id=filter1.id,
when=when,
inc=150,
)
# there should not be enough messages received for this filter to be checked for alerts.
Times_seen_by_minute.update(
log_group_id=group1.id,
filter_id=filter2.id,
when=when,
inc=1000,
)
now = datetime.utcnow()
from_date = now - timedelta(minutes=MINUTE_NORMALIZATION)
filters_to_check = Filter_warning.get_filters_to_check()
self.assertTrue(len(filters_to_check) > 0)
self.assertEqual(filters_to_check[0].normalized_count, 100)
return filters_to_check
def get_filter(self):
return Filter.new(
name=u'Test filter ø',
data_yaml='store: true',
)