def get_logs_per_minute(cls, filter_id):
'''
Returns logs received per minute for the latest interval.
:param filter_id: int
:returns: int
'''
now = datetime.utcnow()
min_date = now - timedelta(minutes=MINUTE_NORMALIZATION)
with new_session() as session:
query = session.query(
func.min(models.Times_seen_by_minute.time).label('time'),
func.sum(models.Times_seen_by_minute.times_seen).label(
'times_seen'),
).filter(
models.Times_seen_by_minute.filter_id == filter_id,
models.Times_seen_by_minute.time >= min_date,
).group_by(models.Times_seen_by_minute.filter_id, ).first()
if not query:
return 0
if query.times_seen == 0:
return 0
seconds_normalized = (now - query.time).seconds / 60
if seconds_normalized == 0:
return 0
return int(query.times_seen / seconds_normalized)
def get_by_team_list(cls, teams):
'''
Returns a grouped list of users that are members.
:param teams: list of tlog.base.team.Team
:returns: list of User
'''
if not teams:
return []
team_ids = []
for team in teams:
team_ids.append(team.id)
with new_session() as session:
query = session.query(
models.User_team,
models.User,
).filter(
models.User_team.team_id.in_(team_ids),
models.User.id==models.User_team.user_id,
).group_by(
models.User.id,
).all()
users = []
for user in query:
users.append(User._format_from_query(user.User))
return users
def get(self, filter_id):
from_ = datetime.utcnow() - timedelta(days=int(self.get_argument('days', 1)))
with new_session() as session:
query = session.query(
func.min(models.Times_seen_by_minute.time).label('time'),
func.sum(models.Times_seen_by_minute.times_seen).label('times_seen'),
).filter(
models.Times_seen_by_minute.filter_id == filter_id,
models.Times_seen_by_minute.time >= from_,
).group_by(
models.Times_seen_by_minute.time,
models.Times_seen_by_minute.filter_id,
).all()
data = self.prefill_data(from_, datetime.utcnow())
for d in query:
timestamp = int(timegm(d.time.timetuple()))
for a in data:
if a['x'] == timestamp:
a['y'] = int(d.times_seen)
self.write_object([
{
"color": "#cae2f7",
"name": "Times seen",
"data": data,
}
])
def change_password(cls, id_, password, current_password=None):
'''
Changes a users password.
Set `current_password` to verify the users current password, before changing it.
:param id: int
:param password: str
:param current_password: str
:returns: boolean
'''
with new_session() as session:
if current_password:
if not cls.verify_password(id_=id_, password=current_password):
return False
session.query(
models.User,
).filter(
models.User.id==id_,
).update({
'password': bcrypt.hashpw(password.encode('utf-8'), bcrypt.gensalt(10)),
})
session.commit()
return True
def get(self, filter_id):
from_ = datetime.utcnow() - timedelta(
days=int(self.get_argument('days', 1)))
with new_session() as session:
query = session.query(
func.min(models.Times_seen_by_minute.time).label('time'),
func.sum(models.Times_seen_by_minute.times_seen).label(
'times_seen'),
).filter(
models.Times_seen_by_minute.filter_id == filter_id,
models.Times_seen_by_minute.time >= from_,
).group_by(
models.Times_seen_by_minute.time,
models.Times_seen_by_minute.filter_id,
).all()
data = self.prefill_data(from_, datetime.utcnow())
for d in query:
timestamp = int(timegm(d.time.timetuple()))
for a in data:
if a['x'] == timestamp:
a['y'] = int(d.times_seen)
self.write_object([{
"color": "#cae2f7",
"name": "Times seen",
"data": data,
}])
def update_count(self, save_filters):
'''
Updates counts for Times_seen_by_minute, Log group times seen and server times seen.
:param session: db session
:param save_filters: list of tlog.base.fliter.Filter
'''
with new_session() as session:
try:
for filter_ in save_filters:
Times_seen_by_minute._update(
session=session,
log_group_id=self.log_group.id,
filter_id=filter_.id,
)
Log_group._inc_seen(session=session, log_group=self.log_group)
Server_count._add(
session=session,
log_group_id=self.log_group.id,
name=self.hostname,
)
session.commit()
except IntegrityError as e:
# try again
logging.exception('Times seen duplicate key, trying again.')
self.update_count(save_filters)
def add(cls, store):
'''
Tries to group the `store` object with other stored objects, that has the same message_hash.
:param store: tlog.receiver.store.Store
:returns: Log_group
'''
with new_session() as session:
query = session.query(models.Log_group, ).filter(
models.Log_group.message_hash == store.message_hash, ).update({
'level':
store.level,
})
if not query:
group = models.Log_group(
message=store.data.get('message', ''),
message_hash=store.message_hash,
level=store.level,
)
try:
session.add(group)
session.commit()
return cls._format_from_query(group)
except IntegrityError as e:
# Try again if there was a duplicate key.
logging.exception(
'Log message group duplicate key, trying again.')
return cls.add(store)
else:
return cls.get(message_hash=store.message_hash, )
def add(cls, store):
'''
Tries to group the `store` object with other stored objects, that has the same message_hash.
:param store: tlog.receiver.store.Store
:returns: Log_group
'''
with new_session() as session:
query = session.query(
models.Log_group,
).filter(
models.Log_group.message_hash==store.message_hash,
).update({
'level': store.level,
})
if not query:
group = models.Log_group(
message = store.data.get('message', ''),
message_hash = store.message_hash,
level = store.level,
)
try:
session.add(group)
session.commit()
return cls._format_from_query(group)
except IntegrityError as e:
# Try again if there was a duplicate key.
logging.exception('Log message group duplicate key, trying again.')
return cls.add(store)
else:
return cls.get(
message_hash=store.message_hash,
)
def get_logs_per_minute(cls, filter_id):
'''
Returns logs received per minute for the latest interval.
:param filter_id: int
:returns: int
'''
now = datetime.utcnow()
min_date = now - timedelta(minutes=MINUTE_NORMALIZATION)
with new_session() as session:
query = session.query(
func.min(models.Times_seen_by_minute.time).label('time'),
func.sum(models.Times_seen_by_minute.times_seen).label('times_seen'),
).filter(
models.Times_seen_by_minute.filter_id == filter_id,
models.Times_seen_by_minute.time >= min_date,
).group_by(
models.Times_seen_by_minute.filter_id,
).first()
if not query:
return 0
if query.times_seen == 0:
return 0
seconds_normalized = (now - query.time).seconds / 60
if seconds_normalized == 0:
return 0
return int(query.times_seen / seconds_normalized)
def get(cls, id_):
'''
:param id_: int
:returns: Team
'''
with new_session() as session:
team = session.query(models.Team, ).filter(
models.Team.id == id_, ).first()
return cls._format_from_query(team)
def get_by_name(cls, name):
'''
:param name: str
:returns: Team
'''
with new_session() as session:
team = session.query(models.Team, ).filter(
models.Team.name == name, ).first()
return cls._format_from_query(team)
def get(cls, id_):
'''
:param id_: int
:returns: Log
'''
with new_session() as session:
query = session.query(models.Log, ).filter(
models.Log.id == id_, ).first()
return cls._format_from_query(query)
def inc_seen(cls, log_group):
'''
Use this funtion to inc times_seen and change last_seen to now.
:param log_group: Log_group
:returns: boolean
'''
with new_session() as session:
cls._inc_seen(session, log_group)
return True
def inc_seen(cls, log_group):
'''
Use this funtion to inc times_seen and change last_seen to now.
:param log_group: Log_group
:returns: boolean
'''
with new_session() as session:
cls._inc_seen(session, log_group)
return True
def get(cls, message_hash):
'''
:param message_hash: str
:returns: Log_group
'''
with new_session() as session:
group = session.query(models.Log_group, ).filter(
models.Log_group.message_hash == message_hash, ).first()
return cls._format_from_query(group)
def get_by_id(cls, id_):
'''
:param id_: int
:returns: Log_group
'''
with new_session() as session:
group = session.query(models.Log_group, ).filter(
models.Log_group.id == id_, ).first()
return cls._format_from_query(group)
def get_by_email(cls, email):
'''
Retrieves the user by its email address.
:param email: str
:returns: User
'''
with new_session() as session:
user = session.query(models.User, ).filter(
models.User.email == email, ).first()
return cls._format_from_query(user)
def get(cls, id_):
'''
Retrieves the user by its id.
:param id_: int
:returns: User
'''
with new_session() as session:
user = session.query(models.User, ).filter(
models.User.id == id_, ).first()
return cls._format_from_query(user)
def add(cls, log_group_id, name):
'''
Creates or updates a servers count for at log group.
:param log_group_id: int
:param name: str
:returns: boolean
'''
with new_session() as session:
cls._add(session, log_group_id, name)
return True
def get(cls, id_):
'''
Retrieves a filter by it's id.
:param id_: int
:returns: Filter
'''
with new_session() as session:
query = session.query(models.Filter, ).filter(
models.Filter.id == id_, ).first()
return cls._format_from_query(query)
def get(cls,
filters,
strict_version=False,
extra_filter=None,
order_by=None,
limit=25,
offset=0):
'''
Retrieves a list of `Log_group` by filters.
:param filters: list of Filter
:param strict_version: boolean
If true the id and version from `filters` has to match in the relation.
:param extra_filter: list of models fields to match (SQLALchemy filter)
extra_filter=[models.Log_group.id==1]
:param: order_by: list of models fields (SQLALchemy order_by)
order_by=[models.Log_group.id, models.Log_group.score]
:param limit: int
:param offset: int
Page 2 would be limit*2
:returns: list of tlog.base.log_group.Log_group
'''
if not filters:
return []
with new_session() as session:
query = session.query(models.Log_group, )
if strict_version:
f = []
for filter_ in filters:
f.append(
and_(
models.Filter_log_group.filter_id == filter_.id,
models.Filter_log_group.filter_version ==
filter_.version))
query = query.filter(or_(*f))
else:
ids = []
for filter_ in filters:
ids.append(filter_.id)
query = query.filter(
models.Filter_log_group.filter_id.in_(ids))
query = query.filter(
models.Log_group.id == models.Filter_log_group.log_group_id, )
if extra_filter:
query = query.filter(*extra_filter)
if order_by:
query = query.order_by(*order_by)
query = query.group_by(models.Log_group.id, )
query = query.limit(limit).offset(offset)
query.all()
groups = []
for group in query:
groups.append(Log_group._format_from_query(group))
return groups
def add(cls, log_group_id, name):
'''
Creates or updates a servers count for at log group.
:param log_group_id: int
:param name: str
:returns: boolean
'''
with new_session() as session:
cls._add(session, log_group_id, name)
return True
def delete(cls, id_):
'''
Deletes the log group and all its relations.
:param id_: int
:returns: boolean
'''
with new_session() as session:
session.query(models.Log_group, ).filter(
models.Log_group.id == id_, ).delete()
session.commit()
return True
def get(cls, id_):
'''
:param id_: int
:returns: Team
'''
with new_session() as session:
team = session.query(
models.Team,
).filter(
models.Team.id==id_,
).first()
return cls._format_from_query(team)
def get_by_name(cls, name):
'''
:param name: str
:returns: Team
'''
with new_session() as session:
team = session.query(
models.Team,
).filter(
models.Team.name==name,
).first()
return cls._format_from_query(team)
def get(cls):
'''
Retrieves all filters.
:returns: list of Filter
'''
with new_session() as session:
query = session.query(models.Filter, ).all()
filters = []
for filter_ in query:
filters.append(Filter._format_from_query(filter_))
return filters
def update_last_log_id(cls, id_, last_log_id):
'''
Updates the last_log_id field.
Should be called after a new message has been written to the logs table.
:param id_: int
:param last_log_id: int
:returns: boolean
'''
with new_session() as session:
cls._update_last_log_id(session, id_, last_log_id)
return True
def get(cls, id_):
'''
:param id_: int
:returns: Log
'''
with new_session() as session:
query = session.query(
models.Log,
).filter(
models.Log.id == id_,
).first()
return cls._format_from_query(query)
def get(cls, log_group_id):
'''
:param log_group_id: int
:returns: list of Server_count
'''
with new_session() as session:
query = session.query(models.Log_group_server, ).filter(
models.Log_group_server.log_group_id == log_group_id, ).all()
servers = []
for server in query:
servers.append(Server_count._format_from_query(server))
return servers
def update_last_log_id(cls, id_, last_log_id):
'''
Updates the last_log_id field.
Should be called after a new message has been written to the logs table.
:param id_: int
:param last_log_id: int
:returns: boolean
'''
with new_session() as session:
cls._update_last_log_id(session, id_, last_log_id)
return True
def get(cls, filters, strict_version=False, extra_filter=None, order_by=None, limit=25, offset=0):
'''
Retrieves a list of `Log_group` by filters.
:param filters: list of Filter
:param strict_version: boolean
If true the id and version from `filters` has to match in the relation.
:param extra_filter: list of models fields to match (SQLALchemy filter)
extra_filter=[models.Log_group.id==1]
:param: order_by: list of models fields (SQLALchemy order_by)
order_by=[models.Log_group.id, models.Log_group.score]
:param limit: int
:param offset: int
Page 2 would be limit*2
:returns: list of tlog.base.log_group.Log_group
'''
if not filters:
return []
with new_session() as session:
query = session.query(
models.Log_group,
)
if strict_version:
f = []
for filter_ in filters:
f.append(and_(models.Filter_log_group.filter_id == filter_.id, models.Filter_log_group.filter_version == filter_.version))
query = query.filter(
or_(*f)
)
else:
ids = []
for filter_ in filters:
ids.append(filter_.id)
query = query.filter(
models.Filter_log_group.filter_id.in_(ids)
)
query = query.filter(
models.Log_group.id == models.Filter_log_group.log_group_id,
)
if extra_filter:
query = query.filter(*extra_filter)
if order_by:
query = query.order_by(*order_by)
query = query.group_by(
models.Log_group.id,
)
query = query.limit(limit).offset(offset)
query.all()
groups = []
for group in query:
groups.append(Log_group._format_from_query(group))
return groups
def update(cls, id_, name):
'''
:param id_: int
:param name: str
:returns: boolean
'''
with new_session() as session:
session.query(models.Team, ).filter(models.Team.id == id_).update({
'name':
name,
})
session.commit()
return True
def get(cls):
'''
Retrieves all teams.
:returns: list of Team
'''
with new_session() as session:
query = session.query(models.Team, ).order_by(
asc(models.Team.name), ).all()
teams = []
for team in query:
teams.append(Team._format_from_query(team))
return teams
def get(cls, message_hash):
'''
:param message_hash: str
:returns: Log_group
'''
with new_session() as session:
group = session.query(
models.Log_group,
).filter(
models.Log_group.message_hash==message_hash,
).first()
return cls._format_from_query(group)
def get(cls):
'''
Retrieves all users.
:returns: list of User
'''
with new_session() as session:
query = session.query(models.User, ).order_by(
asc(models.User.name), ).all()
users = []
for user in query:
users.append(User._format_from_query(user))
return users
def get_next(cls, id_, log_group_id):
'''
Returns a next log in the same log group.
:param id_: int
:returns: Log
'''
with new_session() as session:
query = session.query(models.Log, ).filter(
models.Log.log_group_id == log_group_id,
models.Log.id > id_,
).order_by(asc(models.Log.id), ).first()
return cls._format_from_query(query)
def get_by_log_group_id(cls, log_group_id):
'''
:param log_group_id: int
:returns: list of Times_seen_by_minute
'''
with new_session() as session:
query = session.query(models.Times_seen_by_minute, ).filter(
models.Times_seen_by_minute.log_group_id ==
log_group_id).all()
minutes = []
for minute in query:
minutes.append(cls._format_from_query(minute))
return minutes
def get_by_id(cls, id_):
'''
:param id_: int
:returns: Log_group
'''
with new_session() as session:
group = session.query(
models.Log_group,
).filter(
models.Log_group.id==id_,
).first()
return cls._format_from_query(group)
def get(cls, filter_id, version):
'''
Retrieves a filter from the versioning table.
:param filter_id: int
:param version: int
:returns: Filter_version
'''
with new_session() as session:
query = session.query(models.Filter_version, ).filter(
models.Filter_version.filter_id == filter_id,
models.Filter_version.version == version,
).first()
return cls._format_from_query(query)
def get(cls, id_):
'''
Retrieves a filter by it's id.
:param id_: int
:returns: Filter
'''
with new_session() as session:
query = session.query(
models.Filter,
).filter(
models.Filter.id==id_,
).first()
return cls._format_from_query(query)
def get(cls, id_):
'''
Retrieves the user by its id.
:param id_: int
:returns: User
'''
with new_session() as session:
user = session.query(
models.User,
).filter(
models.User.id==id_,
).first()
return cls._format_from_query(user)
def pong(cls):
'''
When receiving a ping, we have to respond with a pong.
We do this by inserting a heartbeat in the database with a timestamp.
The ping sender can then check the database for the heartbeat and send an alert if it didn't came through.
'''
with new_session() as session:
heartbeat = models.Watchdog(
id=1,
heartbeat=datetime.utcnow(),
)
session.merge(heartbeat)
session.commit()
def pong(cls):
'''
When receiving a ping, we have to respond with a pong.
We do this by inserting a heartbeat in the database with a timestamp.
The ping sender can then check the database for the heartbeat and send an alert if it didn't came through.
'''
with new_session() as session:
heartbeat = models.Watchdog(
id=1,
heartbeat=datetime.utcnow(),
)
session.merge(heartbeat)
session.commit()
def get(cls):
'''
Retrieves all filters.
:returns: list of Filter
'''
with new_session() as session:
query = session.query(
models.Filter,
).all()
filters = []
for filter_ in query:
filters.append(Filter._format_from_query(filter_))
return filters
def get_by_email(cls, email):
'''
Retrieves the user by its email address.
:param email: str
:returns: User
'''
with new_session() as session:
user = session.query(
models.User,
).filter(
models.User.email==email,
).first()
return cls._format_from_query(user)
def check(cls):
f = Filters.get()
filters = {}
filter_ids = []
for filter_ in f:
if 'inactivity' in filter_.data:
if filter_.data['inactivity'].get('enabled', False):
filters[filter_.id] = filter_
filter_ids.append(filter_.id)
if not filters:
return False
with new_session() as session:
times_seen = session.query(
models.Times_seen_by_minute.filter_id,
func.max(models.Times_seen_by_minute.time).label('time'),
).filter(
models.Times_seen_by_minute.filter_id.in_(filter_ids),
).group_by(
models.Times_seen_by_minute.filter_id,
).all()
inactive_filters = []
now = datetime.utcnow()
for seen in times_seen:
max_inactive_minutes = filters[seen.filter_id].data['inactivity'].get('minutes')
if max_inactive_minutes < MINUTE_NORMALIZATION:
max_inactive_minutes = MINUTE_NORMALIZATION
if ((now - seen.time).seconds / 60) > max_inactive_minutes:
inactive_filters.append(filters[seen.filter_id])
filters.pop(seen.filter_id, None)
for filter_ in filters:
inactivity_filters.append(filters[filter_])
logging.info('{} inactive filters'.format(len(inactive_filters)))
if not inactive_filters:
return False
for filter_ in inactive_filters:
if Filter_notification_last_sent.check(filter_id=filter_.id):
Filter_notification_last_sent.update(filter_id=filter_.id)
Notifier.send(
message=u'Early warning. Filter: {} has been marked as inactive.'.format(
filter_.name,
),
filters=[
filter_,
],
force_send=True,
)
return True
def get_by_log_group_id(cls, log_group_id):
'''
:param log_group_id: int
:returns: list of Times_seen_by_minute
'''
with new_session() as session:
query = session.query(
models.Times_seen_by_minute,
).filter(
models.Times_seen_by_minute.log_group_id==log_group_id
).all()
minutes = []
for minute in query:
minutes.append(cls._format_from_query(minute))
return minutes
def delete(cls, session_id):
'''
Removes a user's session from the database.
:param session_id: str
:returns: boolean
'''
with new_session() as session:
session.query(
models.Web_session,
).filter(
models.Web_session.session==session_id,
).delete()
session.commit()
return True
def verify_password(cls, id_, password):
'''
Verifies that `password` is in fact the users password.
:param id_: int
:param password: str
:returns: boolean
'''
with new_session() as session:
user = session.query(models.User, ).filter(
models.User.id == id_, ).first()
if bcrypt.hashpw(password.encode('utf-8'),
user.password) == user.password:
return True
return False
def delete(cls, filter_id, team_id):
'''
Deletes a filter team relation.
:param filter_id: int
:param team_id: int
:returns: boolean
'''
with new_session() as session:
session.query(models.Filter_team, ).filter(
models.Filter_team.filter_id == filter_id,
models.Filter_team.team_id == team_id,
).delete()
session.commit()
return True
def delete(cls, user_id, team_id):
'''
Deletes the relation between a user and a team.
:param user_id: int
:param team_id: int
:returns: boolean
'''
with new_session() as session:
session.query(models.User_team, ).filter(
models.User_team.user_id == user_id,
models.User_team.team_id == team_id,
).delete()
session.commit()
return True
def delete(cls, id_):
'''
Deletes the log group and all its relations.
:param id_: int
:returns: boolean
'''
with new_session() as session:
session.query(
models.Log_group,
).filter(
models.Log_group.id==id_,
).delete()
session.commit()
return True
def update(cls, log_group_id, filter_id, when=None, inc=1):
'''
Increments times_seen for the latest minute by log_group_id and filter_id.
:param log_group_id: int
:param filter_id: int
:param when: datetime - default None
If none a normalized current date and time will be used.
:param inc: int - default 1
The number of times `times_seen` should be incremented.
:returns: boolean
'''
with new_session() as session:
cls._update(session, log_group_id, filter_id, when, inc)
return True
def get(cls, log_group_id):
'''
:param log_group_id: int
:returns: list of Server_count
'''
with new_session() as session:
query = session.query(
models.Log_group_server,
).filter(
models.Log_group_server.log_group_id==log_group_id,
).all()
servers = []
for server in query:
servers.append(Server_count._format_from_query(server))
return servers
def new(cls, filter_id, team_id):
'''
Creates a new filter team relation.
:param filter_id: int
:param team_id: int
:returns: boolean
'''
with new_session() as session:
ft = models.Filter_team(
filter_id=filter_id,
team_id=team_id,
)
session.merge(ft)
session.commit()
return True
def new(cls, user_id, team_id):
'''
Creates a relation between a user and a team.
:param user_id: int
:param team_id: int
:returns: boolean
'''
with new_session() as session:
ut = models.User_team(
user_id=user_id,
team_id=team_id,
)
session.merge(ut)
session.commit()
return True
def get(cls):
'''
Retrieves all users.
:returns: list of User
'''
with new_session() as session:
query = session.query(
models.User,
).order_by(
asc(models.User.name),
).all()
users = []
for user in query:
users.append(User._format_from_query(user))
return users
def update(cls, id_, name):
'''
:param id_: int
:param name: str
:returns: boolean
'''
with new_session() as session:
session.query(
models.Team,
).filter(
models.Team.id==id_
).update({
'name': name,
})
session.commit()
return True
def get(cls):
'''
Retrieves all teams.
:returns: list of Team
'''
with new_session() as session:
query = session.query(
models.Team,
).order_by(
asc(models.Team.name),
).all()
teams = []
for team in query:
teams.append(Team._format_from_query(team))
return teams
def get(cls, filter_id, version):
'''
Retrieves a filter from the versioning table.
:param filter_id: int
:param version: int
:returns: Filter_version
'''
with new_session() as session:
query = session.query(
models.Filter_version,
).filter(
models.Filter_version.filter_id==filter_id,
models.Filter_version.version==version,
).first()
return cls._format_from_query(query)
