def get_logs_per_minute(cls, filter_id): ''' Returns logs received per minute for the latest interval. :param filter_id: int :returns: int ''' now = datetime.utcnow() min_date = now - timedelta(minutes=MINUTE_NORMALIZATION) with new_session() as session: query = session.query( func.min(models.Times_seen_by_minute.time).label('time'), func.sum(models.Times_seen_by_minute.times_seen).label( 'times_seen'), ).filter( models.Times_seen_by_minute.filter_id == filter_id, models.Times_seen_by_minute.time >= min_date, ).group_by(models.Times_seen_by_minute.filter_id, ).first() if not query: return 0 if query.times_seen == 0: return 0 seconds_normalized = (now - query.time).seconds / 60 if seconds_normalized == 0: return 0 return int(query.times_seen / seconds_normalized)
def get_by_team_list(cls, teams): ''' Returns a grouped list of users that are members. :param teams: list of tlog.base.team.Team :returns: list of User ''' if not teams: return [] team_ids = [] for team in teams: team_ids.append(team.id) with new_session() as session: query = session.query( models.User_team, models.User, ).filter( models.User_team.team_id.in_(team_ids), models.User.id==models.User_team.user_id, ).group_by( models.User.id, ).all() users = [] for user in query: users.append(User._format_from_query(user.User)) return users
def get(self, filter_id): from_ = datetime.utcnow() - timedelta(days=int(self.get_argument('days', 1))) with new_session() as session: query = session.query( func.min(models.Times_seen_by_minute.time).label('time'), func.sum(models.Times_seen_by_minute.times_seen).label('times_seen'), ).filter( models.Times_seen_by_minute.filter_id == filter_id, models.Times_seen_by_minute.time >= from_, ).group_by( models.Times_seen_by_minute.time, models.Times_seen_by_minute.filter_id, ).all() data = self.prefill_data(from_, datetime.utcnow()) for d in query: timestamp = int(timegm(d.time.timetuple())) for a in data: if a['x'] == timestamp: a['y'] = int(d.times_seen) self.write_object([ { "color": "#cae2f7", "name": "Times seen", "data": data, } ])
def change_password(cls, id_, password, current_password=None): ''' Changes a users password. Set `current_password` to verify the users current password, before changing it. :param id: int :param password: str :param current_password: str :returns: boolean ''' with new_session() as session: if current_password: if not cls.verify_password(id_=id_, password=current_password): return False session.query( models.User, ).filter( models.User.id==id_, ).update({ 'password': bcrypt.hashpw(password.encode('utf-8'), bcrypt.gensalt(10)), }) session.commit() return True
def get(self, filter_id): from_ = datetime.utcnow() - timedelta( days=int(self.get_argument('days', 1))) with new_session() as session: query = session.query( func.min(models.Times_seen_by_minute.time).label('time'), func.sum(models.Times_seen_by_minute.times_seen).label( 'times_seen'), ).filter( models.Times_seen_by_minute.filter_id == filter_id, models.Times_seen_by_minute.time >= from_, ).group_by( models.Times_seen_by_minute.time, models.Times_seen_by_minute.filter_id, ).all() data = self.prefill_data(from_, datetime.utcnow()) for d in query: timestamp = int(timegm(d.time.timetuple())) for a in data: if a['x'] == timestamp: a['y'] = int(d.times_seen) self.write_object([{ "color": "#cae2f7", "name": "Times seen", "data": data, }])
def update_count(self, save_filters): ''' Updates counts for Times_seen_by_minute, Log group times seen and server times seen. :param session: db session :param save_filters: list of tlog.base.fliter.Filter ''' with new_session() as session: try: for filter_ in save_filters: Times_seen_by_minute._update( session=session, log_group_id=self.log_group.id, filter_id=filter_.id, ) Log_group._inc_seen(session=session, log_group=self.log_group) Server_count._add( session=session, log_group_id=self.log_group.id, name=self.hostname, ) session.commit() except IntegrityError as e: # try again logging.exception('Times seen duplicate key, trying again.') self.update_count(save_filters)
def add(cls, store): ''' Tries to group the `store` object with other stored objects, that has the same message_hash. :param store: tlog.receiver.store.Store :returns: Log_group ''' with new_session() as session: query = session.query(models.Log_group, ).filter( models.Log_group.message_hash == store.message_hash, ).update({ 'level': store.level, }) if not query: group = models.Log_group( message=store.data.get('message', ''), message_hash=store.message_hash, level=store.level, ) try: session.add(group) session.commit() return cls._format_from_query(group) except IntegrityError as e: # Try again if there was a duplicate key. logging.exception( 'Log message group duplicate key, trying again.') return cls.add(store) else: return cls.get(message_hash=store.message_hash, )
def add(cls, store): ''' Tries to group the `store` object with other stored objects, that has the same message_hash. :param store: tlog.receiver.store.Store :returns: Log_group ''' with new_session() as session: query = session.query( models.Log_group, ).filter( models.Log_group.message_hash==store.message_hash, ).update({ 'level': store.level, }) if not query: group = models.Log_group( message = store.data.get('message', ''), message_hash = store.message_hash, level = store.level, ) try: session.add(group) session.commit() return cls._format_from_query(group) except IntegrityError as e: # Try again if there was a duplicate key. logging.exception('Log message group duplicate key, trying again.') return cls.add(store) else: return cls.get( message_hash=store.message_hash, )
def get_logs_per_minute(cls, filter_id): ''' Returns logs received per minute for the latest interval. :param filter_id: int :returns: int ''' now = datetime.utcnow() min_date = now - timedelta(minutes=MINUTE_NORMALIZATION) with new_session() as session: query = session.query( func.min(models.Times_seen_by_minute.time).label('time'), func.sum(models.Times_seen_by_minute.times_seen).label('times_seen'), ).filter( models.Times_seen_by_minute.filter_id == filter_id, models.Times_seen_by_minute.time >= min_date, ).group_by( models.Times_seen_by_minute.filter_id, ).first() if not query: return 0 if query.times_seen == 0: return 0 seconds_normalized = (now - query.time).seconds / 60 if seconds_normalized == 0: return 0 return int(query.times_seen / seconds_normalized)
def get(cls, id_): ''' :param id_: int :returns: Team ''' with new_session() as session: team = session.query(models.Team, ).filter( models.Team.id == id_, ).first() return cls._format_from_query(team)
def get_by_name(cls, name): ''' :param name: str :returns: Team ''' with new_session() as session: team = session.query(models.Team, ).filter( models.Team.name == name, ).first() return cls._format_from_query(team)
def get(cls, id_): ''' :param id_: int :returns: Log ''' with new_session() as session: query = session.query(models.Log, ).filter( models.Log.id == id_, ).first() return cls._format_from_query(query)
def inc_seen(cls, log_group): ''' Use this funtion to inc times_seen and change last_seen to now. :param log_group: Log_group :returns: boolean ''' with new_session() as session: cls._inc_seen(session, log_group) return True
def inc_seen(cls, log_group): ''' Use this funtion to inc times_seen and change last_seen to now. :param log_group: Log_group :returns: boolean ''' with new_session() as session: cls._inc_seen(session, log_group) return True
def get(cls, message_hash): ''' :param message_hash: str :returns: Log_group ''' with new_session() as session: group = session.query(models.Log_group, ).filter( models.Log_group.message_hash == message_hash, ).first() return cls._format_from_query(group)
def get_by_id(cls, id_): ''' :param id_: int :returns: Log_group ''' with new_session() as session: group = session.query(models.Log_group, ).filter( models.Log_group.id == id_, ).first() return cls._format_from_query(group)
def get_by_email(cls, email): ''' Retrieves the user by its email address. :param email: str :returns: User ''' with new_session() as session: user = session.query(models.User, ).filter( models.User.email == email, ).first() return cls._format_from_query(user)
def get(cls, id_): ''' Retrieves the user by its id. :param id_: int :returns: User ''' with new_session() as session: user = session.query(models.User, ).filter( models.User.id == id_, ).first() return cls._format_from_query(user)
def add(cls, log_group_id, name): ''' Creates or updates a servers count for at log group. :param log_group_id: int :param name: str :returns: boolean ''' with new_session() as session: cls._add(session, log_group_id, name) return True
def get(cls, id_): ''' Retrieves a filter by it's id. :param id_: int :returns: Filter ''' with new_session() as session: query = session.query(models.Filter, ).filter( models.Filter.id == id_, ).first() return cls._format_from_query(query)
def get(cls, filters, strict_version=False, extra_filter=None, order_by=None, limit=25, offset=0): ''' Retrieves a list of `Log_group` by filters. :param filters: list of Filter :param strict_version: boolean If true the id and version from `filters` has to match in the relation. :param extra_filter: list of models fields to match (SQLALchemy filter) extra_filter=[models.Log_group.id==1] :param: order_by: list of models fields (SQLALchemy order_by) order_by=[models.Log_group.id, models.Log_group.score] :param limit: int :param offset: int Page 2 would be limit*2 :returns: list of tlog.base.log_group.Log_group ''' if not filters: return [] with new_session() as session: query = session.query(models.Log_group, ) if strict_version: f = [] for filter_ in filters: f.append( and_( models.Filter_log_group.filter_id == filter_.id, models.Filter_log_group.filter_version == filter_.version)) query = query.filter(or_(*f)) else: ids = [] for filter_ in filters: ids.append(filter_.id) query = query.filter( models.Filter_log_group.filter_id.in_(ids)) query = query.filter( models.Log_group.id == models.Filter_log_group.log_group_id, ) if extra_filter: query = query.filter(*extra_filter) if order_by: query = query.order_by(*order_by) query = query.group_by(models.Log_group.id, ) query = query.limit(limit).offset(offset) query.all() groups = [] for group in query: groups.append(Log_group._format_from_query(group)) return groups
def add(cls, log_group_id, name): ''' Creates or updates a servers count for at log group. :param log_group_id: int :param name: str :returns: boolean ''' with new_session() as session: cls._add(session, log_group_id, name) return True
def delete(cls, id_): ''' Deletes the log group and all its relations. :param id_: int :returns: boolean ''' with new_session() as session: session.query(models.Log_group, ).filter( models.Log_group.id == id_, ).delete() session.commit() return True
def get(cls, id_): ''' :param id_: int :returns: Team ''' with new_session() as session: team = session.query( models.Team, ).filter( models.Team.id==id_, ).first() return cls._format_from_query(team)
def get_by_name(cls, name): ''' :param name: str :returns: Team ''' with new_session() as session: team = session.query( models.Team, ).filter( models.Team.name==name, ).first() return cls._format_from_query(team)
def get(cls): ''' Retrieves all filters. :returns: list of Filter ''' with new_session() as session: query = session.query(models.Filter, ).all() filters = [] for filter_ in query: filters.append(Filter._format_from_query(filter_)) return filters
def update_last_log_id(cls, id_, last_log_id): ''' Updates the last_log_id field. Should be called after a new message has been written to the logs table. :param id_: int :param last_log_id: int :returns: boolean ''' with new_session() as session: cls._update_last_log_id(session, id_, last_log_id) return True
def get(cls, id_): ''' :param id_: int :returns: Log ''' with new_session() as session: query = session.query( models.Log, ).filter( models.Log.id == id_, ).first() return cls._format_from_query(query)
def get(cls, log_group_id): ''' :param log_group_id: int :returns: list of Server_count ''' with new_session() as session: query = session.query(models.Log_group_server, ).filter( models.Log_group_server.log_group_id == log_group_id, ).all() servers = [] for server in query: servers.append(Server_count._format_from_query(server)) return servers
def update_last_log_id(cls, id_, last_log_id): ''' Updates the last_log_id field. Should be called after a new message has been written to the logs table. :param id_: int :param last_log_id: int :returns: boolean ''' with new_session() as session: cls._update_last_log_id(session, id_, last_log_id) return True
def get(cls, filters, strict_version=False, extra_filter=None, order_by=None, limit=25, offset=0): ''' Retrieves a list of `Log_group` by filters. :param filters: list of Filter :param strict_version: boolean If true the id and version from `filters` has to match in the relation. :param extra_filter: list of models fields to match (SQLALchemy filter) extra_filter=[models.Log_group.id==1] :param: order_by: list of models fields (SQLALchemy order_by) order_by=[models.Log_group.id, models.Log_group.score] :param limit: int :param offset: int Page 2 would be limit*2 :returns: list of tlog.base.log_group.Log_group ''' if not filters: return [] with new_session() as session: query = session.query( models.Log_group, ) if strict_version: f = [] for filter_ in filters: f.append(and_(models.Filter_log_group.filter_id == filter_.id, models.Filter_log_group.filter_version == filter_.version)) query = query.filter( or_(*f) ) else: ids = [] for filter_ in filters: ids.append(filter_.id) query = query.filter( models.Filter_log_group.filter_id.in_(ids) ) query = query.filter( models.Log_group.id == models.Filter_log_group.log_group_id, ) if extra_filter: query = query.filter(*extra_filter) if order_by: query = query.order_by(*order_by) query = query.group_by( models.Log_group.id, ) query = query.limit(limit).offset(offset) query.all() groups = [] for group in query: groups.append(Log_group._format_from_query(group)) return groups
def update(cls, id_, name): ''' :param id_: int :param name: str :returns: boolean ''' with new_session() as session: session.query(models.Team, ).filter(models.Team.id == id_).update({ 'name': name, }) session.commit() return True
def get(cls): ''' Retrieves all teams. :returns: list of Team ''' with new_session() as session: query = session.query(models.Team, ).order_by( asc(models.Team.name), ).all() teams = [] for team in query: teams.append(Team._format_from_query(team)) return teams
def get(cls, message_hash): ''' :param message_hash: str :returns: Log_group ''' with new_session() as session: group = session.query( models.Log_group, ).filter( models.Log_group.message_hash==message_hash, ).first() return cls._format_from_query(group)
def get(cls): ''' Retrieves all users. :returns: list of User ''' with new_session() as session: query = session.query(models.User, ).order_by( asc(models.User.name), ).all() users = [] for user in query: users.append(User._format_from_query(user)) return users
def get_next(cls, id_, log_group_id): ''' Returns a next log in the same log group. :param id_: int :returns: Log ''' with new_session() as session: query = session.query(models.Log, ).filter( models.Log.log_group_id == log_group_id, models.Log.id > id_, ).order_by(asc(models.Log.id), ).first() return cls._format_from_query(query)
def get_by_log_group_id(cls, log_group_id): ''' :param log_group_id: int :returns: list of Times_seen_by_minute ''' with new_session() as session: query = session.query(models.Times_seen_by_minute, ).filter( models.Times_seen_by_minute.log_group_id == log_group_id).all() minutes = [] for minute in query: minutes.append(cls._format_from_query(minute)) return minutes
def get_by_id(cls, id_): ''' :param id_: int :returns: Log_group ''' with new_session() as session: group = session.query( models.Log_group, ).filter( models.Log_group.id==id_, ).first() return cls._format_from_query(group)
def get(cls, filter_id, version): ''' Retrieves a filter from the versioning table. :param filter_id: int :param version: int :returns: Filter_version ''' with new_session() as session: query = session.query(models.Filter_version, ).filter( models.Filter_version.filter_id == filter_id, models.Filter_version.version == version, ).first() return cls._format_from_query(query)
def get(cls, id_): ''' Retrieves a filter by it's id. :param id_: int :returns: Filter ''' with new_session() as session: query = session.query( models.Filter, ).filter( models.Filter.id==id_, ).first() return cls._format_from_query(query)
def get(cls, id_): ''' Retrieves the user by its id. :param id_: int :returns: User ''' with new_session() as session: user = session.query( models.User, ).filter( models.User.id==id_, ).first() return cls._format_from_query(user)
def pong(cls): ''' When receiving a ping, we have to respond with a pong. We do this by inserting a heartbeat in the database with a timestamp. The ping sender can then check the database for the heartbeat and send an alert if it didn't came through. ''' with new_session() as session: heartbeat = models.Watchdog( id=1, heartbeat=datetime.utcnow(), ) session.merge(heartbeat) session.commit()
def pong(cls): ''' When receiving a ping, we have to respond with a pong. We do this by inserting a heartbeat in the database with a timestamp. The ping sender can then check the database for the heartbeat and send an alert if it didn't came through. ''' with new_session() as session: heartbeat = models.Watchdog( id=1, heartbeat=datetime.utcnow(), ) session.merge(heartbeat) session.commit()
def get(cls): ''' Retrieves all filters. :returns: list of Filter ''' with new_session() as session: query = session.query( models.Filter, ).all() filters = [] for filter_ in query: filters.append(Filter._format_from_query(filter_)) return filters
def get_by_email(cls, email): ''' Retrieves the user by its email address. :param email: str :returns: User ''' with new_session() as session: user = session.query( models.User, ).filter( models.User.email==email, ).first() return cls._format_from_query(user)
def check(cls): f = Filters.get() filters = {} filter_ids = [] for filter_ in f: if 'inactivity' in filter_.data: if filter_.data['inactivity'].get('enabled', False): filters[filter_.id] = filter_ filter_ids.append(filter_.id) if not filters: return False with new_session() as session: times_seen = session.query( models.Times_seen_by_minute.filter_id, func.max(models.Times_seen_by_minute.time).label('time'), ).filter( models.Times_seen_by_minute.filter_id.in_(filter_ids), ).group_by( models.Times_seen_by_minute.filter_id, ).all() inactive_filters = [] now = datetime.utcnow() for seen in times_seen: max_inactive_minutes = filters[seen.filter_id].data['inactivity'].get('minutes') if max_inactive_minutes < MINUTE_NORMALIZATION: max_inactive_minutes = MINUTE_NORMALIZATION if ((now - seen.time).seconds / 60) > max_inactive_minutes: inactive_filters.append(filters[seen.filter_id]) filters.pop(seen.filter_id, None) for filter_ in filters: inactivity_filters.append(filters[filter_]) logging.info('{} inactive filters'.format(len(inactive_filters))) if not inactive_filters: return False for filter_ in inactive_filters: if Filter_notification_last_sent.check(filter_id=filter_.id): Filter_notification_last_sent.update(filter_id=filter_.id) Notifier.send( message=u'Early warning. Filter: {} has been marked as inactive.'.format( filter_.name, ), filters=[ filter_, ], force_send=True, ) return True
def get_by_log_group_id(cls, log_group_id): ''' :param log_group_id: int :returns: list of Times_seen_by_minute ''' with new_session() as session: query = session.query( models.Times_seen_by_minute, ).filter( models.Times_seen_by_minute.log_group_id==log_group_id ).all() minutes = [] for minute in query: minutes.append(cls._format_from_query(minute)) return minutes
def delete(cls, session_id): ''' Removes a user's session from the database. :param session_id: str :returns: boolean ''' with new_session() as session: session.query( models.Web_session, ).filter( models.Web_session.session==session_id, ).delete() session.commit() return True
def verify_password(cls, id_, password): ''' Verifies that `password` is in fact the users password. :param id_: int :param password: str :returns: boolean ''' with new_session() as session: user = session.query(models.User, ).filter( models.User.id == id_, ).first() if bcrypt.hashpw(password.encode('utf-8'), user.password) == user.password: return True return False
def delete(cls, filter_id, team_id): ''' Deletes a filter team relation. :param filter_id: int :param team_id: int :returns: boolean ''' with new_session() as session: session.query(models.Filter_team, ).filter( models.Filter_team.filter_id == filter_id, models.Filter_team.team_id == team_id, ).delete() session.commit() return True
def delete(cls, user_id, team_id): ''' Deletes the relation between a user and a team. :param user_id: int :param team_id: int :returns: boolean ''' with new_session() as session: session.query(models.User_team, ).filter( models.User_team.user_id == user_id, models.User_team.team_id == team_id, ).delete() session.commit() return True
def delete(cls, id_): ''' Deletes the log group and all its relations. :param id_: int :returns: boolean ''' with new_session() as session: session.query( models.Log_group, ).filter( models.Log_group.id==id_, ).delete() session.commit() return True
def update(cls, log_group_id, filter_id, when=None, inc=1): ''' Increments times_seen for the latest minute by log_group_id and filter_id. :param log_group_id: int :param filter_id: int :param when: datetime - default None If none a normalized current date and time will be used. :param inc: int - default 1 The number of times `times_seen` should be incremented. :returns: boolean ''' with new_session() as session: cls._update(session, log_group_id, filter_id, when, inc) return True
def get(cls, log_group_id): ''' :param log_group_id: int :returns: list of Server_count ''' with new_session() as session: query = session.query( models.Log_group_server, ).filter( models.Log_group_server.log_group_id==log_group_id, ).all() servers = [] for server in query: servers.append(Server_count._format_from_query(server)) return servers
def new(cls, filter_id, team_id): ''' Creates a new filter team relation. :param filter_id: int :param team_id: int :returns: boolean ''' with new_session() as session: ft = models.Filter_team( filter_id=filter_id, team_id=team_id, ) session.merge(ft) session.commit() return True
def new(cls, user_id, team_id): ''' Creates a relation between a user and a team. :param user_id: int :param team_id: int :returns: boolean ''' with new_session() as session: ut = models.User_team( user_id=user_id, team_id=team_id, ) session.merge(ut) session.commit() return True
def get(cls): ''' Retrieves all users. :returns: list of User ''' with new_session() as session: query = session.query( models.User, ).order_by( asc(models.User.name), ).all() users = [] for user in query: users.append(User._format_from_query(user)) return users
def update(cls, id_, name): ''' :param id_: int :param name: str :returns: boolean ''' with new_session() as session: session.query( models.Team, ).filter( models.Team.id==id_ ).update({ 'name': name, }) session.commit() return True
def get(cls): ''' Retrieves all teams. :returns: list of Team ''' with new_session() as session: query = session.query( models.Team, ).order_by( asc(models.Team.name), ).all() teams = [] for team in query: teams.append(Team._format_from_query(team)) return teams
def get(cls, filter_id, version): ''' Retrieves a filter from the versioning table. :param filter_id: int :param version: int :returns: Filter_version ''' with new_session() as session: query = session.query( models.Filter_version, ).filter( models.Filter_version.filter_id==filter_id, models.Filter_version.version==version, ).first() return cls._format_from_query(query)