def get(self, request): firewall_ids = get_firewall_rules_id_from_request(request) application = request.data.get('application', '') timestamp = request.data.get('timestamp') kwargs = { 'firewall_rule__in': firewall_ids, } if application: kwargs['application'] = application if timestamp: timestamp = int(timestamp) tz_ktm = pytz.timezone('Asia/Kathmandu') start_date = datetime.datetime.fromtimestamp(timestamp).astimezone( tz_ktm) end_date = start_date + datetime.timedelta(hours=1) kwargs['received_datetime__gte'] = start_date kwargs['received_datetime__lt'] = end_date objects = ThreatLogDetail.objects.filter(**kwargs).order_by('id') page = self.paginate_queryset(objects) if page is not None: serializer = self.serializer_class(page, many=True) return self.get_paginated_response(serializer.data) return Response({})
def post(self, request, format=None): top_count = int(request.data.get('topcount', 5)) basis = request.data.get('basis', 'bytes') country = request.data.get('country', '') # Send all applications if top_count is 0 # Assuming applications < 10000 top_count = top_count if top_count else 10000 kwargs = get_model_kwargs(request) if not kwargs.get('firewall_rule__in'): firewall_rule_ids = get_firewall_rules_id_from_request(request) kwargs['firewall_rule__in'] = firewall_rule_ids if kwargs.get('application__in'): del kwargs['application__in'] if country: kwargs['source_country'] = country objects = get_objects_with_date_filtered( request, TrafficLogDetailHourly, 'logged_datetime', **kwargs).values('logged_datetime', 'application').annotate( bytes=Sum('sum_bytes_sent') + Sum('sum_bytes_received'), packets=Sum('sum_packets_sent') + Sum('sum_packets_received'), count=Count('firewall_rule_id'), ).values( 'bytes', 'packets', 'count', date=F('logged_datetime'), application_name=F('application'), ) applications = [] max = 0 # To store the top applications data = defaultdict(int) # To store the data for each application temp = defaultdict(list) for obj in objects: data[obj['application_name']] += obj['bytes'] timestamp = obj['date'].timestamp() value = obj[basis] # Data is sent in TBPC order temp[obj['application_name']].append([timestamp, value]) max = value if value > max else max # Get the top n applications, sorted by bytes top_apps = sorted(data, key=data.get, reverse=True)[:top_count] # Only send data of the top n applications response = {i: temp[i] for i in top_apps} return Response({'data': response, 'max': max})
def get(self, request): firewall_ids = get_firewall_rules_id_from_request(request) id = request.data.get('log_id', '') ip = request.data.get('ip', '') timestamp = request.data.get('timestamp', '') hour = request.data.get('hour', '') kwargs = { 'firewall_rule__in': firewall_ids, } if id: kwargs['traffic_log__id'] = int(id) if ip: kwargs['source_address'] = ip if hour: kwargs['logged_datetime__hour'] = hour if timestamp: tz_ktm = pytz.timezone('Asia/Kathmandu') start_date = datetime.datetime.fromtimestamp( int(timestamp)).astimezone(tz_ktm) end_date = start_date + datetime.timedelta(hours=1) kwargs['logged_datetime__gte'] = start_date kwargs['logged_datetime__lt'] = end_date objects = TrafficLogDetailHourly.objects.filter( **kwargs).order_by('-id') page = self.paginate_queryset(objects) if page is not None: serializer = self.serializer_class(page, many=True) return self.get_paginated_response(serializer.data)
def close_tt(request, id): now = datetime.datetime.now() try: firewall_ids = get_firewall_rules_id_from_request(request) trouble_ticket = TroubleTicketAnomaly.objects.get( id=id, firewall_rule__in=firewall_ids) except Exception as e: return Response({ "traceback": str(traceback.format_exc()), "exception": str(e) }, status=HTTP_400_BAD_REQUEST) description = request.data.get('description', '') severity_level = request.data.get('severity_level') is_anomaly = int(request.data.get('is_anomaly', 'true') == 'true') user = get_user_from_token(request) trouble_ticket.is_closed = True trouble_ticket.is_anomaly = is_anomaly trouble_ticket.severity_level = severity_level trouble_ticket.assigned_to = user trouble_ticket.description = description trouble_ticket.verified_datetime = now trouble_ticket.verified_by = user trouble_ticket.save() follow_up = TroubleTicketFollowUpAnomaly( assigned_by=user, assigned_to=user, description=description, trouble_ticket=trouble_ticket, follow_up_datetime=now ) follow_up.save() return Response({'ok': 'tt closed'})
def get(self, request): firewall_ids = get_firewall_rules_id_from_request(request) application = request.data.get('application', '') timestamp = int(request.data.get('timestamp')) country = request.data.get('country', '') tz_ktm = pytz.timezone('Asia/Kathmandu') start_date = datetime.datetime.fromtimestamp(timestamp).astimezone( tz_ktm) end_date = start_date + datetime.timedelta(hours=1) kwargs = { 'firewall_rule__in': firewall_ids, 'application': application, 'logged_datetime__gte': start_date, 'logged_datetime__lt': end_date } if country: kwargs['source_country'] = country objects = TrafficLogDetailHourly.objects.filter( **kwargs).order_by('id') page = self.paginate_queryset(objects) if page is not None: serializer = self.serializer_class(page, many=True) return self.get_paginated_response(serializer.data) return Response({})
def post(self, request): firewall_ids = get_firewall_rules_id_from_request(request) ip = request.data.get('ip', '') alias = request.data.get('alias', '') source_kwargs = {} destination_kwargs = {} if ip: source_kwargs['source_address__icontains'] = ip destination_kwargs['destination_address__icontains'] = ip if alias: source_kwargs['alias__icontains'] = alias destination_kwargs['alias__icontains'] = alias source_objects = TrafficMisNewSourceIPDaily.objects.filter( firewall_rule__in=firewall_ids, **source_kwargs).annotate(address=F('source_address')).values_list( 'address', 'alias').order_by('alias') destination_objects = (TrafficMisNewDestinationIPDaily.objects.filter( firewall_rule__in=firewall_ids, **destination_kwargs).annotate( address=F('destination_address')).values_list( 'address', 'alias').order_by('alias')) objects = [] for i in (set(source_objects) | set(destination_objects)): objects.append({'address': i[0], 'alias': i[1]}) page = self.paginate_queryset(objects) if page is not None: serializer = self.serializer_class(page, many=True) return self.get_paginated_response(serializer.data)
def get(self, request): firewall_ids = get_firewall_rules_id_from_request(request) objects = TrafficMisRequestFromBlacklistedIPDaily.objects.filter( firewall_rule__in=firewall_ids ) page = self.paginate_queryset(objects.order_by('-logged_datetime')) if page is not None: serializer = self.serializer_class(page, many=True) return self.get_paginated_response(serializer.data)
def get_filtered_objects(self, request, **kwargs): firewall_rule_ids = get_firewall_rules_id_from_request(request) query = self.get_search_queries(request) objects = ThreatLogDetailEvent.objects.filter( firewall_rule__in=firewall_rule_ids, **kwargs, **query, ) return objects.order_by('id')
def post(self, request): firewall_ids = get_firewall_rules_id_from_request(request) objects = ThreatLogDetailEvent.objects.filter( firewall_rule__in=firewall_ids ).distinct('source_address').values('source_address') countries = set() for obj in objects: countries.add(get_country_name_and_code(obj['source_address'])) return Response(countries)
def get_filtered_objects(self, request, **kwargs): firewall_rule_ids = get_firewall_rules_id_from_request(request) query = self.get_search_queries(request) objects = TroubleTicketAnomaly.objects.filter( firewall_rule__in=firewall_rule_ids, **kwargs, **query, ) return objects.order_by('-logged_datetime')
def post(self, request, format=None): top_count = int(request.data.get('topcount', 5)) # Send all applications if top_count is 0 # Assuming applications < 10000 top_count = top_count if top_count else 10000 queries = get_query_from_request( request, model_name='', datetime_field_name='received_datetime') objects = get_objects_from_query(queries, model=ThreatLogDetailEvent) firewall_rule_ids = get_firewall_rules_id_from_request(request) kwargs = { 'firewall_rule__in': firewall_rule_ids, } country = request.data.get('country', '') # if country: # kwargs['source_country'] = country.title() applications = objects.values('application').annotate( sum=Sum('repeat_count') ).order_by('-sum').values('application')[:top_count] objects = objects.filter( application__in=applications, **kwargs ).values( 'received_datetime', 'application' ).annotate( repeat_count=Sum('repeat_count'), count=Count('firewall_rule_id'), ).values( 'count', 'repeat_count', date=F('received_datetime'), application_name=F('application'), ).order_by('received_datetime') applications = [] max = 0 data = defaultdict(list) for obj in objects: timestamp = obj['date'].timestamp() repeat_count = obj['repeat_count'] if max < repeat_count: max = repeat_count data[obj['application_name']].append([ timestamp, repeat_count ]) return Response({ 'data': data, 'max': max })
def get_response(self, request, model): firewall_ids = get_firewall_rules_id_from_request(request) try: objects = model.objects.filter( firewall_rule__in=firewall_ids).latest('id') date = get_date_from_filename( objects.log) - datetime.timedelta(days=1) return Response({'date': date.date()}) except Exception as e: return Response({'date': datetime.date.today()})
def get(self, request): firewall_ids = get_firewall_rules_id_from_request(request) objects = ProcessedThreatLogDetail.objects.filter( firewall_rule__in=firewall_ids).values( 'log', 'processed_datetime').annotate(rows=Sum('rows'), size=F('rows') * self.SIZE_PER_LOG) page = self.paginate_queryset(objects.order_by('-log')) if page is not None: serializer = self.serializer_class(page, many=True) return self.get_paginated_response(serializer.data)
def _get_items(self, request, model, field_name): firewall_ids = get_firewall_rules_id_from_request(request) date = str_to_date(request.data.get('date', '')) if not date: date = model.objects.latest( 'logged_datetime' ).logged_datetime objects = model.objects.filter( firewall_rule__in=firewall_ids, logged_datetime=date ).values_list(field_name, 'firewall_rule__name') return objects
def post(self, request): firewall_ids = get_firewall_rules_id_from_request(request) ip = request.data.get('ip') objects = list( TrafficMisRequestFromBlacklistedIPDaily.objects.filter( firewall_rule__in=firewall_ids, source_address=ip)) objects += list( TrafficMisResponseToBlacklistedIPDaily.objects.filter( firewall_rule__in=firewall_ids, destination_address=ip)) page = self.paginate_queryset(objects) if page is not None: serializer = self.serializer_class(page, many=True) return self.get_paginated_response(serializer.data)
def _get_items(self, request, model): firewall_ids = get_firewall_rules_id_from_request(request) objects = model.objects.filter( firewall_rule__in=firewall_ids ).values('logged_datetime').annotate( count=Count('firewall_rule'), date=F('logged_datetime') ).values('date', 'count').order_by('logged_datetime') items = [] for obj in objects: timestamp = obj['date'].strftime("%s") count = obj['count'] items.append([timestamp, count]) return items
def post(self, request, format=None): basis = request.data.get('basis', 'bytes') except_countries = request.data.get('except_countries', '') kwargs = get_model_kwargs(request) if not kwargs.get('firewall_rule__in'): firewall_rule_ids = get_firewall_rules_id_from_request(request) kwargs['firewall_rule__in'] = firewall_rule_ids objects = get_objects_with_date_filtered( request, TrafficLogDetailHourly, 'logged_datetime', **kwargs).values('source_country').annotate( bytes=Sum('sum_bytes_sent') + Sum('sum_bytes_received'), packets=Sum('sum_packets_sent') + Sum('sum_packets_received'), count=Count('firewall_rule_id')).values( 'source_country', basis) return Response(objects)
def get_filtered_objects(self, request, return_firewall_rules=False, **kwargs): firewall_rule_ids = get_firewall_rules_id_from_request(request) query = self.get_search_queries(request) alias = request.data.get('alias', None) aliased_ips = self._get_alias_ips(alias) objects = TrafficRule.objects.filter( firewall_rule__in=firewall_rule_ids, **kwargs, **query, ) if aliased_ips is not None: aliased_ips = [i[0] for i in aliased_ips] objects = objects.filter( Q(source_address__in=aliased_ips) | Q( destination_address__in=aliased_ips) ) if return_firewall_rules: return objects, firewall_rule_ids return objects
def post(self, request): firewall_ids = get_firewall_rules_id_from_request(request) objects = ThreatLogDetailEvent.objects.filter( firewall_rule__in=firewall_ids ) firewall_rule = objects.values_list( 'firewall_rule', 'firewall_rule__name').distinct() application = self._get_objs(objects, 'application') protocol = self._get_objs(objects, 'protocol') source_zone = self._get_objs(objects, 'source_zone') destination_zone = self._get_objs(objects, 'destination_zone') response = { "firewall_rule": firewall_rule, "application": application, "protocol": protocol, "source_zone": source_zone, "destination_zone": destination_zone } return Response(response)
def post(self, request, format=None): firewall_rule_ids = get_firewall_rules_id_from_request(request) kwargs = {'firewall_rule__in': firewall_rule_ids} objects = get_objects_with_date_filtered(request, TrafficLogDetailHourly, 'logged_datetime', **kwargs) src = objects.values_list('source_country').distinct() dst = objects.values_list('destination_country').distinct() countries = map(lambda x: x[0], (set(src) | set(dst))) response = [] for code in countries: try: country = pycountry.countries.get(alpha_2=code.upper()).name except AttributeError: country = '' response.append([country, code]) return Response(response)
def post(self, request): firewall_ids = get_firewall_rules_id_from_request(request) query = self.get_search_queries(request) country = request.data.get('country', '') kwargs = { 'firewall_rule__in': firewall_ids, } if country: kwargs['source_country'] = country objects = ThreatLogDetailEvent.objects.filter( **kwargs, **query ) page = self.paginate_queryset(objects.order_by('id')) if page is not None: serializer = self.serializer_class(page, many=True) return self.get_paginated_response(serializer.data) return Response()
def post(self, request): firewall_ids = get_firewall_rules_id_from_request(request) serializer = SourceDestinationIPSerializer(data=request.data) if not serializer.is_valid(): return Response(serializer.errors) source_address = serializer.data['source_address'] destination_address = serializer.data['destination_address'] objects = TrafficLogDetailHourly.objects.filter( firewall_rule__in=firewall_ids, source_address=source_address, destination_address=destination_address).order_by('id') page = self.paginate_queryset(objects) if page is not None: serializer = self.serializer_class(page, many=True) return self.get_paginated_response(serializer.data) return Response({})
def post(self, request): firewall_ids = get_firewall_rules_id_from_request(request) ip = request.data.get('ip') alias = request.data.get('alias') try: objects = TrafficMisNewDestinationIPDaily.objects.filter( firewall_rule__in=firewall_ids, destination_address=ip, ) self.set_alias(objects, ip, alias) objects = TrafficMisNewSourceIPDaily.objects.filter( firewall_rule__in=firewall_ids, source_address=ip, ) self.set_alias(objects, ip, alias) except Exception as e: return Response({"error": e}, status=HTTP_204_NO_CONTENT) return Response({"status": "saved"})
def get(self, request): country = request.data.get('country', '') firewall_ids = get_firewall_rules_id_from_request(request) kwargs = {'firewall_rule__in': firewall_ids} if country: kwargs['source_country'] = country objects = TrafficLogDetailHourly.objects.filter(**kwargs) aggregates = objects.aggregate( bytes_sent=Sum('sum_bytes_sent'), bytes_received=Sum('sum_bytes_received'), rows=Count('firewall_rule_id')) page = self.paginate_queryset(objects.order_by('id')) if page is not None: serializer = self.serializer_class(page, many=True) response = self.get_paginated_response(serializer.data) response.data['bytes_sent'] = aggregates['bytes_sent'] response.data['bytes_received'] = aggregates['bytes_received'] response.data['rows'] = aggregates['rows'] return response return Response({})
def post(self, request): firewall_ids = get_firewall_rules_id_from_request(request) queries = get_query_from_request( request, model_name='', datetime_field_name='received_datetime') objects = get_objects_from_query( queries, model=ThreatLogDetailEvent ) except_countries = request.data.get('except_countries', '') if except_countries: except_countries = except_countries.split(',') values = defaultdict(int) for obj in objects: name, code = get_country_name_and_code(obj.source_address) values[code] += obj.repeat_count return Response({ i: values[i] for i in values if i not in except_countries })
def post(self, request, id): firewall_rule_ids = get_firewall_rules_id_from_request(request) try: tt = TroubleTicketAnomaly.objects.get( id=id, firewall_rule__in=firewall_rule_ids ) if tt.reasons: reasons = [ self.REASONS_MAP[i.strip()] for i in tt.reasons.split(',') ] ip = tt.source_address objects = TrafficLogDetailHourly.objects.filter( source_address=ip, firewall_rule__in=firewall_rule_ids ) numeric_values = objects.values('source_address').aggregate( ###### NUMERIC COLS SUM ############## bytes_sent=Sum('sum_bytes_sent'), bytes_received=Sum('sum_bytes_received'), packets_sent=Sum('sum_packets_sent'), packets_received=Sum('sum_packets_received'), time_elapsed=Sum('sum_time_elapsed'), n_records=Count('application') # Count any arbitrary field ) n_records = numeric_values.pop('n_records') - 1 numeric = {} numeric_reasons = set() for reason in reasons: try: difference = numeric_values[reason] - getattr(tt, reason) numeric[reason] = format(difference/n_records, '.2f') numeric_reasons.add(reason) except KeyError as e: pass categorical = {} categorical_reasons = set(reasons) - numeric_reasons for reason in categorical_reasons: query = {reason: getattr(tt, reason)} count = objects.filter(**query).count() categorical[reason] = count return Response({ "numeric": numeric, "categorical": categorical }) query = {} max = objects.count() if max == 0: return Response({ "categorical": {}, "numeric": {} }) for reason in reasons: if reason not in {'logged_datetime'}: query[reason] = self.get_stats(objects, reason, max) stats = objects.aggregate(**query) categorical = {} numeric = {} for stat in stats: if stat in self._numeric_cols: numeric[stat] = format(stats[stat], '.2f') else: categorical[stat] = stats[stat] return Response({'reasons': { 'categorical': categorical, 'numeric': numeric }}) except TroubleTicketAnomaly.DoesNotExist as e: return Response({"error": 'yes'})
def get_objects(self, request, model): firewall_ids = get_firewall_rules_id_from_request(request) objects = model.objects.filter(firewall_rule__in=firewall_ids).values('source_address', 'destination_address').distinct() return objects