def get(self, request):
firewall_ids = get_firewall_rules_id_from_request(request)
application = request.data.get('application', '')
timestamp = request.data.get('timestamp')
kwargs = {
'firewall_rule__in': firewall_ids,
}
if application:
kwargs['application'] = application
if timestamp:
timestamp = int(timestamp)
tz_ktm = pytz.timezone('Asia/Kathmandu')
start_date = datetime.datetime.fromtimestamp(timestamp).astimezone(
tz_ktm)
end_date = start_date + datetime.timedelta(hours=1)
kwargs['received_datetime__gte'] = start_date
kwargs['received_datetime__lt'] = end_date
objects = ThreatLogDetail.objects.filter(**kwargs).order_by('id')
page = self.paginate_queryset(objects)
if page is not None:
serializer = self.serializer_class(page, many=True)
return self.get_paginated_response(serializer.data)
return Response({})
def post(self, request, format=None):
top_count = int(request.data.get('topcount', 5))
basis = request.data.get('basis', 'bytes')
country = request.data.get('country', '')
# Send all applications if top_count is 0
# Assuming applications < 10000
top_count = top_count if top_count else 10000
kwargs = get_model_kwargs(request)
if not kwargs.get('firewall_rule__in'):
firewall_rule_ids = get_firewall_rules_id_from_request(request)
kwargs['firewall_rule__in'] = firewall_rule_ids
if kwargs.get('application__in'):
del kwargs['application__in']
if country:
kwargs['source_country'] = country
objects = get_objects_with_date_filtered(
request, TrafficLogDetailHourly, 'logged_datetime',
**kwargs).values('logged_datetime', 'application').annotate(
bytes=Sum('sum_bytes_sent') + Sum('sum_bytes_received'),
packets=Sum('sum_packets_sent') + Sum('sum_packets_received'),
count=Count('firewall_rule_id'),
).values(
'bytes',
'packets',
'count',
date=F('logged_datetime'),
application_name=F('application'),
)
applications = []
max = 0
# To store the top applications
data = defaultdict(int)
# To store the data for each application
temp = defaultdict(list)
for obj in objects:
data[obj['application_name']] += obj['bytes']
timestamp = obj['date'].timestamp()
value = obj[basis]
# Data is sent in TBPC order
temp[obj['application_name']].append([timestamp, value])
max = value if value > max else max
# Get the top n applications, sorted by bytes
top_apps = sorted(data, key=data.get, reverse=True)[:top_count]
# Only send data of the top n applications
response = {i: temp[i] for i in top_apps}
return Response({'data': response, 'max': max})
def get(self, request):
firewall_ids = get_firewall_rules_id_from_request(request)
id = request.data.get('log_id', '')
ip = request.data.get('ip', '')
timestamp = request.data.get('timestamp', '')
hour = request.data.get('hour', '')
kwargs = {
'firewall_rule__in': firewall_ids,
}
if id:
kwargs['traffic_log__id'] = int(id)
if ip:
kwargs['source_address'] = ip
if hour:
kwargs['logged_datetime__hour'] = hour
if timestamp:
tz_ktm = pytz.timezone('Asia/Kathmandu')
start_date = datetime.datetime.fromtimestamp(
int(timestamp)).astimezone(tz_ktm)
end_date = start_date + datetime.timedelta(hours=1)
kwargs['logged_datetime__gte'] = start_date
kwargs['logged_datetime__lt'] = end_date
objects = TrafficLogDetailHourly.objects.filter(
**kwargs).order_by('-id')
page = self.paginate_queryset(objects)
if page is not None:
serializer = self.serializer_class(page, many=True)
return self.get_paginated_response(serializer.data)
def close_tt(request, id):
now = datetime.datetime.now()
try:
firewall_ids = get_firewall_rules_id_from_request(request)
trouble_ticket = TroubleTicketAnomaly.objects.get(
id=id, firewall_rule__in=firewall_ids)
except Exception as e:
return Response({
"traceback": str(traceback.format_exc()),
"exception": str(e)
}, status=HTTP_400_BAD_REQUEST)
description = request.data.get('description', '')
severity_level = request.data.get('severity_level')
is_anomaly = int(request.data.get('is_anomaly', 'true') == 'true')
user = get_user_from_token(request)
trouble_ticket.is_closed = True
trouble_ticket.is_anomaly = is_anomaly
trouble_ticket.severity_level = severity_level
trouble_ticket.assigned_to = user
trouble_ticket.description = description
trouble_ticket.verified_datetime = now
trouble_ticket.verified_by = user
trouble_ticket.save()
follow_up = TroubleTicketFollowUpAnomaly(
assigned_by=user,
assigned_to=user,
description=description,
trouble_ticket=trouble_ticket,
follow_up_datetime=now
)
follow_up.save()
return Response({'ok': 'tt closed'})
def get(self, request):
firewall_ids = get_firewall_rules_id_from_request(request)
application = request.data.get('application', '')
timestamp = int(request.data.get('timestamp'))
country = request.data.get('country', '')
tz_ktm = pytz.timezone('Asia/Kathmandu')
start_date = datetime.datetime.fromtimestamp(timestamp).astimezone(
tz_ktm)
end_date = start_date + datetime.timedelta(hours=1)
kwargs = {
'firewall_rule__in': firewall_ids,
'application': application,
'logged_datetime__gte': start_date,
'logged_datetime__lt': end_date
}
if country:
kwargs['source_country'] = country
objects = TrafficLogDetailHourly.objects.filter(
**kwargs).order_by('id')
page = self.paginate_queryset(objects)
if page is not None:
serializer = self.serializer_class(page, many=True)
return self.get_paginated_response(serializer.data)
return Response({})
def post(self, request):
firewall_ids = get_firewall_rules_id_from_request(request)
ip = request.data.get('ip', '')
alias = request.data.get('alias', '')
source_kwargs = {}
destination_kwargs = {}
if ip:
source_kwargs['source_address__icontains'] = ip
destination_kwargs['destination_address__icontains'] = ip
if alias:
source_kwargs['alias__icontains'] = alias
destination_kwargs['alias__icontains'] = alias
source_objects = TrafficMisNewSourceIPDaily.objects.filter(
firewall_rule__in=firewall_ids,
**source_kwargs).annotate(address=F('source_address')).values_list(
'address', 'alias').order_by('alias')
destination_objects = (TrafficMisNewDestinationIPDaily.objects.filter(
firewall_rule__in=firewall_ids, **destination_kwargs).annotate(
address=F('destination_address')).values_list(
'address', 'alias').order_by('alias'))
objects = []
for i in (set(source_objects) | set(destination_objects)):
objects.append({'address': i[0], 'alias': i[1]})
page = self.paginate_queryset(objects)
if page is not None:
serializer = self.serializer_class(page, many=True)
return self.get_paginated_response(serializer.data)
def get(self, request):
firewall_ids = get_firewall_rules_id_from_request(request)
objects = TrafficMisRequestFromBlacklistedIPDaily.objects.filter(
firewall_rule__in=firewall_ids
)
page = self.paginate_queryset(objects.order_by('-logged_datetime'))
if page is not None:
serializer = self.serializer_class(page, many=True)
return self.get_paginated_response(serializer.data)
def get_filtered_objects(self, request, **kwargs):
firewall_rule_ids = get_firewall_rules_id_from_request(request)
query = self.get_search_queries(request)
objects = ThreatLogDetailEvent.objects.filter(
firewall_rule__in=firewall_rule_ids,
**kwargs,
**query,
)
return objects.order_by('id')
def post(self, request):
firewall_ids = get_firewall_rules_id_from_request(request)
objects = ThreatLogDetailEvent.objects.filter(
firewall_rule__in=firewall_ids
).distinct('source_address').values('source_address')
countries = set()
for obj in objects:
countries.add(get_country_name_and_code(obj['source_address']))
return Response(countries)
def get_filtered_objects(self, request, **kwargs):
firewall_rule_ids = get_firewall_rules_id_from_request(request)
query = self.get_search_queries(request)
objects = TroubleTicketAnomaly.objects.filter(
firewall_rule__in=firewall_rule_ids,
**kwargs,
**query,
)
return objects.order_by('-logged_datetime')
def post(self, request, format=None):
top_count = int(request.data.get('topcount', 5))
# Send all applications if top_count is 0
# Assuming applications < 10000
top_count = top_count if top_count else 10000
queries = get_query_from_request(
request, model_name='', datetime_field_name='received_datetime')
objects = get_objects_from_query(queries, model=ThreatLogDetailEvent)
firewall_rule_ids = get_firewall_rules_id_from_request(request)
kwargs = {
'firewall_rule__in': firewall_rule_ids,
}
country = request.data.get('country', '')
# if country:
# kwargs['source_country'] = country.title()
applications = objects.values('application').annotate(
sum=Sum('repeat_count')
).order_by('-sum').values('application')[:top_count]
objects = objects.filter(
application__in=applications, **kwargs
).values(
'received_datetime',
'application'
).annotate(
repeat_count=Sum('repeat_count'),
count=Count('firewall_rule_id'),
).values(
'count',
'repeat_count',
date=F('received_datetime'),
application_name=F('application'),
).order_by('received_datetime')
applications = []
max = 0
data = defaultdict(list)
for obj in objects:
timestamp = obj['date'].timestamp()
repeat_count = obj['repeat_count']
if max < repeat_count:
max = repeat_count
data[obj['application_name']].append([
timestamp,
repeat_count
])
return Response({
'data': data,
'max': max
})
def get_response(self, request, model):
firewall_ids = get_firewall_rules_id_from_request(request)
try:
objects = model.objects.filter(
firewall_rule__in=firewall_ids).latest('id')
date = get_date_from_filename(
objects.log) - datetime.timedelta(days=1)
return Response({'date': date.date()})
except Exception as e:
return Response({'date': datetime.date.today()})
def get(self, request):
firewall_ids = get_firewall_rules_id_from_request(request)
objects = ProcessedThreatLogDetail.objects.filter(
firewall_rule__in=firewall_ids).values(
'log', 'processed_datetime').annotate(rows=Sum('rows'),
size=F('rows') *
self.SIZE_PER_LOG)
page = self.paginate_queryset(objects.order_by('-log'))
if page is not None:
serializer = self.serializer_class(page, many=True)
return self.get_paginated_response(serializer.data)
def _get_items(self, request, model, field_name):
firewall_ids = get_firewall_rules_id_from_request(request)
date = str_to_date(request.data.get('date', ''))
if not date:
date = model.objects.latest(
'logged_datetime'
).logged_datetime
objects = model.objects.filter(
firewall_rule__in=firewall_ids,
logged_datetime=date
).values_list(field_name, 'firewall_rule__name')
return objects
def post(self, request):
firewall_ids = get_firewall_rules_id_from_request(request)
ip = request.data.get('ip')
objects = list(
TrafficMisRequestFromBlacklistedIPDaily.objects.filter(
firewall_rule__in=firewall_ids, source_address=ip))
objects += list(
TrafficMisResponseToBlacklistedIPDaily.objects.filter(
firewall_rule__in=firewall_ids, destination_address=ip))
page = self.paginate_queryset(objects)
if page is not None:
serializer = self.serializer_class(page, many=True)
return self.get_paginated_response(serializer.data)
def _get_items(self, request, model):
firewall_ids = get_firewall_rules_id_from_request(request)
objects = model.objects.filter(
firewall_rule__in=firewall_ids
).values('logged_datetime').annotate(
count=Count('firewall_rule'),
date=F('logged_datetime')
).values('date', 'count').order_by('logged_datetime')
items = []
for obj in objects:
timestamp = obj['date'].strftime("%s")
count = obj['count']
items.append([timestamp, count])
return items
def post(self, request, format=None):
basis = request.data.get('basis', 'bytes')
except_countries = request.data.get('except_countries', '')
kwargs = get_model_kwargs(request)
if not kwargs.get('firewall_rule__in'):
firewall_rule_ids = get_firewall_rules_id_from_request(request)
kwargs['firewall_rule__in'] = firewall_rule_ids
objects = get_objects_with_date_filtered(
request, TrafficLogDetailHourly, 'logged_datetime',
**kwargs).values('source_country').annotate(
bytes=Sum('sum_bytes_sent') + Sum('sum_bytes_received'),
packets=Sum('sum_packets_sent') + Sum('sum_packets_received'),
count=Count('firewall_rule_id')).values(
'source_country', basis)
return Response(objects)
def get_filtered_objects(self, request, return_firewall_rules=False, **kwargs):
firewall_rule_ids = get_firewall_rules_id_from_request(request)
query = self.get_search_queries(request)
alias = request.data.get('alias', None)
aliased_ips = self._get_alias_ips(alias)
objects = TrafficRule.objects.filter(
firewall_rule__in=firewall_rule_ids,
**kwargs,
**query,
)
if aliased_ips is not None:
aliased_ips = [i[0] for i in aliased_ips]
objects = objects.filter(
Q(source_address__in=aliased_ips) | Q(
destination_address__in=aliased_ips)
)
if return_firewall_rules:
return objects, firewall_rule_ids
return objects
def post(self, request):
firewall_ids = get_firewall_rules_id_from_request(request)
objects = ThreatLogDetailEvent.objects.filter(
firewall_rule__in=firewall_ids
)
firewall_rule = objects.values_list(
'firewall_rule', 'firewall_rule__name').distinct()
application = self._get_objs(objects, 'application')
protocol = self._get_objs(objects, 'protocol')
source_zone = self._get_objs(objects, 'source_zone')
destination_zone = self._get_objs(objects, 'destination_zone')
response = {
"firewall_rule": firewall_rule,
"application": application,
"protocol": protocol,
"source_zone": source_zone,
"destination_zone": destination_zone
}
return Response(response)
def post(self, request, format=None):
firewall_rule_ids = get_firewall_rules_id_from_request(request)
kwargs = {'firewall_rule__in': firewall_rule_ids}
objects = get_objects_with_date_filtered(request,
TrafficLogDetailHourly,
'logged_datetime', **kwargs)
src = objects.values_list('source_country').distinct()
dst = objects.values_list('destination_country').distinct()
countries = map(lambda x: x[0], (set(src) | set(dst)))
response = []
for code in countries:
try:
country = pycountry.countries.get(alpha_2=code.upper()).name
except AttributeError:
country = ''
response.append([country, code])
return Response(response)
def post(self, request):
firewall_ids = get_firewall_rules_id_from_request(request)
query = self.get_search_queries(request)
country = request.data.get('country', '')
kwargs = {
'firewall_rule__in': firewall_ids,
}
if country:
kwargs['source_country'] = country
objects = ThreatLogDetailEvent.objects.filter(
**kwargs,
**query
)
page = self.paginate_queryset(objects.order_by('id'))
if page is not None:
serializer = self.serializer_class(page, many=True)
return self.get_paginated_response(serializer.data)
return Response()
def post(self, request):
firewall_ids = get_firewall_rules_id_from_request(request)
serializer = SourceDestinationIPSerializer(data=request.data)
if not serializer.is_valid():
return Response(serializer.errors)
source_address = serializer.data['source_address']
destination_address = serializer.data['destination_address']
objects = TrafficLogDetailHourly.objects.filter(
firewall_rule__in=firewall_ids,
source_address=source_address,
destination_address=destination_address).order_by('id')
page = self.paginate_queryset(objects)
if page is not None:
serializer = self.serializer_class(page, many=True)
return self.get_paginated_response(serializer.data)
return Response({})
def post(self, request):
firewall_ids = get_firewall_rules_id_from_request(request)
ip = request.data.get('ip')
alias = request.data.get('alias')
try:
objects = TrafficMisNewDestinationIPDaily.objects.filter(
firewall_rule__in=firewall_ids,
destination_address=ip,
)
self.set_alias(objects, ip, alias)
objects = TrafficMisNewSourceIPDaily.objects.filter(
firewall_rule__in=firewall_ids,
source_address=ip,
)
self.set_alias(objects, ip, alias)
except Exception as e:
return Response({"error": e}, status=HTTP_204_NO_CONTENT)
return Response({"status": "saved"})
def get(self, request):
country = request.data.get('country', '')
firewall_ids = get_firewall_rules_id_from_request(request)
kwargs = {'firewall_rule__in': firewall_ids}
if country:
kwargs['source_country'] = country
objects = TrafficLogDetailHourly.objects.filter(**kwargs)
aggregates = objects.aggregate(
bytes_sent=Sum('sum_bytes_sent'),
bytes_received=Sum('sum_bytes_received'),
rows=Count('firewall_rule_id'))
page = self.paginate_queryset(objects.order_by('id'))
if page is not None:
serializer = self.serializer_class(page, many=True)
response = self.get_paginated_response(serializer.data)
response.data['bytes_sent'] = aggregates['bytes_sent']
response.data['bytes_received'] = aggregates['bytes_received']
response.data['rows'] = aggregates['rows']
return response
return Response({})
def post(self, request):
firewall_ids = get_firewall_rules_id_from_request(request)
queries = get_query_from_request(
request, model_name='', datetime_field_name='received_datetime')
objects = get_objects_from_query(
queries,
model=ThreatLogDetailEvent
)
except_countries = request.data.get('except_countries', '')
if except_countries:
except_countries = except_countries.split(',')
values = defaultdict(int)
for obj in objects:
name, code = get_country_name_and_code(obj.source_address)
values[code] += obj.repeat_count
return Response({
i: values[i] for i in values if i not in except_countries
})
def post(self, request, id):
firewall_rule_ids = get_firewall_rules_id_from_request(request)
try:
tt = TroubleTicketAnomaly.objects.get(
id=id,
firewall_rule__in=firewall_rule_ids
)
if tt.reasons:
reasons = [
self.REASONS_MAP[i.strip()]
for i in tt.reasons.split(',')
]
ip = tt.source_address
objects = TrafficLogDetailHourly.objects.filter(
source_address=ip,
firewall_rule__in=firewall_rule_ids
)
numeric_values = objects.values('source_address').aggregate(
###### NUMERIC COLS SUM ##############
bytes_sent=Sum('sum_bytes_sent'),
bytes_received=Sum('sum_bytes_received'),
packets_sent=Sum('sum_packets_sent'),
packets_received=Sum('sum_packets_received'),
time_elapsed=Sum('sum_time_elapsed'),
n_records=Count('application') # Count any arbitrary field
)
n_records = numeric_values.pop('n_records') - 1
numeric = {}
numeric_reasons = set()
for reason in reasons:
try:
difference = numeric_values[reason] - getattr(tt, reason)
numeric[reason] = format(difference/n_records, '.2f')
numeric_reasons.add(reason)
except KeyError as e:
pass
categorical = {}
categorical_reasons = set(reasons) - numeric_reasons
for reason in categorical_reasons:
query = {reason: getattr(tt, reason)}
count = objects.filter(**query).count()
categorical[reason] = count
return Response({
"numeric": numeric,
"categorical": categorical
})
query = {}
max = objects.count()
if max == 0:
return Response({
"categorical": {},
"numeric": {}
})
for reason in reasons:
if reason not in {'logged_datetime'}:
query[reason] = self.get_stats(objects, reason, max)
stats = objects.aggregate(**query)
categorical = {}
numeric = {}
for stat in stats:
if stat in self._numeric_cols:
numeric[stat] = format(stats[stat], '.2f')
else:
categorical[stat] = stats[stat]
return Response({'reasons': {
'categorical': categorical,
'numeric': numeric
}})
except TroubleTicketAnomaly.DoesNotExist as e:
return Response({"error": 'yes'})
def get_objects(self, request, model):
firewall_ids = get_firewall_rules_id_from_request(request)
objects = model.objects.filter(firewall_rule__in=firewall_ids).values('source_address', 'destination_address').distinct()
return objects
