def forward_incident(request):
try:
with transaction.atomic():
incident = Incident.objects.get(id=request.POST['id'])
user = User.objects.get(id=request.POST['user_id'])
ScheduledNotification.remove_all_for_incident(incident)
NotificationHelper.notify_user_about_incident(incident, user)
event_log_message = "%s changed assignee of incident : %s to %s" % (
request.user.username, incident.incident_key, user.username)
event_log = EventLog()
event_log.user = request.user
event_log.action = "forward"
event_log.incident_key = incident
event_log.service_key = incident.service_key
event_log.data = event_log_message
event_log.occurred_at = timezone.now()
event_log.save()
except Incident.DoesNotExist:
messages.error(request, 'Incident not found')
return HttpResponseRedirect(request.POST['url'])
except User.DoesNotExist:
messages.error(request, 'Incident not found')
return HttpResponseRedirect(request.POST['url'])
except ValidationError as e:
messages.error(request, e.messages)
return HttpResponseRedirect(request.POST['url'])
def silence(request, incident_id):
try:
incident = Incident.objects.get(id=incident_id)
silence_for = request.POST.get('silence_for')
url = request.POST.get("url")
if IncidentSilenced.objects.filter(incident=incident).count() < 1:
silenced_incident = IncidentSilenced()
silenced_incident.incident = incident
silenced_incident.silenced_until = timezone.now(
) + timezone.timedelta(hours=int(silence_for))
silenced_incident.silenced = True
silenced_incident.save()
event_log_message = "%s silenced incident %s for %s hours" % (
request.user.username, incident.incident_key, silence_for)
event_log = EventLog()
event_log.incident_key = incident
event_log.action = 'silence_incident'
event_log.user = request.user
event_log.service_key = incident.service_key
event_log.data = event_log_message
event_log.occurred_at = timezone.now()
event_log.save()
ScheduledNotification.remove_all_for_incident(incident)
incident.event_type = Incident.ACKNOWLEDGE
incident.save()
unsilence_incident.apply_async(
(incident_id,), eta=silenced_incident.silenced_until)
return HttpResponseRedirect(url)
except Service.DoesNotExist:
raise Http404
def silence(request, incident_id):
silence_for = request.POST.get('silence_for', 0)
url = reverse('IncidentsListView')
request_url = request.POST.get('url', url)
try:
incident = Incident.objects.get(id=int(incident_id))
except Incident.DoesNotExist:
messages.error(request, 'Incident not found')
return HttpResponseRedirect(request_url)
if IncidentSilenced.objects.filter(incident=incident).count() < 1:
silenced_incident = IncidentSilenced.objects.create(
incident=incident,
silenced_until=timezone.now() +
timezone.timedelta(hours=int(silence_for)),
silenced=True)
event_log_message = f"{request.user.username} silenced " \
f"incident {incident.incident_key} for {silence_for} hours"
EventLog.objects.create(incident_key=incident,
action='silence_incident',
user=request.user,
service_key=incident.service_key,
data=event_log_message,
occurred_at=timezone.now())
ScheduledNotification.remove_all_for_incident(incident)
incident.event_type = Incident.ACKNOWLEDGE
incident.save()
unsilence_incident.apply_async((incident_id, ),
eta=silenced_incident.silenced_until)
messages.success(request, event_log_message)
return HttpResponseRedirect(request_url)
def _update_type(user, ids, event_type):
for incident_id in ids:
with transaction.atomic():
incident = Incident.objects.get(id=int(incident_id))
unesc = False
if incident.service_to_escalate_to is not None:
incident.service_to_escalate_to = None
unesc = True
logmessage = EventLog()
logmessage.service_key = incident.service_key
logmessage.user = user
logmessage.action = event_type
logmessage.data = "%s changed %s from %s to %s" % (
user.username,
incident.incident_key,
incident.event_type,
event_type)
if unesc:
logmessage.data += ", unescalated"
logmessage.occurred_at = timezone.now()
incident.event_type = event_type
incident.occurred_at = timezone.now()
incident.save()
logmessage.incident_key = incident
logmessage.save()
if incident.event_type == Incident.RESOLVE or incident.event_type == Incident.ACKNOWLEDGE:
ScheduledNotification.remove_all_for_incident(incident)
def escalate(self, request, *args, **kwargs):
#get arguments
try:
token = Token.objects.get(key=request.data["service_key"])
serviceToken = ServiceTokens.objects.get(token_id=token)
service = serviceToken.service_id
except ServiceTokens.DoesNotExist:
return Response({"Service key does not exist"}, status=status.HTTP_404_NOT_FOUND)
except Token.DoesNotExist:
return Response({"No service key"}, status=status.HTTP_403_FORBIDDEN)
try:
token2 = Token.objects.get(key=request.data["service_key_to_escalate_to"])
serviceToken2 = ServiceTokens.objects.get(token_id=token2)
service2 = serviceToken2.service_id
except ServiceTokens.DoesNotExist:
return Response({"Service to escalate to key does not exist"}, status=status.HTTP_404_NOT_FOUND)
except Token.DoesNotExist:
return Response({"No service to escalate to key"}, status=status.HTTP_403_FORBIDDEN)
#modify incident
with transaction.atomic():
try:
# get service_to_escalate to and modify incident object
incident = Incident.objects.get(incident_key=request.data["incident_key"],service_key=service)
incident.service_to_escalate_to = service2
incident.event_type = "escalated"
if request.data["incident_details"]:
incident.details = request.data["incident_details"]
# incident.description = "[escalated] " + incident.description
incident.save()
event_log_message = "%s escalated to service escalation policy : %s to %s" % (request.user.username, incident.incident_key, service2.name)
event_log = EventLog()
event_log.user = request.user
event_log.action = "escalate"
event_log.incident_key = incident
event_log.service_key = incident.service_key
event_log.data = event_log_message
event_log.occurred_at = timezone.now()
event_log.save()
except (Incident.DoesNotExist, KeyError):
return Response({"Incident does not exist"}, status=status.HTTP_404_NOT_FOUND)
except (Service.DoesNotExist, KeyError):
return Response({"Service does not exist"}, status=status.HTTP_400_BAD_REQUEST)
# remove all planned notifs
ScheduledNotification.remove_all_for_incident(incident)
# notify anew, this time notify_incident will detect the service_to_escalate to and notify its escalation rule
NotificationHelper.notify_incident(incident)
headers = self.get_success_headers(request.POST)
return Response({"Incident successfully escalated to service " + service2.name + " escalation policy"},status=status.HTTP_200_OK,headers=headers)
def generate_notifications_for_user(incident, user, delay=None, preparedmsg = None):
now = timezone.make_aware(datetime.now(), timezone.get_current_timezone())
current_time = now
notifications = []
methods = user.notification_methods.order_by('position').all()
method_index = 0
for method in methods:
if delay is None:
notification_time = incident.service_key.retry * method_index + incident.service_key.escalate_after
else:
notification_time = method_index * delay
notify_at = current_time + timedelta(minutes=notification_time)
notification = ScheduledNotification()
notification.incident = incident
notification.user_to_notify = user
notification.notifier = method.method
notification.send_at = notify_at
if preparedmsg is None:
uri = settings.BASE_URL + "/incidents/details/" + str(incident.id)
notification.message = "A Service is experiencing a problem: " + incident.incident_key + " " + incident.description + ". Handle at: " + uri
else:
notification.message = preparedmsg
notifications.append(notification)
if notification.incident:
print("[%s] Notify %s at %s with method: %s" % (notification.incident.event_type, user.username, notify_at, notification.notifier))
method_index += 1
# todo: error handling
return notifications
def _update_type(user, ids, event_type):
for incident_id in ids:
incident = Incident.objects.get(id=int(incident_id))
log_message_data = f"{user.username} changed {incident.incident_key} from {incident.event_type} to {event_type}"
if incident.service_to_escalate_to is not None:
incident.service_to_escalate_to = None
log_message_data += ", unescalated"
EventLog.objects.create(service_key=incident.service_key,
user=user,
action=event_type,
data=log_message_data,
occurred_at=timezone.now(),
incident_key=incident)
incident.event_type = event_type
incident.occurred_at = timezone.now()
incident.save()
if incident.event_type == Incident.RESOLVE or incident.event_type == Incident.ACKNOWLEDGE:
ScheduledNotification.remove_all_for_incident(incident)
def generate_notifications_for_incident(incident):
now = timezone.make_aware(datetime.now(), timezone.get_current_timezone())
duty_officers = []
# if incident has been escalated, notify according to the escalated service's escalation rule
if hasattr(incident, 'service_to_escalate_to') and incident.service_to_escalate_to is not None:
print("escalation rule in place to " + incident.service_to_escalate_to.name)
duty_officers = get_escalation_for_service(incident.service_to_escalate_to)
with transaction.atomic():
incident.description = "[escalated] " + incident.description
incident.save()
else:
duty_officers = get_escalation_for_service(incident.service_key)
current_time = now
notifications = []
group_index = {}
user_method_index = {}
for officer_index, duty_officer in enumerate(duty_officers):
if incident.event_type == Incident.RESOLVE and not duty_officer.profile.send_resolve_enabled:
print("Skipping notification for %s because type is RESOLVE and user %s has send_resolve_enabled OFF" % (incident.incident_key, duty_officer.username))
continue
index = 0
if hasattr(duty_officer ,'came_from_group' ):
if not duty_officer.came_from_group in group_index:
group_index[duty_officer.came_from_group] = officer_index
index = group_index[duty_officer.came_from_group]
else:
index = officer_index
escalation_time = incident.service_key.escalate_after * (index + 1)
escalate_at = current_time + timedelta(minutes=escalation_time)
user_method_index[duty_officer.username] = 0
methods = duty_officer.notification_methods.order_by('position').all()
for method in methods:
notification_time = incident.service_key.retry * user_method_index[duty_officer.username] + incident.service_key.escalate_after * index
notify_at = current_time + timedelta(minutes=notification_time)
if notify_at < escalate_at:
notification = ScheduledNotification()
notification.incident = incident
notification.user_to_notify = duty_officer
notification.notifier = method.method
notification.send_at = notify_at
uri = settings.BASE_URL + "/incidents/details/" + str(incident.id)
notification.message = incident.description + ". Handle at: " + uri + " Details: " + incident.details
notifications.append(notification)
print("[%s] Notify %s about %s at %s with method: %s" % (
notification.incident.event_type, duty_officer.username,
notification.incident.incident_key, notify_at, notification.notifier))
else:
break
user_method_index[duty_officer.username] += 1
# todo: error handling
return notifications
def forward_incident(request):
url = reverse('IncidentsListView')
request_url = request.POST.get('url', url)
try:
incident = Incident.objects.get(id=request.POST.get('id'))
user = User.objects.get(id=request.POST.get('user_id'))
ScheduledNotification.remove_all_for_incident(incident)
NotificationHelper.notify_user_about_incident(incident, user)
event_log_message = f"{request.user.username} changed assignee of " \
f"incident : {incident.incident_key} to {user.username}"
EventLog.objects.create(user=request.user,
action="forward",
incident_key=incident,
service_key=incident.service_key,
data=event_log_message,
occurred_at=timezone.now())
except Incident.DoesNotExist:
messages.error(request, 'Incident not found')
return HttpResponseRedirect(request_url)
except User.DoesNotExist:
messages.error(request, 'Incident not found')
return HttpResponseRedirect(request_url)
return HttpResponseRedirect(request_url)
def create(self, request, *args, **kwargs):
try:
token = Token.objects.get(key=request.data["service_key"])
serviceToken = ServiceTokens.objects.get(token_id=token)
service = serviceToken.service_id
except ServiceTokens.DoesNotExist:
return Response({"Service key does not exist"}, status=status.HTTP_404_NOT_FOUND)
except Token.DoesNotExist:
return Response({"No service key"}, status=status.HTTP_403_FORBIDDEN)
with transaction.atomic():
try:
esc = False
incident = Incident.objects.get(incident_key=request.data["incident_key"],service_key=service)
print("Received %s for %s on service %s" % (request.data['event_type'],request.data['incident_key'],serviceToken.name))
#check if type is ACK or resolve and if there's an escalation to a different escalation policy, remove it
if request.data['event_type'] == Incident.ACKNOWLEDGE or request.data['event_type'] == Incident.RESOLVE:
print("ACK or Resolve, removing specific escalation")
esc = True
incident.service_to_escalate_to = None
incident.save()
# check if incident is resolved and refuse to ACK
if not (incident.event_type == Incident.RESOLVE and request.data['event_type'] == Incident.ACKNOWLEDGE):
event_log_message = "%s api key changed %s from %s to %s" % (
serviceToken.name, incident.incident_key,
incident.event_type, request.data['event_type'])
if esc:
event_log_message += ", unescalated"
else:
response = {}
response["status"] = "failure"
response["message"] = "Can\'t ACK a resolved incident!"
return Response(response, status=status.HTTP_400_BAD_REQUEST)
except (Incident.DoesNotExist, KeyError):
incident = Incident()
try:
incident.incident_key = request.data["incident_key"]
except KeyError:
if request.data["event_type"] == Incident.TRIGGER:
incident.incident_key = base64.urlsafe_b64encode(
uuid.uuid1().bytes).replace(
'=',
'')
else:
response = {}
response["status"] = "failure"
response["message"] = "Mandatory parameter missing"
return Response(
response,
status=status.HTTP_400_BAD_REQUEST)
incident.service_key = service
event_log_message = "%s api key created %s with status %s" % (
serviceToken.name, incident.incident_key, request.data['event_type'])
if self.is_relevant(incident, request.data['event_type']):
event_log = EventLog()
# Anonymous user for testing
if request.user.is_anonymous:
user = None
else:
user = request.user
event_log.user = user
event_log.service_key = incident.service_key
event_log.data = event_log_message
event_log.occurred_at = timezone.now()
incident.event_type = request.data["event_type"]
incident.description = request.data["description"][:100]
incident.details = request.data.get("details", "")
incident.occurred_at = timezone.now()
try:
incident.full_clean()
except ValidationError as e:
return Response(
{'errors': e.messages},
status=status.HTTP_400_BAD_REQUEST)
incident.save()
event_log.incident_key = incident
event_log.action = incident.event_type
event_log.save()
servicesilenced = ServiceSilenced.objects.filter(
service=service).count() > 0
if incident.event_type == Incident.TRIGGER and not servicesilenced:
NotificationHelper.notify_incident(incident)
if incident.event_type == Incident.RESOLVE or incident.event_type == Incident.ACKNOWLEDGE:
ScheduledNotification.remove_all_for_incident(incident)
if incident.event_type == Incident.RESOLVE and service.send_resolve_enabled:
NotificationHelper.notify_incident(incident)
headers = self.get_success_headers(request.POST)
response = {}
response["status"] = "success"
response["message"] = "Event processed"
response["incident_key"] = incident.incident_key
return Response(response,status=status.HTTP_201_CREATED,headers=headers)
def create(self, request, *args, **kwargs):
try:
assert request.data.get("event_type")
assert request.data.get("incident_key")
assert request.data.get("service_key")
except AssertionError:
response = {
"status": "failure",
"message": "Mandatory parameter missing"
}
return Response(response, status=status.HTTP_400_BAD_REQUEST)
try:
token = Token.objects.get(key=request.data.get("service_key"))
service_token = ServiceTokens.objects.get(token_id=token)
service = service_token.service_id
except Token.DoesNotExist:
return Response({"No service key"},
status=status.HTTP_403_FORBIDDEN)
except ServiceTokens.DoesNotExist:
return Response({"Service key does not exist"},
status=status.HTTP_404_NOT_FOUND)
esc = False
incidents = Incident.objects.filter(
incident_key=request.data.get("incident_key"), service_key=service)
if incidents:
incident = incidents[0]
print(
f"Received {request.data.get('event_type')} for"
f" {request.data.get('incident_key')} on service {service_token.name}"
)
# check if type is ACK or resolve and if there's an
# escalation to a different escalation policy, remove it
if request.data.get('event_type') in [
Incident.ACKNOWLEDGE, Incident.RESOLVE
]:
print("ACK or Resolve, removing specific escalation")
esc = True
incident.service_to_escalate_to = None
incident.save()
# check if incident is resolved and refuse to ACK
event_log_message = f"{service_token.name} api key changed {incident.incident_key} " \
f"from {incident.event_type} to {request.data.get('event_type')}"
if incident.event_type not in [
Incident.RESOLVE, Incident.ACKNOWLEDGE
] and esc:
event_log_message += ", unescalated"
else:
event_type = request.data.get("event_type")
incident_key = str(uuid.uuid1()) if (
event_type
== Incident.TRIGGER) else request.data.get("incident_key")
incident_service_key = service
incident = Incident.objects.create(
event_type=event_type,
incident_key=incident_key,
service_key=incident_service_key,
description=request.data.get("description", " "),
details=request.data.get("details", " "),
occurred_at=timezone.now())
event_log_message = f"{service_token.name} api key created " \
f"{incident.incident_key} with status {request.data.get('event_type')}"
if self.is_relevant(incident, request.data.get('event_type')):
user = None if request.user.is_anonymous else request.user
event_log = EventLog(user=user,
service_key=incident.service_key,
data=event_log_message,
occurred_at=timezone.now())
event_log.incident_key = incident
event_log.action = incident.event_type
event_log.save()
service_silenced = ServiceSilenced.objects.filter(
service=service).count() > 0
if incident.event_type == Incident.TRIGGER and not service_silenced:
NotificationHelper.notify_incident(incident)
if incident.event_type == Incident.RESOLVE or incident.event_type == Incident.ACKNOWLEDGE:
ScheduledNotification.remove_all_for_incident(incident)
if incident.event_type == Incident.RESOLVE and service.send_resolve_enabled:
NotificationHelper.notify_incident(incident)
response = {
"status": "success",
"message": "Event processed",
"incident_key": incident.incident_key
}
return Response(response, status=status.HTTP_201_CREATED)
def escalate(self, request, *args, **kwargs):
"""
escalate an incident to another service's escalation rule; persists until ACK
"""
try:
token = Token.objects.get(key=request.data.get("service_key"))
service_token = ServiceTokens.objects.get(token_id=token)
service = service_token.service_id
except ServiceTokens.DoesNotExist:
return Response({"Service key does not exist"},
status=status.HTTP_404_NOT_FOUND)
except Token.DoesNotExist:
return Response({"No service key"},
status=status.HTTP_403_FORBIDDEN)
try:
token2 = Token.objects.get(
key=request.data.get("service_key_to_escalate_to"))
service_token2 = ServiceTokens.objects.get(token_id=token2)
service2 = service_token2.service_id
except ServiceTokens.DoesNotExist:
return Response({"Service to escalate to key does not exist"},
status=status.HTTP_404_NOT_FOUND)
except Token.DoesNotExist:
return Response({"No service to escalate to key"},
status=status.HTTP_403_FORBIDDEN)
try:
# get service_to_escalate to and modify incident object
incident = Incident.objects.get(
incident_key=request.data.get("incident_key"),
service_key=service)
except (Incident.DoesNotExist, KeyError):
return Response({"Incident does not exist"},
status=status.HTTP_404_NOT_FOUND)
incident.service_to_escalate_to = service2
incident.event_type = "escalated"
if request.data.get("IncidentDetailView"):
incident.details = request.data.get("IncidentDetailView")
incident.save()
username = '******' if request.user.is_anonymous else request.user.username
event_log_message = f"{username} escalated to " \
f"service escalation policy : {incident.incident_key} to {service2.name}"
EventLog.objects.create(user=request.user,
action="escalate",
incident_key=incident,
service_key=incident.service_key,
data=event_log_message,
occurred_at=timezone.now())
# remove all planned notifs
ScheduledNotification.remove_all_for_incident(incident)
# notify anew, this time notify_incident will detect the service_to_escalate to and notify its escalation rule
NotificationHelper.notify_incident(incident)
headers = self.get_success_headers(request.POST)
return Response(
{
f"Incident successfully escalated to service {service2.name} escalation policy"
},
status=status.HTTP_200_OK,
headers=headers)
