def incident_report_reminders(db_session=None):
"""Sends report reminders to incident commanders for active incidents."""
incidents = incident_service.get_all_by_status(
db_session=db_session, status=IncidentStatus.active
)
for incident in incidents:
for report_type in ReportTypes:
try:
remind_after = incident.created_at
if report_type == ReportTypes.tactical_report:
notification_hour = incident.incident_priority.tactical_report_reminder
if incident.last_tactical_report:
remind_after = incident.last_tactical_report.created_at
elif report_type == ReportTypes.executive_report:
notification_hour = incident.incident_priority.executive_report_reminder
if incident.last_executive_report:
remind_after = incident.last_executive_report.created_at
now = datetime.utcnow() - remind_after
# we calculate the number of hours and seconds since last report was sent
hours, seconds = divmod((now.days * 86400) + now.seconds, 3600)
q, r = divmod(hours, notification_hour)
if q >= 1 and r == 0: # it's time to send the reminder
send_incident_report_reminder(incident, report_type)
except Exception as e:
# we shouldn't fail to send all reminders when one fails
sentry_sdk.capture_exception(e)
def status_report_reminder(db_session=None):
"""Sends status report reminders to active incident commanders."""
incidents = get_all_by_status(db_session=db_session,
status=IncidentStatus.active)
for incident in incidents:
try:
notification_hour = incident.incident_priority.status_reminder
if incident.last_status_report:
remind_after = incident.last_status_report.created_at
else:
remind_after = incident.created_at
now = datetime.utcnow() - remind_after
# we calculate the number of hours and seconds since last CAN was sent
hours, seconds = divmod((now.days * 86400) + now.seconds, 3600)
q, r = divmod(hours, notification_hour)
if q >= 1 and r == 0: # it's time to send the reminder
send_incident_status_report_reminder(incident)
except Exception as e:
# we shouldn't fail to update all incidents when one fails
sentry_sdk.capture_exception(e)
def sync_tasks(db_session, incidents, notify: bool = False):
"""Syncs tasks and sends update notifications to incident channels."""
drive_task_plugin = plugins.get(INCIDENT_PLUGIN_TASK_SLUG)
for incident in incidents:
for doc_type in [
INCIDENT_RESOURCE_INVESTIGATION_DOCUMENT,
INCIDENT_RESOURCE_INCIDENT_REVIEW_DOCUMENT,
]:
try:
# we get the document object
document = get_document(db_session=db_session,
incident_id=incident.id,
resource_type=doc_type)
if not document:
# the document may have not been created yet (e.g. incident review document)
break
# we get the list of tasks in the document
tasks = drive_task_plugin.list(file_id=document.resource_id)
for task in tasks:
# we get the task information
try:
create_or_update_task(db_session,
incident,
task["task"],
notify=notify)
except Exception as e:
log.exception(e)
sentry_sdk.capture_exception(e)
except Exception as e:
log.exception(e)
sentry_sdk.capture_exception(e)
def calculate_locations_cost(db_session=None):
"""Calculates the cost of all locations."""
# we want to update all locations, all the time
locations = get_all(db_session=db_session)
for location in locations:
try:
# we calculate the cost
location_cost = calculate_cost(location.id, db_session)
# if the cost hasn't changed, don't continue
if location.cost == location_cost:
continue
# we update the location
location.cost = location_cost
db_session.add(location)
db_session.commit()
log.debug(f"Location cost for {location.name} updated in the database.")
if location.ticket.resource_id:
# we update the external ticket
update_external_location_ticket(location, db_session)
log.debug(f"Location cost for {location.name} updated in the ticket.")
else:
log.debug(f"Ticket not found. Location cost for {location.name} not updated.")
except Exception as e:
# we shouldn't fail to update all locations when one fails
sentry_sdk.capture_exception(e)
def calculate_incidents_cost(db_session=None):
"""Calculates the cost of all incidents."""
# we want to update all incidents, all the time
incidents = get_all(db_session=db_session)
for incident in incidents:
try:
# we calculate the cost
incident_cost = calculate_cost(incident.id, db_session)
# if the cost hasn't changed, don't continue
if incident.cost == incident_cost:
continue
# we update the incident
incident.cost = incident_cost
db_session.add(incident)
db_session.commit()
log.debug(f"Incident cost for {incident.name} updated in the database.")
if incident.ticket.resource_id:
# we update the external ticket
update_external_incident_ticket(incident, db_session)
log.debug(f"Incident cost for {incident.name} updated in the ticket.")
else:
log.debug(f"Ticket not found. Incident cost for {incident.name} not updated.")
except Exception as e:
# we shouldn't fail to update all incidents when one fails
sentry_sdk.capture_exception(e)
def status_report_reminder(db_session=None):
"""Sends status report reminders to active incident commanders."""
incidents = get_all_by_status(db_session=db_session,
status=IncidentStatus.active)
convo_plugin = plugins.get(INCIDENT_PLUGIN_CONVERSATION_SLUG)
status_report_command = convo_plugin.get_command_name(
ConversationCommands.status_report)
for incident in incidents:
try:
notification_hour = STATUS_REPORT_REMINDER_MAPPING[
incident.incident_priority.name.lower()]
if incident.last_status_report:
remind_after = incident.last_status_report.created_at
else:
remind_after = incident.created_at
now = datetime.utcnow() - remind_after
# we calculate the number of hours and seconds since last CAN was sent
hours, seconds = divmod((now.days * 86400) + now.seconds, 3600)
q, r = divmod(hours, notification_hour)
if q >= 1 and r == 0: # it's time to send the reminder
if incident.ticket: # TODO remove once we get clean data
items = [{
"name": incident.name,
"ticket_weblink": incident.ticket.weblink,
"title": incident.title,
"command": status_report_command,
}]
convo_plugin.send_direct(
incident.commander.email,
"Incident Status Report Reminder",
INCIDENT_STATUS_REPORT_REMINDER,
MessageType.incident_status_report,
items=items,
)
except Exception as e:
# we shouldn't fail to update all incidents when one fails
sentry_sdk.capture_exception(e)
def auto_tagger(db_session):
"""Attempts to take existing tags and associate them with locations."""
tags = tag_service.get_all(db_session=db_session).all()
log.debug(f"Fetched {len(tags)} tags from database.")
tag_strings = [t.name.lower() for t in tags if t.discoverable]
phrases = build_term_vocab(tag_strings)
matcher = build_phrase_matcher("dispatch-tag", phrases)
p = plugins.get(
INCIDENT_PLUGIN_STORAGE_SLUG
) # this may need to be refactored if we support multiple document types
for location in get_all(db_session=db_session).all():
log.debug(f"Processing location. Name: {location.name}")
doc = location.location_document
try:
mime_type = "text/plain"
text = p.get(doc.resource_id, mime_type)
except Exception as e:
log.debug(f"Failed to get document. Reason: {e}")
sentry_sdk.capture_exception(e)
continue
extracted_tags = list(set(extract_terms_from_text(text, matcher)))
matched_tags = (
db_session.query(Tag)
.filter(func.upper(Tag.name).in_([func.upper(t) for t in extracted_tags]))
.all()
)
location.tags.extend(matched_tags)
db_session.commit()
log.debug(
f"Associating tags with location. Location: {location.name}, Tags: {extracted_tags}"
)
def sync_document_terms(db_session=None):
"""Performs term extraction from known documents."""
documents = get_all(db_session=db_session)
for doc in documents:
log.debug(f"Processing document. Name: {doc.name}")
p = plugins.get(
INCIDENT_PLUGIN_STORAGE_SLUG
) # this may need to be refactored if we support multiple document types
try:
if "sheet" in doc.resource_type:
mime_type = "text/csv"
else:
mime_type = "text/plain"
doc_text = p.get(doc.resource_id, mime_type)
extracted_terms = route_service.get_terms(
db_session=db_session, model=Term, text=doc_text
)
matched_terms = (
db_session.query(Term)
.filter(func.upper(Term.text).in_([func.upper(t) for t in extracted_terms]))
.all()
)
log.debug(f"Extracted the following terms from {doc.weblink}. Terms: {extracted_terms}")
if matched_terms:
doc.terms = matched_terms
db_session.commit()
except Exception as e:
# even if one document fails we don't want them to all fail
sentry_sdk.capture_exception(e)
log.exception(e)
def active_incidents_cost(db_session=None):
"""Calculates the cost of all active incidents."""
active_incidents = get_all_by_status(db_session=db_session,
status=IncidentStatus.active)
for incident in active_incidents:
# we calculate the cost
try:
incident_cost = calculate_cost(incident.id, db_session)
# we update the incident
incident.cost = incident_cost
db_session.add(incident)
db_session.commit()
if incident.ticket.resource_id:
# we update the external ticket
ticket_plugin = plugins.get(INCIDENT_PLUGIN_TICKET_SLUG)
ticket_plugin.update(
incident.ticket.resource_id,
cost=incident_cost,
incident_type=incident.incident_type.name,
)
log.debug(
f"Incident cost for {incident.name} updated in the ticket."
)
else:
log.debug(
f"Incident cost for {incident.name} not updated. Ticket not found."
)
log.debug(
f"Incident cost for {incident.name} updated in the database.")
except Exception as e:
# we shouldn't fail to update all incidents when one fails
sentry_sdk.capture_exception(e)
def daily_summary(db_session=None):
"""Fetches all open incidents and provides a daily summary."""
blocks = []
blocks.append({
"type": "section",
"text": {
"type": "mrkdwn",
"text": f"*{INCIDENT_DAILY_SUMMARY_DESCRIPTION}*"
},
})
active_incidents = get_all_by_status(db_session=db_session,
status=IncidentStatus.active)
if active_incidents:
blocks.append({
"type": "section",
"text": {
"type": "mrkdwn",
"text":
f"*{INCIDENT_DAILY_SUMMARY_ACTIVE_INCIDENTS_DESCRIPTION}*",
},
})
for idx, incident in enumerate(active_incidents):
if incident.visibility == Visibility.open:
try:
blocks.append({
"type": "section",
"text": {
"type":
"mrkdwn",
"text":
(f"*<{incident.ticket.weblink}|{incident.name}>*\n"
f"*Title*: {incident.title}\n"
f"*Type*: {incident.incident_type.name}\n"
f"*Priority*: {incident.incident_priority.name}\n"
f"*Incident Commander*: <{incident.commander.weblink}|{incident.commander.name}>"
),
},
"block_id":
f"{ConversationButtonActions.invite_user}-active-{idx}",
"accessory": {
"type": "button",
"text": {
"type": "plain_text",
"text": "Join Incident"
},
"value": f"{incident.id}",
},
})
except Exception as e:
sentry_sdk.capture_exception(e)
else:
blocks.append({
"type": "section",
"text": {
"type": "mrkdwn",
"text": INCIDENT_DAILY_SUMMARY_NO_ACTIVE_INCIDENTS_DESCRIPTION,
},
})
blocks.append({"type": "divider"})
blocks.append({
"type": "section",
"text": {
"type":
"mrkdwn",
"text":
f"*{INCIDENT_DAILY_SUMMARY_STABLE_CLOSED_INCIDENTS_DESCRIPTION}*",
},
})
hours = 24
stable_incidents = get_all_last_x_hours_by_status(
db_session=db_session, status=IncidentStatus.stable, hours=hours)
closed_incidents = get_all_last_x_hours_by_status(
db_session=db_session, status=IncidentStatus.closed, hours=hours)
if stable_incidents or closed_incidents:
for idx, incident in enumerate(stable_incidents):
if incident.visibility == Visibility.open:
try:
blocks.append({
"type": "section",
"text": {
"type":
"mrkdwn",
"text":
(f"*<{incident.ticket.weblink}|{incident.name}>*\n"
f"*Title*: {incident.title}\n"
f"*Type*: {incident.incident_type.name}\n"
f"*Priority*: {incident.incident_priority.name}\n"
f"*Incident Commander*: <{incident.commander.weblink}|{incident.commander.name}>\n"
f"*Status*: {incident.status}"),
},
"block_id":
f"{ConversationButtonActions.invite_user}-{idx}",
"accessory": {
"type": "button",
"text": {
"type": "plain_text",
"text": "Join Incident"
},
"value": f"{incident.id}",
},
})
except Exception as e:
sentry_sdk.capture_exception(e)
for incident in closed_incidents:
if incident.visibility == Visibility.open:
try:
blocks.append({
"type": "section",
"text": {
"type":
"mrkdwn",
"text":
(f"*<{incident.ticket.weblink}|{incident.name}>*\n"
f"*Title*: {incident.title}\n"
f"*Type*: {incident.incident_type.name}\n"
f"*Priority*: {incident.incident_priority.name}\n"
f"*Incident Commander*: <{incident.commander.weblink}|{incident.commander.name}>\n"
f"*Status*: {incident.status}"),
},
})
except Exception as e:
sentry_sdk.capture_exception(e)
else:
blocks.append({
"type": "section",
"text": {
"type":
"mrkdwn",
"text":
INCIDENT_DAILY_SUMMARY_NO_STABLE_CLOSED_INCIDENTS_DESCRIPTION,
},
})
# NOTE INCIDENT_DAILY_SUMMARY_ONCALL_SERVICE_ID is optional
if INCIDENT_DAILY_SUMMARY_ONCALL_SERVICE_ID:
oncall_service = service_service.get_by_external_id(
db_session=db_session,
external_id=INCIDENT_DAILY_SUMMARY_ONCALL_SERVICE_ID)
oncall_plugin = plugins.get(oncall_service.type)
oncall_email = oncall_plugin.get(
service_id=INCIDENT_DAILY_SUMMARY_ONCALL_SERVICE_ID)
oncall_individual = individual_service.resolve_user_by_email(
oncall_email)
blocks.append({
"type":
"context",
"elements": [{
"type":
"mrkdwn",
"text":
f"For questions about this notification, reach out to <{oncall_individual['weblink']}|{oncall_individual['fullname']}> (current on-call)",
}],
})
convo_plugin = plugins.get(INCIDENT_PLUGIN_CONVERSATION_SLUG)
for c in INCIDENT_NOTIFICATION_CONVERSATIONS:
convo_plugin.send(c, "Incident Daily Summary", {}, "", blocks=blocks)
def incident_create_flow(*,
incident_id: int,
checkpoint: str = None,
db_session=None):
"""Creates all resources required for new incidents."""
incident = incident_service.get(db_session=db_session,
incident_id=incident_id)
# get the incident participants based on incident type and priority
individual_participants, team_participants = get_incident_participants(
incident, db_session)
# add individuals to incident
for individual in individual_participants:
participant_flows.add_participant(user_email=individual.email,
incident_id=incident.id,
db_session=db_session)
event_service.log(
db_session=db_session,
source="Dispatch Core App",
description="Incident participants added to incident",
incident_id=incident.id,
)
# create the incident ticket
ticket = create_incident_ticket(incident, db_session)
incident.ticket = ticket_service.create(db_session=db_session,
ticket_in=TicketCreate(**ticket))
event_service.log(
db_session=db_session,
source="Dispatch Core App",
description="External ticket added to incident",
incident_id=incident.id,
)
# we set the incident name
name = ticket["resource_id"]
incident.name = name
# we create the participant groups (tactical and notification)
individual_participants = [x.individual for x in incident.participants]
tactical_group, notification_group = create_participant_groups(
incident, individual_participants, team_participants, db_session)
for g in [tactical_group, notification_group]:
group_in = GroupCreate(
name=g["name"],
email=g["email"],
resource_type=g["resource_type"],
resource_id=g["resource_id"],
weblink=g["weblink"],
)
incident.groups.append(
group_service.create(db_session=db_session, group_in=group_in))
event_service.log(
db_session=db_session,
source="Dispatch Core App",
description="Tactical and notification groups added to incident",
incident_id=incident.id,
)
# we create storage resource
storage = create_incident_storage(
incident, [tactical_group["email"], notification_group["email"]],
db_session)
incident.storage = storage_service.create(
db_session=db_session,
resource_id=storage["resource_id"],
resource_type=storage["resource_type"],
weblink=storage["weblink"],
)
event_service.log(
db_session=db_session,
source="Dispatch Core App",
description="Storage added to incident",
incident_id=incident.id,
)
# we create the incident documents
collab_documents = create_collaboration_documents(incident, db_session)
for d in collab_documents:
document_in = DocumentCreate(
name=d["name"],
resource_id=d["resource_id"],
resource_type=d["resource_type"],
weblink=d["weblink"],
)
incident.documents.append(
document_service.create(db_session=db_session,
document_in=document_in))
event_service.log(
db_session=db_session,
source="Dispatch Core App",
description="Documents added to incident",
incident_id=incident.id,
)
conference = create_conference(incident, [tactical_group["email"]],
db_session)
conference_in = ConferenceCreate(
resource_id=conference["resource_id"],
resource_type=conference["resource_type"],
weblink=conference["weblink"],
conference_id=conference["id"],
conference_challenge=conference["challenge"],
)
incident.conference = conference_service.create(
db_session=db_session, conference_in=conference_in)
event_service.log(
db_session=db_session,
source="Dispatch Core App",
description="Conference added to incident",
incident_id=incident.id,
)
# we create the conversation for real-time communications
participant_emails = [x.individual.email for x in incident.participants]
conversation = create_conversation(incident, participant_emails,
db_session)
conversation_in = ConversationCreate(
resource_id=conversation["resource_id"],
resource_type=conversation["resource_type"],
weblink=conversation["weblink"],
channel_id=conversation["id"],
)
incident.conversation = conversation_service.create(
db_session=db_session, conversation_in=conversation_in)
event_service.log(
db_session=db_session,
source="Dispatch Core App",
description="Conversation added to incident",
incident_id=incident.id,
)
db_session.add(incident)
db_session.commit()
# we set the conversation topic
set_conversation_topic(incident)
# we update the incident ticket
update_external_incident_ticket(incident, db_session)
# we update the investigation document
update_document(
incident.incident_document.resource_id,
incident.name,
incident.incident_priority.name,
incident.status,
incident.incident_type.name,
incident.title,
incident.description,
incident.commander.name,
incident.conversation.weblink,
incident.incident_document.weblink,
incident.storage.weblink,
incident.ticket.weblink,
incident.conference.weblink,
incident.conference.conference_challenge,
)
if incident.visibility == Visibility.open:
send_incident_notifications(incident, db_session)
event_service.log(
db_session=db_session,
source="Dispatch Core App",
description="Incident notifications sent",
incident_id=incident.id,
)
suggested_document_items = get_suggested_document_items(
incident, db_session)
for participant in incident.participants:
# we announce the participant in the conversation
# should protect ourselves from failures of any one participant
try:
send_incident_participant_announcement_message(
participant.individual.email, incident.id, db_session)
# we send the welcome messages to the participant
send_incident_welcome_participant_messages(
participant.individual.email, incident.id, db_session)
send_incident_suggested_reading_messages(incident,
suggested_document_items,
participant.email)
except Exception as e:
log.exception(e)
sentry_sdk.capture_exception(e)
event_service.log(
db_session=db_session,
source="Dispatch Core App",
description="Participants announced and welcome messages sent",
incident_id=incident.id,
)
