def incident_report_reminders(db_session=None): """Sends report reminders to incident commanders for active incidents.""" incidents = incident_service.get_all_by_status( db_session=db_session, status=IncidentStatus.active ) for incident in incidents: for report_type in ReportTypes: try: remind_after = incident.created_at if report_type == ReportTypes.tactical_report: notification_hour = incident.incident_priority.tactical_report_reminder if incident.last_tactical_report: remind_after = incident.last_tactical_report.created_at elif report_type == ReportTypes.executive_report: notification_hour = incident.incident_priority.executive_report_reminder if incident.last_executive_report: remind_after = incident.last_executive_report.created_at now = datetime.utcnow() - remind_after # we calculate the number of hours and seconds since last report was sent hours, seconds = divmod((now.days * 86400) + now.seconds, 3600) q, r = divmod(hours, notification_hour) if q >= 1 and r == 0: # it's time to send the reminder send_incident_report_reminder(incident, report_type) except Exception as e: # we shouldn't fail to send all reminders when one fails sentry_sdk.capture_exception(e)
def status_report_reminder(db_session=None): """Sends status report reminders to active incident commanders.""" incidents = get_all_by_status(db_session=db_session, status=IncidentStatus.active) for incident in incidents: try: notification_hour = incident.incident_priority.status_reminder if incident.last_status_report: remind_after = incident.last_status_report.created_at else: remind_after = incident.created_at now = datetime.utcnow() - remind_after # we calculate the number of hours and seconds since last CAN was sent hours, seconds = divmod((now.days * 86400) + now.seconds, 3600) q, r = divmod(hours, notification_hour) if q >= 1 and r == 0: # it's time to send the reminder send_incident_status_report_reminder(incident) except Exception as e: # we shouldn't fail to update all incidents when one fails sentry_sdk.capture_exception(e)
def sync_tasks(db_session, incidents, notify: bool = False): """Syncs tasks and sends update notifications to incident channels.""" drive_task_plugin = plugins.get(INCIDENT_PLUGIN_TASK_SLUG) for incident in incidents: for doc_type in [ INCIDENT_RESOURCE_INVESTIGATION_DOCUMENT, INCIDENT_RESOURCE_INCIDENT_REVIEW_DOCUMENT, ]: try: # we get the document object document = get_document(db_session=db_session, incident_id=incident.id, resource_type=doc_type) if not document: # the document may have not been created yet (e.g. incident review document) break # we get the list of tasks in the document tasks = drive_task_plugin.list(file_id=document.resource_id) for task in tasks: # we get the task information try: create_or_update_task(db_session, incident, task["task"], notify=notify) except Exception as e: log.exception(e) sentry_sdk.capture_exception(e) except Exception as e: log.exception(e) sentry_sdk.capture_exception(e)
def calculate_locations_cost(db_session=None): """Calculates the cost of all locations.""" # we want to update all locations, all the time locations = get_all(db_session=db_session) for location in locations: try: # we calculate the cost location_cost = calculate_cost(location.id, db_session) # if the cost hasn't changed, don't continue if location.cost == location_cost: continue # we update the location location.cost = location_cost db_session.add(location) db_session.commit() log.debug(f"Location cost for {location.name} updated in the database.") if location.ticket.resource_id: # we update the external ticket update_external_location_ticket(location, db_session) log.debug(f"Location cost for {location.name} updated in the ticket.") else: log.debug(f"Ticket not found. Location cost for {location.name} not updated.") except Exception as e: # we shouldn't fail to update all locations when one fails sentry_sdk.capture_exception(e)
def calculate_incidents_cost(db_session=None): """Calculates the cost of all incidents.""" # we want to update all incidents, all the time incidents = get_all(db_session=db_session) for incident in incidents: try: # we calculate the cost incident_cost = calculate_cost(incident.id, db_session) # if the cost hasn't changed, don't continue if incident.cost == incident_cost: continue # we update the incident incident.cost = incident_cost db_session.add(incident) db_session.commit() log.debug(f"Incident cost for {incident.name} updated in the database.") if incident.ticket.resource_id: # we update the external ticket update_external_incident_ticket(incident, db_session) log.debug(f"Incident cost for {incident.name} updated in the ticket.") else: log.debug(f"Ticket not found. Incident cost for {incident.name} not updated.") except Exception as e: # we shouldn't fail to update all incidents when one fails sentry_sdk.capture_exception(e)
def status_report_reminder(db_session=None): """Sends status report reminders to active incident commanders.""" incidents = get_all_by_status(db_session=db_session, status=IncidentStatus.active) convo_plugin = plugins.get(INCIDENT_PLUGIN_CONVERSATION_SLUG) status_report_command = convo_plugin.get_command_name( ConversationCommands.status_report) for incident in incidents: try: notification_hour = STATUS_REPORT_REMINDER_MAPPING[ incident.incident_priority.name.lower()] if incident.last_status_report: remind_after = incident.last_status_report.created_at else: remind_after = incident.created_at now = datetime.utcnow() - remind_after # we calculate the number of hours and seconds since last CAN was sent hours, seconds = divmod((now.days * 86400) + now.seconds, 3600) q, r = divmod(hours, notification_hour) if q >= 1 and r == 0: # it's time to send the reminder if incident.ticket: # TODO remove once we get clean data items = [{ "name": incident.name, "ticket_weblink": incident.ticket.weblink, "title": incident.title, "command": status_report_command, }] convo_plugin.send_direct( incident.commander.email, "Incident Status Report Reminder", INCIDENT_STATUS_REPORT_REMINDER, MessageType.incident_status_report, items=items, ) except Exception as e: # we shouldn't fail to update all incidents when one fails sentry_sdk.capture_exception(e)
def auto_tagger(db_session): """Attempts to take existing tags and associate them with locations.""" tags = tag_service.get_all(db_session=db_session).all() log.debug(f"Fetched {len(tags)} tags from database.") tag_strings = [t.name.lower() for t in tags if t.discoverable] phrases = build_term_vocab(tag_strings) matcher = build_phrase_matcher("dispatch-tag", phrases) p = plugins.get( INCIDENT_PLUGIN_STORAGE_SLUG ) # this may need to be refactored if we support multiple document types for location in get_all(db_session=db_session).all(): log.debug(f"Processing location. Name: {location.name}") doc = location.location_document try: mime_type = "text/plain" text = p.get(doc.resource_id, mime_type) except Exception as e: log.debug(f"Failed to get document. Reason: {e}") sentry_sdk.capture_exception(e) continue extracted_tags = list(set(extract_terms_from_text(text, matcher))) matched_tags = ( db_session.query(Tag) .filter(func.upper(Tag.name).in_([func.upper(t) for t in extracted_tags])) .all() ) location.tags.extend(matched_tags) db_session.commit() log.debug( f"Associating tags with location. Location: {location.name}, Tags: {extracted_tags}" )
def sync_document_terms(db_session=None): """Performs term extraction from known documents.""" documents = get_all(db_session=db_session) for doc in documents: log.debug(f"Processing document. Name: {doc.name}") p = plugins.get( INCIDENT_PLUGIN_STORAGE_SLUG ) # this may need to be refactored if we support multiple document types try: if "sheet" in doc.resource_type: mime_type = "text/csv" else: mime_type = "text/plain" doc_text = p.get(doc.resource_id, mime_type) extracted_terms = route_service.get_terms( db_session=db_session, model=Term, text=doc_text ) matched_terms = ( db_session.query(Term) .filter(func.upper(Term.text).in_([func.upper(t) for t in extracted_terms])) .all() ) log.debug(f"Extracted the following terms from {doc.weblink}. Terms: {extracted_terms}") if matched_terms: doc.terms = matched_terms db_session.commit() except Exception as e: # even if one document fails we don't want them to all fail sentry_sdk.capture_exception(e) log.exception(e)
def active_incidents_cost(db_session=None): """Calculates the cost of all active incidents.""" active_incidents = get_all_by_status(db_session=db_session, status=IncidentStatus.active) for incident in active_incidents: # we calculate the cost try: incident_cost = calculate_cost(incident.id, db_session) # we update the incident incident.cost = incident_cost db_session.add(incident) db_session.commit() if incident.ticket.resource_id: # we update the external ticket ticket_plugin = plugins.get(INCIDENT_PLUGIN_TICKET_SLUG) ticket_plugin.update( incident.ticket.resource_id, cost=incident_cost, incident_type=incident.incident_type.name, ) log.debug( f"Incident cost for {incident.name} updated in the ticket." ) else: log.debug( f"Incident cost for {incident.name} not updated. Ticket not found." ) log.debug( f"Incident cost for {incident.name} updated in the database.") except Exception as e: # we shouldn't fail to update all incidents when one fails sentry_sdk.capture_exception(e)
def daily_summary(db_session=None): """Fetches all open incidents and provides a daily summary.""" blocks = [] blocks.append({ "type": "section", "text": { "type": "mrkdwn", "text": f"*{INCIDENT_DAILY_SUMMARY_DESCRIPTION}*" }, }) active_incidents = get_all_by_status(db_session=db_session, status=IncidentStatus.active) if active_incidents: blocks.append({ "type": "section", "text": { "type": "mrkdwn", "text": f"*{INCIDENT_DAILY_SUMMARY_ACTIVE_INCIDENTS_DESCRIPTION}*", }, }) for idx, incident in enumerate(active_incidents): if incident.visibility == Visibility.open: try: blocks.append({ "type": "section", "text": { "type": "mrkdwn", "text": (f"*<{incident.ticket.weblink}|{incident.name}>*\n" f"*Title*: {incident.title}\n" f"*Type*: {incident.incident_type.name}\n" f"*Priority*: {incident.incident_priority.name}\n" f"*Incident Commander*: <{incident.commander.weblink}|{incident.commander.name}>" ), }, "block_id": f"{ConversationButtonActions.invite_user}-active-{idx}", "accessory": { "type": "button", "text": { "type": "plain_text", "text": "Join Incident" }, "value": f"{incident.id}", }, }) except Exception as e: sentry_sdk.capture_exception(e) else: blocks.append({ "type": "section", "text": { "type": "mrkdwn", "text": INCIDENT_DAILY_SUMMARY_NO_ACTIVE_INCIDENTS_DESCRIPTION, }, }) blocks.append({"type": "divider"}) blocks.append({ "type": "section", "text": { "type": "mrkdwn", "text": f"*{INCIDENT_DAILY_SUMMARY_STABLE_CLOSED_INCIDENTS_DESCRIPTION}*", }, }) hours = 24 stable_incidents = get_all_last_x_hours_by_status( db_session=db_session, status=IncidentStatus.stable, hours=hours) closed_incidents = get_all_last_x_hours_by_status( db_session=db_session, status=IncidentStatus.closed, hours=hours) if stable_incidents or closed_incidents: for idx, incident in enumerate(stable_incidents): if incident.visibility == Visibility.open: try: blocks.append({ "type": "section", "text": { "type": "mrkdwn", "text": (f"*<{incident.ticket.weblink}|{incident.name}>*\n" f"*Title*: {incident.title}\n" f"*Type*: {incident.incident_type.name}\n" f"*Priority*: {incident.incident_priority.name}\n" f"*Incident Commander*: <{incident.commander.weblink}|{incident.commander.name}>\n" f"*Status*: {incident.status}"), }, "block_id": f"{ConversationButtonActions.invite_user}-{idx}", "accessory": { "type": "button", "text": { "type": "plain_text", "text": "Join Incident" }, "value": f"{incident.id}", }, }) except Exception as e: sentry_sdk.capture_exception(e) for incident in closed_incidents: if incident.visibility == Visibility.open: try: blocks.append({ "type": "section", "text": { "type": "mrkdwn", "text": (f"*<{incident.ticket.weblink}|{incident.name}>*\n" f"*Title*: {incident.title}\n" f"*Type*: {incident.incident_type.name}\n" f"*Priority*: {incident.incident_priority.name}\n" f"*Incident Commander*: <{incident.commander.weblink}|{incident.commander.name}>\n" f"*Status*: {incident.status}"), }, }) except Exception as e: sentry_sdk.capture_exception(e) else: blocks.append({ "type": "section", "text": { "type": "mrkdwn", "text": INCIDENT_DAILY_SUMMARY_NO_STABLE_CLOSED_INCIDENTS_DESCRIPTION, }, }) # NOTE INCIDENT_DAILY_SUMMARY_ONCALL_SERVICE_ID is optional if INCIDENT_DAILY_SUMMARY_ONCALL_SERVICE_ID: oncall_service = service_service.get_by_external_id( db_session=db_session, external_id=INCIDENT_DAILY_SUMMARY_ONCALL_SERVICE_ID) oncall_plugin = plugins.get(oncall_service.type) oncall_email = oncall_plugin.get( service_id=INCIDENT_DAILY_SUMMARY_ONCALL_SERVICE_ID) oncall_individual = individual_service.resolve_user_by_email( oncall_email) blocks.append({ "type": "context", "elements": [{ "type": "mrkdwn", "text": f"For questions about this notification, reach out to <{oncall_individual['weblink']}|{oncall_individual['fullname']}> (current on-call)", }], }) convo_plugin = plugins.get(INCIDENT_PLUGIN_CONVERSATION_SLUG) for c in INCIDENT_NOTIFICATION_CONVERSATIONS: convo_plugin.send(c, "Incident Daily Summary", {}, "", blocks=blocks)
def incident_create_flow(*, incident_id: int, checkpoint: str = None, db_session=None): """Creates all resources required for new incidents.""" incident = incident_service.get(db_session=db_session, incident_id=incident_id) # get the incident participants based on incident type and priority individual_participants, team_participants = get_incident_participants( incident, db_session) # add individuals to incident for individual in individual_participants: participant_flows.add_participant(user_email=individual.email, incident_id=incident.id, db_session=db_session) event_service.log( db_session=db_session, source="Dispatch Core App", description="Incident participants added to incident", incident_id=incident.id, ) # create the incident ticket ticket = create_incident_ticket(incident, db_session) incident.ticket = ticket_service.create(db_session=db_session, ticket_in=TicketCreate(**ticket)) event_service.log( db_session=db_session, source="Dispatch Core App", description="External ticket added to incident", incident_id=incident.id, ) # we set the incident name name = ticket["resource_id"] incident.name = name # we create the participant groups (tactical and notification) individual_participants = [x.individual for x in incident.participants] tactical_group, notification_group = create_participant_groups( incident, individual_participants, team_participants, db_session) for g in [tactical_group, notification_group]: group_in = GroupCreate( name=g["name"], email=g["email"], resource_type=g["resource_type"], resource_id=g["resource_id"], weblink=g["weblink"], ) incident.groups.append( group_service.create(db_session=db_session, group_in=group_in)) event_service.log( db_session=db_session, source="Dispatch Core App", description="Tactical and notification groups added to incident", incident_id=incident.id, ) # we create storage resource storage = create_incident_storage( incident, [tactical_group["email"], notification_group["email"]], db_session) incident.storage = storage_service.create( db_session=db_session, resource_id=storage["resource_id"], resource_type=storage["resource_type"], weblink=storage["weblink"], ) event_service.log( db_session=db_session, source="Dispatch Core App", description="Storage added to incident", incident_id=incident.id, ) # we create the incident documents collab_documents = create_collaboration_documents(incident, db_session) for d in collab_documents: document_in = DocumentCreate( name=d["name"], resource_id=d["resource_id"], resource_type=d["resource_type"], weblink=d["weblink"], ) incident.documents.append( document_service.create(db_session=db_session, document_in=document_in)) event_service.log( db_session=db_session, source="Dispatch Core App", description="Documents added to incident", incident_id=incident.id, ) conference = create_conference(incident, [tactical_group["email"]], db_session) conference_in = ConferenceCreate( resource_id=conference["resource_id"], resource_type=conference["resource_type"], weblink=conference["weblink"], conference_id=conference["id"], conference_challenge=conference["challenge"], ) incident.conference = conference_service.create( db_session=db_session, conference_in=conference_in) event_service.log( db_session=db_session, source="Dispatch Core App", description="Conference added to incident", incident_id=incident.id, ) # we create the conversation for real-time communications participant_emails = [x.individual.email for x in incident.participants] conversation = create_conversation(incident, participant_emails, db_session) conversation_in = ConversationCreate( resource_id=conversation["resource_id"], resource_type=conversation["resource_type"], weblink=conversation["weblink"], channel_id=conversation["id"], ) incident.conversation = conversation_service.create( db_session=db_session, conversation_in=conversation_in) event_service.log( db_session=db_session, source="Dispatch Core App", description="Conversation added to incident", incident_id=incident.id, ) db_session.add(incident) db_session.commit() # we set the conversation topic set_conversation_topic(incident) # we update the incident ticket update_external_incident_ticket(incident, db_session) # we update the investigation document update_document( incident.incident_document.resource_id, incident.name, incident.incident_priority.name, incident.status, incident.incident_type.name, incident.title, incident.description, incident.commander.name, incident.conversation.weblink, incident.incident_document.weblink, incident.storage.weblink, incident.ticket.weblink, incident.conference.weblink, incident.conference.conference_challenge, ) if incident.visibility == Visibility.open: send_incident_notifications(incident, db_session) event_service.log( db_session=db_session, source="Dispatch Core App", description="Incident notifications sent", incident_id=incident.id, ) suggested_document_items = get_suggested_document_items( incident, db_session) for participant in incident.participants: # we announce the participant in the conversation # should protect ourselves from failures of any one participant try: send_incident_participant_announcement_message( participant.individual.email, incident.id, db_session) # we send the welcome messages to the participant send_incident_welcome_participant_messages( participant.individual.email, incident.id, db_session) send_incident_suggested_reading_messages(incident, suggested_document_items, participant.email) except Exception as e: log.exception(e) sentry_sdk.capture_exception(e) event_service.log( db_session=db_session, source="Dispatch Core App", description="Participants announced and welcome messages sent", incident_id=incident.id, )