def incident_report_reminders(db_session=None): """Sends report reminders to incident commanders for active incidents.""" for project in project_service.get_all(db_session=db_session): incidents = incident_service.get_all_by_status( db_session=db_session, project_id=project.id, status=IncidentStatus.active ) for incident in incidents: for report_type in ReportTypes: try: remind_after = incident.created_at if report_type == ReportTypes.tactical_report: notification_hour = incident.incident_priority.tactical_report_reminder if incident.last_tactical_report: remind_after = incident.last_tactical_report.created_at elif report_type == ReportTypes.executive_report: notification_hour = incident.incident_priority.executive_report_reminder if incident.last_executive_report: remind_after = incident.last_executive_report.created_at now = datetime.utcnow() - remind_after # we calculate the number of hours and seconds since last report was sent hours, seconds = divmod((now.days * 86400) + now.seconds, 3600) q, r = divmod(hours, notification_hour) if q >= 1 and r == 0: # it's time to send the reminder send_incident_report_reminder(incident, report_type, db_session) except Exception as e: # we shouldn't fail to send all reminders when one fails log.exception(e)
def project_select_block(db_session: Session, initial_option: dict = None): """Builds the incident project select block.""" project_options = [] for project in project_service.get_all(db_session=db_session): project_options.append( option_from_template(text=project.name, value=project.name)) block = { "block_id": IncidentBlockId.project, "type": "input", "dispatch_action": True, "label": { "text": "Project", "type": "plain_text", }, "element": { "type": "static_select", "placeholder": { "type": "plain_text", "text": "Select Project" }, "options": project_options, "action_id": ReportIncidentCallbackId.update_view, }, } if initial_option: block["element"].update({ "initial_option": option_from_template(text=initial_option.name, value=initial_option.name) }) return block
def build_tag_models(db_session=None): """Builds the intensive tag recommendation models.""" # incident model for project in project_service.get_all(db_session=db_session): incidents = incident_service.get_all(db_session=db_session, project_id=project.id).all() log.debug("Starting to build the incident/tag model...") build_model(incidents, "incident") log.debug("Successfully built the incident/tag model.")
def calculate_incidents_response_cost(db_session=None): """ Calculates and saves the response cost for all incidents. """ for project in project_service.get_all(db_session=db_session): response_cost_type = incident_cost_type_service.get_default( db_session=db_session, project_id=project.id) if response_cost_type is None: log.warning( f"A default cost type for response cost does not exist in the {project.name} project. Response costs won't be calculated." ) continue # we want to update the response cost of all incidents, all the time incidents = incident_service.get_all(db_session=db_session, project_id=project.id) for incident in incidents: try: # we get the response cost for the given incident incident_response_cost = get_by_incident_id_and_incident_cost_type_id( db_session=db_session, incident_id=incident.id, incident_cost_type_id=response_cost_type.id, ) if incident_response_cost is None: # we create the response cost if it doesn't exist incident_cost_type = IncidentCostTypeRead.from_orm( response_cost_type) incident_cost_in = IncidentCostCreate( incident_cost_type=incident_cost_type, project=project) incident_response_cost = create( db_session=db_session, incident_cost_in=incident_cost_in) # we calculate the response cost amount amount = calculate_incident_response_cost( incident.id, db_session) # we don't need to update the cost amount if it hasn't changed if incident_response_cost.amount == amount: continue # we save the new incident cost amount incident_response_cost.amount = amount incident.incident_costs.append(incident_response_cost) db_session.add(incident) db_session.commit() log.debug( f"Response cost amount for {incident.name} incident has been updated in the database." ) except Exception as e: # we shouldn't fail to update all incidents when one fails log.exception(e)
def sync_active_stable_workflows(db_session=None): """Syncs incident workflows.""" # we get all active and stable incidents for project in project_service.get_all(db_session=db_session): active_incidents = incident_service.get_all_by_status( db_session=db_session, project_id=project.id, status=IncidentStatus.active) stable_incidents = incident_service.get_all_by_status( db_session=db_session, project_id=project.id, status=IncidentStatus.stable) incidents = active_incidents + stable_incidents sync_workflows(db_session, incidents, notify=True)
def close_incident_reminder(db_session=None): """Sends a reminder to the IC to close out their incident.""" for project in project_service.get_all(db_session=db_session): incidents = get_all_by_status(db_session=db_session, project_id=project.id, status=IncidentStatus.stable) for incident in incidents: span = datetime.utcnow() - incident.stable_at q, r = divmod( span.days, 7 ) # only for incidents that have been stable longer than a week if q >= 1 and r == 0: if date.today().isoweekday() == 1: # lets only send on mondays send_incident_close_reminder(incident, db_session)
def sync_tags(db_session=None): """Syncs tags from external sources.""" for project in project_service.get_all(db_session=db_session): plugin = plugin_service.get_active_instance( db_session=db_session, plugin_type="tag", project_id=project.id ) if not plugin: continue log.debug(f"Getting tags via: {plugin.plugin.slug}") for t in plugin.instance.get(): log.debug(f"Adding Tag. Tag: {t}") # we always use the plugin project when syncing t["tag_type"].update({"project": project}) tag_in = TagCreate(**t, project=project) tag_service.get_or_create(db_session=db_session, tag_in=tag_in)
def auto_tagger(db_session): """Attempts to take existing tags and associate them with incidents.""" for project in project_service.get_all(db_session=db_session): tags = tag_service.get_all(db_session=db_session, project_id=project.id).all() log.debug(f"Fetched {len(tags)} tags from database.") tag_strings = [t.name.lower() for t in tags if t.discoverable] phrases = build_term_vocab(tag_strings) matcher = build_phrase_matcher("dispatch-tag", phrases) for incident in get_all(db_session=db_session, project_id=project.id).all(): plugin = plugin_service.get_active_instance( db_session=db_session, project_id=incident.project.id, plugin_type="storage") log.debug(f"Processing incident. Name: {incident.name}") doc = incident.incident_document if doc: try: mime_type = "text/plain" text = plugin.instance.get(doc.resource_id, mime_type) except Exception as e: log.debug(f"Failed to get document. Reason: {e}") log.exception(e) continue extracted_tags = list( set(extract_terms_from_text(text, matcher))) matched_tags = (db_session.query(Tag).filter( func.upper(Tag.name).in_( [func.upper(t) for t in extracted_tags])).all()) incident.tags.extend(matched_tags) db_session.commit() log.debug( f"Associating tags with incident. Incident: {incident.name}, Tags: {extracted_tags}" )
def wrapper(*args, **kwargs): db_session = SessionLocal() metrics_provider.counter("function.call.counter", tags={"function": fullname(func)}) start = time.perf_counter() # iterate for all schema for organization in organization_service.get_all(db_session=db_session): schema_engine = engine.execution_options( schema_translate_map={None: f"dispatch_organization_{organization.slug}"} ) schema_session = sessionmaker(bind=schema_engine)() kwargs["db_session"] = schema_session for project in project_service.get_all(db_session=schema_session): kwargs["project"] = project try: func(*args, **kwargs) except Exception as e: log.exception(e) schema_session.close() elapsed_time = time.perf_counter() - start metrics_provider.timer( "function.elapsed.time", value=elapsed_time, tags={"function": fullname(func)} ) db_session.close()
def list_incidents( user_id: str, user_email: str, channel_id: str, incident_id: int, config: SlackConversationConfiguration = None, command: dict = None, db_session=None, slack_client=None, ): """Returns the list of current active and stable incidents, and closed incidents in the last 24 hours.""" projects = [] incidents = [] args = command["text"].split(" ") # scopes reply to the current incident's project incident = incident_service.get(db_session=db_session, incident_id=incident_id) if incident: # command was run in an incident conversation projects.append(incident.project) else: # command was run in a non-incident conversation if len(args) == 2: project = project_service.get_by_name(db_session=db_session, name=args[1]) if project: projects.append() else: raise ValidationError( [ ErrorWrapper( NotFoundError( msg=f"Project name '{args[1]}' in organization '{args[0]}' not found. Check your spelling." ), loc="project", ) ], model=BaseModel, ) else: projects = project_service.get_all(db_session=db_session) for project in projects: # we fetch active incidents incidents.extend( incident_service.get_all_by_status( db_session=db_session, project_id=project.id, status=IncidentStatus.active ) ) # We fetch stable incidents incidents.extend( incident_service.get_all_by_status( db_session=db_session, project_id=project.id, status=IncidentStatus.stable, ) ) # We fetch closed incidents in the last 24 hours incidents.extend( incident_service.get_all_last_x_hours_by_status( db_session=db_session, project_id=project.id, status=IncidentStatus.closed, hours=24, ) ) blocks = [] blocks.append({"type": "header", "text": {"type": "plain_text", "text": "List of Incidents"}}) if incidents: for incident in incidents: if incident.visibility == Visibility.open: ticket_weblink = resolve_attr(incident, "ticket.weblink") try: blocks.append( { "type": "section", "text": { "type": "mrkdwn", "text": ( f"*<{ticket_weblink}|{incident.name}>*\n" f"*Title*: {incident.title}\n" f"*Type*: {incident.incident_type.name}\n" f"*Priority*: {incident.incident_priority.name}\n" f"*Status*: {incident.status}\n" f"*Incident Commander*: <{incident.commander.individual.weblink}|{incident.commander.individual.name}>\n" f"*Project*: {incident.project.name}" ), }, } ) except Exception as e: log.exception(e) dispatch_slack_service.send_ephemeral_message( slack_client, channel_id, user_id, "Incident List", blocks=blocks, )
def list_incidents( user_id: str, user_email: str, channel_id: str, incident_id: int, command: dict = None, db_session=None, slack_client=None, ): """Returns the list of current active and stable incidents, and closed incidents in the last 24 hours.""" projects = [] incidents = [] # scopes reply to the current incident's project incident = incident_service.get(db_session=db_session, incident_id=incident_id) if incident: # command was run in an incident conversation projects.append(incident.project) else: # command was run in a non-incident conversation projects = project_service.get_all(db_session=db_session) for project in projects: # we fetch active incidents incidents.extend( incident_service.get_all_by_status(db_session=db_session, project_id=project.id, status=IncidentStatus.active)) # We fetch stable incidents incidents.extend( incident_service.get_all_by_status( db_session=db_session, project_id=project.id, status=IncidentStatus.stable, )) # We fetch closed incidents in the last 24 hours incidents.extend( incident_service.get_all_last_x_hours_by_status( db_session=db_session, project_id=project.id, status=IncidentStatus.closed, hours=24, )) blocks = [] blocks.append({ "type": "header", "text": { "type": "plain_text", "text": "List of Incidents" } }) if incidents: for incident in incidents: if incident.visibility == Visibility.open: ticket_weblink = resolve_attr(incident, "ticket.weblink") try: blocks.append({ "type": "section", "text": { "type": "mrkdwn", "text": (f"*<{ticket_weblink}|{incident.name}>*\n" f"*Title*: {incident.title}\n" f"*Type*: {incident.incident_type.name}\n" f"*Priority*: {incident.incident_priority.name}\n" f"*Status*: {incident.status}\n" f"*Incident Commander*: <{incident.commander.individual.weblink}|{incident.commander.individual.name}>\n" f"*Project*: {incident.project.name}"), }, }) except Exception as e: log.exception(e) dispatch_slack_service.send_ephemeral_message( slack_client, channel_id, user_id, "Incident List", blocks=blocks, )
def daily_sync_workflow(db_session=None): """Syncs all incident workflows daily.""" for project in project_service.get_all(db_session=db_session): incidents = incident_service.get_all(db_session=db_session, project_id=project.id).all() sync_workflows(db_session, incidents, notify=False)
def daily_report(db_session=None): """ Creates and sends incident daily reports based on notifications. """ for project in project_service.get_all(db_session=db_session): # we fetch all active, stable and closed incidents active_incidents = get_all_by_status( db_session=db_session, project_id=project.id, status=IncidentStatus.active.value) stable_incidents = get_all_last_x_hours_by_status( db_session=db_session, project_id=project.id, status=IncidentStatus.stable.value, hours=24, ) closed_incidents = get_all_last_x_hours_by_status( db_session=db_session, project_id=project.id, status=IncidentStatus.closed.value, hours=24, ) incidents = active_incidents + stable_incidents + closed_incidents # we map incidents to notification filters incidents_notification_filters_mapping = defaultdict( lambda: defaultdict(lambda: [])) notifications = notification_service.get_all_enabled( db_session=db_session, project_id=project.id) for incident in incidents: for notification in notifications: for search_filter in notification.filters: match = search_filter_service.match( db_session=db_session, filter_spec=search_filter.expression, class_instance=incident, ) if match: incidents_notification_filters_mapping[ notification.id][search_filter.id].append(incident) if not notification.filters: incidents_notification_filters_mapping[ notification.id][0].append(incident) # we create and send an incidents daily report for each notification filter for notification_id, search_filter_dict in incidents_notification_filters_mapping.items( ): for search_filter_id, incidents in search_filter_dict.items(): items_grouped = [] items_grouped_template = INCIDENT for idx, incident in enumerate(incidents): try: item = { "commander_fullname": incident.commander.individual.name, "commander_team": incident.commander.team, "commander_weblink": incident.commander.individual.weblink, "incident_id": incident.id, "name": incident.name, "priority": incident.incident_priority.name, "priority_description": incident.incident_priority.description, "status": incident.status, "ticket_weblink": resolve_attr(incident, "ticket.weblink"), "title": incident.title, "type": incident.incident_type.name, "type_description": incident.incident_type.description, } if incident.status != IncidentStatus.closed.value: item.update({ "button_text": "Join Incident", "button_value": str(incident.id), "button_action": f"{ConversationButtonActions.invite_user.value}-{incident.status}-{idx}", }) items_grouped.append(item) except Exception as e: log.exception(e) notification_kwargs = { "contact_fullname": DISPATCH_HELP_EMAIL, "contact_weblink": DISPATCH_HELP_EMAIL, "items_grouped": items_grouped, "items_grouped_template": items_grouped_template, } notification_params = { "text": INCIDENT_DAILY_REPORT_TITLE, "type": MessageType.incident_daily_report, "template": INCIDENT_DAILY_REPORT, "kwargs": notification_kwargs, } notification = notification_service.get( db_session=db_session, notification_id=notification_id) notification_service.send( db_session=db_session, project_id=notification.project.id, notification=notification, notification_params=notification_params, )
def create_task_reminders(db_session=None): """Creates multiple task reminders.""" for project in project_service.get_all(db_session=db_session): tasks = task_service.get_overdue_tasks(db_session=db_session, project_id=project.id) log.debug(f"New tasks that need reminders. NumTasks: {len(tasks)}") # let's only remind for active incidents for now tasks = [ t for t in tasks if t.incident.status == IncidentStatus.active ] if tasks: contact_fullname = contact_weblink = DISPATCH_HELP_EMAIL # NOTE INCIDENT_ONCALL_SERVICE_ID is optional if INCIDENT_ONCALL_SERVICE_ID: oncall_service = service_service.get_by_external_id( db_session=db_session, external_id=INCIDENT_ONCALL_SERVICE_ID) if not oncall_service: log.warning( "INCIDENT_ONCALL_SERVICE_ID configured in the .env file, but not found in the database. Did you create the oncall service in the UI?" ) return oncall_plugin = plugin_service.get_active_instance( db_session=db_session, project_id=project.id, plugin_type="oncall") if not oncall_plugin: log.warning( f"Unable to resolve oncall. No oncall plugin is enabled. Project: {project.name}" ) if oncall_plugin.plugin.slug != oncall_service.type: log.warning( f"Unable to resolve the oncall. Oncall plugin enabled not of type {oncall_plugin.plugin.slug}." ) return if not oncall_plugin: log.warning( f"Unable to resolve the oncall, INCIDENT_ONCALL_SERVICE_ID configured, but associated plugin ({oncall_plugin.plugin.slug}) is not enabled." ) contact_fullname = "Unknown" contact_weblink = None else: oncall_email = oncall_plugin.instance.get( service_id=INCIDENT_ONCALL_SERVICE_ID) oncall_individual = individual_service.resolve_user_by_email( oncall_email, db_session) contact_fullname = oncall_individual["fullname"] contact_weblink = oncall_individual["weblink"] grouped_tasks = group_tasks_by_assignee(tasks) for assignee, tasks in grouped_tasks.items(): create_reminder(db_session, assignee, tasks, contact_fullname, contact_weblink)