def incident_report_reminders(db_session=None):
"""Sends report reminders to incident commanders for active incidents."""
for project in project_service.get_all(db_session=db_session):
incidents = incident_service.get_all_by_status(
db_session=db_session, project_id=project.id, status=IncidentStatus.active
)
for incident in incidents:
for report_type in ReportTypes:
try:
remind_after = incident.created_at
if report_type == ReportTypes.tactical_report:
notification_hour = incident.incident_priority.tactical_report_reminder
if incident.last_tactical_report:
remind_after = incident.last_tactical_report.created_at
elif report_type == ReportTypes.executive_report:
notification_hour = incident.incident_priority.executive_report_reminder
if incident.last_executive_report:
remind_after = incident.last_executive_report.created_at
now = datetime.utcnow() - remind_after
# we calculate the number of hours and seconds since last report was sent
hours, seconds = divmod((now.days * 86400) + now.seconds, 3600)
q, r = divmod(hours, notification_hour)
if q >= 1 and r == 0: # it's time to send the reminder
send_incident_report_reminder(incident, report_type, db_session)
except Exception as e:
# we shouldn't fail to send all reminders when one fails
log.exception(e)
def project_select_block(db_session: Session, initial_option: dict = None):
"""Builds the incident project select block."""
project_options = []
for project in project_service.get_all(db_session=db_session):
project_options.append(
option_from_template(text=project.name, value=project.name))
block = {
"block_id": IncidentBlockId.project,
"type": "input",
"dispatch_action": True,
"label": {
"text": "Project",
"type": "plain_text",
},
"element": {
"type": "static_select",
"placeholder": {
"type": "plain_text",
"text": "Select Project"
},
"options": project_options,
"action_id": ReportIncidentCallbackId.update_view,
},
}
if initial_option:
block["element"].update({
"initial_option":
option_from_template(text=initial_option.name,
value=initial_option.name)
})
return block
def build_tag_models(db_session=None):
"""Builds the intensive tag recommendation models."""
# incident model
for project in project_service.get_all(db_session=db_session):
incidents = incident_service.get_all(db_session=db_session, project_id=project.id).all()
log.debug("Starting to build the incident/tag model...")
build_model(incidents, "incident")
log.debug("Successfully built the incident/tag model.")
def calculate_incidents_response_cost(db_session=None):
"""
Calculates and saves the response cost for all incidents.
"""
for project in project_service.get_all(db_session=db_session):
response_cost_type = incident_cost_type_service.get_default(
db_session=db_session, project_id=project.id)
if response_cost_type is None:
log.warning(
f"A default cost type for response cost does not exist in the {project.name} project. Response costs won't be calculated."
)
continue
# we want to update the response cost of all incidents, all the time
incidents = incident_service.get_all(db_session=db_session,
project_id=project.id)
for incident in incidents:
try:
# we get the response cost for the given incident
incident_response_cost = get_by_incident_id_and_incident_cost_type_id(
db_session=db_session,
incident_id=incident.id,
incident_cost_type_id=response_cost_type.id,
)
if incident_response_cost is None:
# we create the response cost if it doesn't exist
incident_cost_type = IncidentCostTypeRead.from_orm(
response_cost_type)
incident_cost_in = IncidentCostCreate(
incident_cost_type=incident_cost_type, project=project)
incident_response_cost = create(
db_session=db_session,
incident_cost_in=incident_cost_in)
# we calculate the response cost amount
amount = calculate_incident_response_cost(
incident.id, db_session)
# we don't need to update the cost amount if it hasn't changed
if incident_response_cost.amount == amount:
continue
# we save the new incident cost amount
incident_response_cost.amount = amount
incident.incident_costs.append(incident_response_cost)
db_session.add(incident)
db_session.commit()
log.debug(
f"Response cost amount for {incident.name} incident has been updated in the database."
)
except Exception as e:
# we shouldn't fail to update all incidents when one fails
log.exception(e)
def sync_active_stable_workflows(db_session=None):
"""Syncs incident workflows."""
# we get all active and stable incidents
for project in project_service.get_all(db_session=db_session):
active_incidents = incident_service.get_all_by_status(
db_session=db_session,
project_id=project.id,
status=IncidentStatus.active)
stable_incidents = incident_service.get_all_by_status(
db_session=db_session,
project_id=project.id,
status=IncidentStatus.stable)
incidents = active_incidents + stable_incidents
sync_workflows(db_session, incidents, notify=True)
def close_incident_reminder(db_session=None):
"""Sends a reminder to the IC to close out their incident."""
for project in project_service.get_all(db_session=db_session):
incidents = get_all_by_status(db_session=db_session,
project_id=project.id,
status=IncidentStatus.stable)
for incident in incidents:
span = datetime.utcnow() - incident.stable_at
q, r = divmod(
span.days, 7
) # only for incidents that have been stable longer than a week
if q >= 1 and r == 0:
if date.today().isoweekday() == 1: # lets only send on mondays
send_incident_close_reminder(incident, db_session)
def sync_tags(db_session=None):
"""Syncs tags from external sources."""
for project in project_service.get_all(db_session=db_session):
plugin = plugin_service.get_active_instance(
db_session=db_session, plugin_type="tag", project_id=project.id
)
if not plugin:
continue
log.debug(f"Getting tags via: {plugin.plugin.slug}")
for t in plugin.instance.get():
log.debug(f"Adding Tag. Tag: {t}")
# we always use the plugin project when syncing
t["tag_type"].update({"project": project})
tag_in = TagCreate(**t, project=project)
tag_service.get_or_create(db_session=db_session, tag_in=tag_in)
def auto_tagger(db_session):
"""Attempts to take existing tags and associate them with incidents."""
for project in project_service.get_all(db_session=db_session):
tags = tag_service.get_all(db_session=db_session,
project_id=project.id).all()
log.debug(f"Fetched {len(tags)} tags from database.")
tag_strings = [t.name.lower() for t in tags if t.discoverable]
phrases = build_term_vocab(tag_strings)
matcher = build_phrase_matcher("dispatch-tag", phrases)
for incident in get_all(db_session=db_session,
project_id=project.id).all():
plugin = plugin_service.get_active_instance(
db_session=db_session,
project_id=incident.project.id,
plugin_type="storage")
log.debug(f"Processing incident. Name: {incident.name}")
doc = incident.incident_document
if doc:
try:
mime_type = "text/plain"
text = plugin.instance.get(doc.resource_id, mime_type)
except Exception as e:
log.debug(f"Failed to get document. Reason: {e}")
log.exception(e)
continue
extracted_tags = list(
set(extract_terms_from_text(text, matcher)))
matched_tags = (db_session.query(Tag).filter(
func.upper(Tag.name).in_(
[func.upper(t) for t in extracted_tags])).all())
incident.tags.extend(matched_tags)
db_session.commit()
log.debug(
f"Associating tags with incident. Incident: {incident.name}, Tags: {extracted_tags}"
)
def wrapper(*args, **kwargs):
db_session = SessionLocal()
metrics_provider.counter("function.call.counter", tags={"function": fullname(func)})
start = time.perf_counter()
# iterate for all schema
for organization in organization_service.get_all(db_session=db_session):
schema_engine = engine.execution_options(
schema_translate_map={None: f"dispatch_organization_{organization.slug}"}
)
schema_session = sessionmaker(bind=schema_engine)()
kwargs["db_session"] = schema_session
for project in project_service.get_all(db_session=schema_session):
kwargs["project"] = project
try:
func(*args, **kwargs)
except Exception as e:
log.exception(e)
schema_session.close()
elapsed_time = time.perf_counter() - start
metrics_provider.timer(
"function.elapsed.time", value=elapsed_time, tags={"function": fullname(func)}
)
db_session.close()
def list_incidents(
user_id: str,
user_email: str,
channel_id: str,
incident_id: int,
config: SlackConversationConfiguration = None,
command: dict = None,
db_session=None,
slack_client=None,
):
"""Returns the list of current active and stable incidents,
and closed incidents in the last 24 hours."""
projects = []
incidents = []
args = command["text"].split(" ")
# scopes reply to the current incident's project
incident = incident_service.get(db_session=db_session, incident_id=incident_id)
if incident:
# command was run in an incident conversation
projects.append(incident.project)
else:
# command was run in a non-incident conversation
if len(args) == 2:
project = project_service.get_by_name(db_session=db_session, name=args[1])
if project:
projects.append()
else:
raise ValidationError(
[
ErrorWrapper(
NotFoundError(
msg=f"Project name '{args[1]}' in organization '{args[0]}' not found. Check your spelling."
),
loc="project",
)
],
model=BaseModel,
)
else:
projects = project_service.get_all(db_session=db_session)
for project in projects:
# we fetch active incidents
incidents.extend(
incident_service.get_all_by_status(
db_session=db_session, project_id=project.id, status=IncidentStatus.active
)
)
# We fetch stable incidents
incidents.extend(
incident_service.get_all_by_status(
db_session=db_session,
project_id=project.id,
status=IncidentStatus.stable,
)
)
# We fetch closed incidents in the last 24 hours
incidents.extend(
incident_service.get_all_last_x_hours_by_status(
db_session=db_session,
project_id=project.id,
status=IncidentStatus.closed,
hours=24,
)
)
blocks = []
blocks.append({"type": "header", "text": {"type": "plain_text", "text": "List of Incidents"}})
if incidents:
for incident in incidents:
if incident.visibility == Visibility.open:
ticket_weblink = resolve_attr(incident, "ticket.weblink")
try:
blocks.append(
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": (
f"*<{ticket_weblink}|{incident.name}>*\n"
f"*Title*: {incident.title}\n"
f"*Type*: {incident.incident_type.name}\n"
f"*Priority*: {incident.incident_priority.name}\n"
f"*Status*: {incident.status}\n"
f"*Incident Commander*: <{incident.commander.individual.weblink}|{incident.commander.individual.name}>\n"
f"*Project*: {incident.project.name}"
),
},
}
)
except Exception as e:
log.exception(e)
dispatch_slack_service.send_ephemeral_message(
slack_client,
channel_id,
user_id,
"Incident List",
blocks=blocks,
)
def list_incidents(
user_id: str,
user_email: str,
channel_id: str,
incident_id: int,
command: dict = None,
db_session=None,
slack_client=None,
):
"""Returns the list of current active and stable incidents,
and closed incidents in the last 24 hours."""
projects = []
incidents = []
# scopes reply to the current incident's project
incident = incident_service.get(db_session=db_session,
incident_id=incident_id)
if incident:
# command was run in an incident conversation
projects.append(incident.project)
else:
# command was run in a non-incident conversation
projects = project_service.get_all(db_session=db_session)
for project in projects:
# we fetch active incidents
incidents.extend(
incident_service.get_all_by_status(db_session=db_session,
project_id=project.id,
status=IncidentStatus.active))
# We fetch stable incidents
incidents.extend(
incident_service.get_all_by_status(
db_session=db_session,
project_id=project.id,
status=IncidentStatus.stable,
))
# We fetch closed incidents in the last 24 hours
incidents.extend(
incident_service.get_all_last_x_hours_by_status(
db_session=db_session,
project_id=project.id,
status=IncidentStatus.closed,
hours=24,
))
blocks = []
blocks.append({
"type": "header",
"text": {
"type": "plain_text",
"text": "List of Incidents"
}
})
if incidents:
for incident in incidents:
if incident.visibility == Visibility.open:
ticket_weblink = resolve_attr(incident, "ticket.weblink")
try:
blocks.append({
"type": "section",
"text": {
"type":
"mrkdwn",
"text":
(f"*<{ticket_weblink}|{incident.name}>*\n"
f"*Title*: {incident.title}\n"
f"*Type*: {incident.incident_type.name}\n"
f"*Priority*: {incident.incident_priority.name}\n"
f"*Status*: {incident.status}\n"
f"*Incident Commander*: <{incident.commander.individual.weblink}|{incident.commander.individual.name}>\n"
f"*Project*: {incident.project.name}"),
},
})
except Exception as e:
log.exception(e)
dispatch_slack_service.send_ephemeral_message(
slack_client,
channel_id,
user_id,
"Incident List",
blocks=blocks,
)
def daily_sync_workflow(db_session=None):
"""Syncs all incident workflows daily."""
for project in project_service.get_all(db_session=db_session):
incidents = incident_service.get_all(db_session=db_session,
project_id=project.id).all()
sync_workflows(db_session, incidents, notify=False)
def daily_report(db_session=None):
"""
Creates and sends incident daily reports based on notifications.
"""
for project in project_service.get_all(db_session=db_session):
# we fetch all active, stable and closed incidents
active_incidents = get_all_by_status(
db_session=db_session,
project_id=project.id,
status=IncidentStatus.active.value)
stable_incidents = get_all_last_x_hours_by_status(
db_session=db_session,
project_id=project.id,
status=IncidentStatus.stable.value,
hours=24,
)
closed_incidents = get_all_last_x_hours_by_status(
db_session=db_session,
project_id=project.id,
status=IncidentStatus.closed.value,
hours=24,
)
incidents = active_incidents + stable_incidents + closed_incidents
# we map incidents to notification filters
incidents_notification_filters_mapping = defaultdict(
lambda: defaultdict(lambda: []))
notifications = notification_service.get_all_enabled(
db_session=db_session, project_id=project.id)
for incident in incidents:
for notification in notifications:
for search_filter in notification.filters:
match = search_filter_service.match(
db_session=db_session,
filter_spec=search_filter.expression,
class_instance=incident,
)
if match:
incidents_notification_filters_mapping[
notification.id][search_filter.id].append(incident)
if not notification.filters:
incidents_notification_filters_mapping[
notification.id][0].append(incident)
# we create and send an incidents daily report for each notification filter
for notification_id, search_filter_dict in incidents_notification_filters_mapping.items(
):
for search_filter_id, incidents in search_filter_dict.items():
items_grouped = []
items_grouped_template = INCIDENT
for idx, incident in enumerate(incidents):
try:
item = {
"commander_fullname":
incident.commander.individual.name,
"commander_team":
incident.commander.team,
"commander_weblink":
incident.commander.individual.weblink,
"incident_id":
incident.id,
"name":
incident.name,
"priority":
incident.incident_priority.name,
"priority_description":
incident.incident_priority.description,
"status":
incident.status,
"ticket_weblink":
resolve_attr(incident, "ticket.weblink"),
"title":
incident.title,
"type":
incident.incident_type.name,
"type_description":
incident.incident_type.description,
}
if incident.status != IncidentStatus.closed.value:
item.update({
"button_text":
"Join Incident",
"button_value":
str(incident.id),
"button_action":
f"{ConversationButtonActions.invite_user.value}-{incident.status}-{idx}",
})
items_grouped.append(item)
except Exception as e:
log.exception(e)
notification_kwargs = {
"contact_fullname": DISPATCH_HELP_EMAIL,
"contact_weblink": DISPATCH_HELP_EMAIL,
"items_grouped": items_grouped,
"items_grouped_template": items_grouped_template,
}
notification_params = {
"text": INCIDENT_DAILY_REPORT_TITLE,
"type": MessageType.incident_daily_report,
"template": INCIDENT_DAILY_REPORT,
"kwargs": notification_kwargs,
}
notification = notification_service.get(
db_session=db_session, notification_id=notification_id)
notification_service.send(
db_session=db_session,
project_id=notification.project.id,
notification=notification,
notification_params=notification_params,
)
def create_task_reminders(db_session=None):
"""Creates multiple task reminders."""
for project in project_service.get_all(db_session=db_session):
tasks = task_service.get_overdue_tasks(db_session=db_session,
project_id=project.id)
log.debug(f"New tasks that need reminders. NumTasks: {len(tasks)}")
# let's only remind for active incidents for now
tasks = [
t for t in tasks if t.incident.status == IncidentStatus.active
]
if tasks:
contact_fullname = contact_weblink = DISPATCH_HELP_EMAIL
# NOTE INCIDENT_ONCALL_SERVICE_ID is optional
if INCIDENT_ONCALL_SERVICE_ID:
oncall_service = service_service.get_by_external_id(
db_session=db_session,
external_id=INCIDENT_ONCALL_SERVICE_ID)
if not oncall_service:
log.warning(
"INCIDENT_ONCALL_SERVICE_ID configured in the .env file, but not found in the database. Did you create the oncall service in the UI?"
)
return
oncall_plugin = plugin_service.get_active_instance(
db_session=db_session,
project_id=project.id,
plugin_type="oncall")
if not oncall_plugin:
log.warning(
f"Unable to resolve oncall. No oncall plugin is enabled. Project: {project.name}"
)
if oncall_plugin.plugin.slug != oncall_service.type:
log.warning(
f"Unable to resolve the oncall. Oncall plugin enabled not of type {oncall_plugin.plugin.slug}."
)
return
if not oncall_plugin:
log.warning(
f"Unable to resolve the oncall, INCIDENT_ONCALL_SERVICE_ID configured, but associated plugin ({oncall_plugin.plugin.slug}) is not enabled."
)
contact_fullname = "Unknown"
contact_weblink = None
else:
oncall_email = oncall_plugin.instance.get(
service_id=INCIDENT_ONCALL_SERVICE_ID)
oncall_individual = individual_service.resolve_user_by_email(
oncall_email, db_session)
contact_fullname = oncall_individual["fullname"]
contact_weblink = oncall_individual["weblink"]
grouped_tasks = group_tasks_by_assignee(tasks)
for assignee, tasks in grouped_tasks.items():
create_reminder(db_session, assignee, tasks, contact_fullname,
contact_weblink)
