def sync_active_stable_tasks(db_session: SessionLocal, project: Project):
"""Syncs incident tasks."""
task_plugin = plugin_service.get_active_instance(db_session=db_session,
project_id=project.id,
plugin_type="task")
if not task_plugin:
log.warning(
f"Skipping task sync no task plugin enabled. ProjectId: {project.id}"
)
return
# we get all active and stable incidents
active_incidents = incident_service.get_all_by_status(
db_session=db_session,
project_id=project.id,
status=IncidentStatus.active)
stable_incidents = incident_service.get_all_by_status(
db_session=db_session,
project_id=project.id,
status=IncidentStatus.stable)
incidents = active_incidents + stable_incidents
lookback = 60 * 5 # 5 min
sync_tasks(db_session,
task_plugin,
incidents,
lookback=lookback,
notify=True)
def list_incidents(incident_id: int, command: dict = None, db_session=None):
"""Returns the list of current active and stable incidents,
and closed incidents in the last 24 hours."""
incidents = []
# We fetch active incidents
incidents = incident_service.get_all_by_status(
db_session=db_session, status=IncidentStatus.active.value)
# We fetch stable incidents
incidents.extend(
incident_service.get_all_by_status(db_session=db_session,
status=IncidentStatus.stable.value))
# We fetch closed incidents in the last 24 hours
incidents.extend(
incident_service.get_all_last_x_hours_by_status(
db_session=db_session,
status=IncidentStatus.closed.value,
hours=24))
blocks = []
blocks.append({
"type": "header",
"text": {
"type": "plain_text",
"text": "List of Incidents"
}
})
if incidents:
for incident in incidents:
if incident.visibility == Visibility.open:
ticket_weblink = resolve_attr(incident, "ticket.weblink")
try:
blocks.append({
"type": "section",
"text": {
"type":
"mrkdwn",
"text":
(f"*<{ticket_weblink}|{incident.name}>*\n"
f"*Title*: {incident.title}\n"
f"*Type*: {incident.incident_type.name}\n"
f"*Priority*: {incident.incident_priority.name}\n"
f"*Status*: {incident.status}\n"
f"*Incident Commander*: <{incident.commander.individual.weblink}|{incident.commander.individual.name}>"
),
},
})
except Exception as e:
log.exception(e)
dispatch_slack_service.send_ephemeral_message(
slack_client,
command["channel_id"],
command["user_id"],
"Incident List",
blocks=blocks,
)
def sync_active_stable_workflows(db_session=None):
"""Syncs incident workflows."""
# we get all active and stable incidents
active_incidents = incident_service.get_all_by_status(
db_session=db_session, status=IncidentStatus.active)
stable_incidents = incident_service.get_all_by_status(
db_session=db_session, status=IncidentStatus.stable)
incidents = active_incidents + stable_incidents
sync_workflows(db_session, incidents, notify=True)
def sync_active_stable_tasks(db_session=None):
"""Syncs incident tasks."""
for project in project_service.get_all(db_session=db_session):
# we get all active and stable incidents
active_incidents = incident_service.get_all_by_status(
db_session=db_session,
project_id=project.id,
status=IncidentStatus.active)
stable_incidents = incident_service.get_all_by_status(
db_session=db_session,
project_id=project.id,
status=IncidentStatus.stable)
incidents = active_incidents + stable_incidents
sync_tasks(db_session, incidents, notify=True)
def incident_report_reminders(db_session=None):
"""Sends report reminders to incident commanders for active incidents."""
incidents = incident_service.get_all_by_status(
db_session=db_session, status=IncidentStatus.active
)
for incident in incidents:
for report_type in ReportTypes:
try:
remind_after = incident.created_at
if report_type == ReportTypes.tactical_report:
notification_hour = incident.incident_priority.tactical_report_reminder
if incident.last_tactical_report:
remind_after = incident.last_tactical_report.created_at
elif report_type == ReportTypes.executive_report:
notification_hour = incident.incident_priority.executive_report_reminder
if incident.last_executive_report:
remind_after = incident.last_executive_report.created_at
now = datetime.utcnow() - remind_after
# we calculate the number of hours and seconds since last report was sent
hours, seconds = divmod((now.days * 86400) + now.seconds, 3600)
q, r = divmod(hours, notification_hour)
if q >= 1 and r == 0: # it's time to send the reminder
send_incident_report_reminder(incident, report_type)
except Exception as e:
# we shouldn't fail to send all reminders when one fails
sentry_sdk.capture_exception(e)
def sync_active_stable_monitors(db_session: SessionLocal, project: Project):
"""Syncs incident monitors."""
monitor_plugin = plugin_service.get_active_instance(db_session=db_session,
project_id=project.id,
plugin_type="monitor")
if not monitor_plugin:
log.warning(f"No monitor plugin is enabled. ProjectId: {project.id}")
return
# we get all active and stable incidents
active_incidents = incident_service.get_all_by_status(
db_session=db_session,
project_id=project.id,
status=IncidentStatus.active)
stable_incidents = incident_service.get_all_by_status(
db_session=db_session,
project_id=project.id,
status=IncidentStatus.stable)
incidents = active_incidents + stable_incidents
run_monitors(db_session, project, monitor_plugin, incidents, notify=True)
def list_incidents(
user_id: str,
user_email: str,
channel_id: str,
incident_id: int,
config: SlackConversationConfiguration = None,
command: dict = None,
db_session=None,
slack_client=None,
):
"""Returns the list of current active and stable incidents,
and closed incidents in the last 24 hours."""
projects = []
incidents = []
args = command["text"].split(" ")
# scopes reply to the current incident's project
incident = incident_service.get(db_session=db_session, incident_id=incident_id)
if incident:
# command was run in an incident conversation
projects.append(incident.project)
else:
# command was run in a non-incident conversation
if len(args) == 2:
project = project_service.get_by_name(db_session=db_session, name=args[1])
if project:
projects.append()
else:
raise ValidationError(
[
ErrorWrapper(
NotFoundError(
msg=f"Project name '{args[1]}' in organization '{args[0]}' not found. Check your spelling."
),
loc="project",
)
],
model=BaseModel,
)
else:
projects = project_service.get_all(db_session=db_session)
for project in projects:
# we fetch active incidents
incidents.extend(
incident_service.get_all_by_status(
db_session=db_session, project_id=project.id, status=IncidentStatus.active
)
)
# We fetch stable incidents
incidents.extend(
incident_service.get_all_by_status(
db_session=db_session,
project_id=project.id,
status=IncidentStatus.stable,
)
)
# We fetch closed incidents in the last 24 hours
incidents.extend(
incident_service.get_all_last_x_hours_by_status(
db_session=db_session,
project_id=project.id,
status=IncidentStatus.closed,
hours=24,
)
)
blocks = []
blocks.append({"type": "header", "text": {"type": "plain_text", "text": "List of Incidents"}})
if incidents:
for incident in incidents:
if incident.visibility == Visibility.open:
ticket_weblink = resolve_attr(incident, "ticket.weblink")
try:
blocks.append(
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": (
f"*<{ticket_weblink}|{incident.name}>*\n"
f"*Title*: {incident.title}\n"
f"*Type*: {incident.incident_type.name}\n"
f"*Priority*: {incident.incident_priority.name}\n"
f"*Status*: {incident.status}\n"
f"*Incident Commander*: <{incident.commander.individual.weblink}|{incident.commander.individual.name}>\n"
f"*Project*: {incident.project.name}"
),
},
}
)
except Exception as e:
log.exception(e)
dispatch_slack_service.send_ephemeral_message(
slack_client,
channel_id,
user_id,
"Incident List",
blocks=blocks,
)
def list_incidents(
user_id: str,
user_email: str,
channel_id: str,
incident_id: int,
command: dict = None,
db_session=None,
slack_client=None,
):
"""Returns the list of current active and stable incidents,
and closed incidents in the last 24 hours."""
projects = []
incidents = []
# scopes reply to the current incident's project
incident = incident_service.get(db_session=db_session,
incident_id=incident_id)
if incident:
# command was run in an incident conversation
projects.append(incident.project)
else:
# command was run in a non-incident conversation
projects = project_service.get_all(db_session=db_session)
for project in projects:
# we fetch active incidents
incidents.extend(
incident_service.get_all_by_status(db_session=db_session,
project_id=project.id,
status=IncidentStatus.active))
# We fetch stable incidents
incidents.extend(
incident_service.get_all_by_status(
db_session=db_session,
project_id=project.id,
status=IncidentStatus.stable,
))
# We fetch closed incidents in the last 24 hours
incidents.extend(
incident_service.get_all_last_x_hours_by_status(
db_session=db_session,
project_id=project.id,
status=IncidentStatus.closed,
hours=24,
))
blocks = []
blocks.append({
"type": "header",
"text": {
"type": "plain_text",
"text": "List of Incidents"
}
})
if incidents:
for incident in incidents:
if incident.visibility == Visibility.open:
ticket_weblink = resolve_attr(incident, "ticket.weblink")
try:
blocks.append({
"type": "section",
"text": {
"type":
"mrkdwn",
"text":
(f"*<{ticket_weblink}|{incident.name}>*\n"
f"*Title*: {incident.title}\n"
f"*Type*: {incident.incident_type.name}\n"
f"*Priority*: {incident.incident_priority.name}\n"
f"*Status*: {incident.status}\n"
f"*Incident Commander*: <{incident.commander.individual.weblink}|{incident.commander.individual.name}>\n"
f"*Project*: {incident.project.name}"),
},
})
except Exception as e:
log.exception(e)
dispatch_slack_service.send_ephemeral_message(
slack_client,
channel_id,
user_id,
"Incident List",
blocks=blocks,
)
def sync_tasks(db_session=None):
"""Syncs incident tasks."""
# we get all active and stable incidents
active_incidents = incident_service.get_all_by_status(
db_session=db_session, status=IncidentStatus.active)
stable_incidents = incident_service.get_all_by_status(
db_session=db_session, status=IncidentStatus.stable)
incidents = active_incidents + stable_incidents
# we create an instance of the drive task plugin
drive_task_plugin = plugins.get(INCIDENT_PLUGIN_TASK_SLUG)
for incident in incidents:
for doc_type in [
INCIDENT_RESOURCE_INVESTIGATION_DOCUMENT,
INCIDENT_RESOURCE_INCIDENT_REVIEW_DOCUMENT,
]:
# we get the document object
document = get_document(db_session=db_session,
incident_id=incident.id,
resource_type=doc_type)
if not document:
# the document may have not been created yet (e.g. incident review document)
break
# we get the list of tasks in the document
tasks = drive_task_plugin.list(file_id=document.resource_id)
for t in tasks:
# we get the task information
creator = t["task"]["owner"]
assignees = ", ".join(t["task"]["assignees"])
description = t["task"]["description"][0]
status = TaskStatus.open if not t["task"][
"status"] else TaskStatus.resolved
resource_id = t["task"]["id"]
weblink = t["task"]["web_link"]
incident_task = task_service.get_by_resource_id(
db_session=db_session, resource_id=t["task"]["id"])
if incident_task:
if status == TaskStatus.open:
# we don't need to take any actions if the status of the task in the collaboration doc is open
break
else:
if incident_task.status == TaskStatus.resolved:
# we don't need to take any actions if the task has already been marked as resolved in the database
break
else:
# we mark the task as resolved in the database
incident_task.status = TaskStatus.resolved
db_session.add(incident_task)
db_session.commit()
# we send a notification to the incident conversation
notification_text = "Incident Notification"
notification_type = "incident-notification"
convo_plugin = plugins.get(
INCIDENT_PLUGIN_CONVERSATION_SLUG)
convo_plugin.send(
incident.conversation.channel_id,
notification_text,
INCIDENT_TASK_RESOLVED_NOTIFICATION,
notification_type,
task_assignees=assignees,
task_description=description,
task_weblink=weblink,
)
else:
# we add the task to the incident
task = task_service.create(
db_session=db_session,
creator=creator,
assignees=assignees,
description=description,
status=status,
resource_id=resource_id,
resource_type=INCIDENT_RESOURCE_INCIDENT_TASK,
weblink=weblink,
)
incident.tasks.append(task)
db_session.add(incident)
db_session.commit()
# we send a notification to the incident conversation
notification_text = "Incident Notification"
notification_type = "incident-notification"
convo_plugin = plugins.get(
INCIDENT_PLUGIN_CONVERSATION_SLUG)
convo_plugin.send(
incident.conversation.channel_id,
notification_text,
INCIDENT_TASK_NEW_NOTIFICATION,
notification_type,
task_assignees=assignees,
task_description=description,
task_weblink=weblink,
)
