def create_escalation_policy(token, name):
""" create an escalation policy in PD """
users = pd.request(token=token,
endpoint="users",
params={
"limit": 1,
"query": CHECK_USER
})
user = users['users'][0]
body = {
"escalation_policy": {
"type":
"escalation_policy",
"name":
name,
"escalation_rules": [{
"escalation_delay_in_minutes":
10,
"targets": [{
"id": user['id'],
"type": "user_reference"
}]
}],
"description":
"PDprobe transient"
}
}
if TEAM_ID:
body['teams'] = [{"type": "team_reference", "id": TEAM_ID}]
return pd.request(token=token,
endpoint="escalation_policies",
method="POST",
data=body)
def copyincident():
token = request.args.get('token')
if token == None:
print("no token in request")
return "ok"
body = request.get_json()
if body == None:
print("no JSON body")
return "ok"
try:
incident_url = body["messages"][0]["incident"]["html_url"]
message = body["messages"][0]
event = message['event']
if event != 'incident.custom':
print(f"Event is {event}, doing nothing")
return "ok"
user_id = message['log_entries'][0]['agent']['id']
user = pd.request(api_key=token, endpoint=f"users/{user_id}")
from_email = user['user']['email']
incident_id = message['incident']['id']
incident = pd.request(api_key=token,
endpoint=f"incidents/{incident_id}")
del incident["incident"]["id"]
incident["incident"]["status"] = "triggered"
incident["incident"][
"title"] = f"Copy of {incident['incident']['title']}"
del incident["incident"]["assignments"]
del incident["incident"]["incident_key"]
incident_post_result = pd.request(api_key=token,
endpoint="incidents",
method="POST",
data=incident,
addheaders={"From": from_email})
new_incident_id = incident_post_result["incident"]["id"]
print(f"Copied incident {incident_url} to {new_incident_id}")
alerts_thread = Thread(target=process_alerts,
args=(token, from_email, incident_id,
new_incident_id))
alerts_thread.start()
print(f"started thread for incident alerts on {new_incident_id}")
notes_thread = Thread(target=process_notes,
args=(token, incident_id, new_incident_id))
notes_thread.start()
print(f"started thread for incident notes on {new_incident_id}")
except Exception as e:
traceback.print_exc()
r = "ok"
return r
def resolve(incident):
i = pd.request(api_key=incident['domain']['token'], endpoint=f"/incidents/{incident['id']}")
user_id = i['incident']['assignments'][-1]['assignee']['id']
u = pd.request(api_key=incident['domain']['token'], endpoint=f"/users/{user_id}")
user_email = u['user']['email']
user_name = u['user']['name']
print(f" Resolve by {user_name} - {user_email} ({user_id})")
r = pd.request(api_key=incident['domain']['token'],
endpoint=f"/incidents/{incident['id']}",
method="PUT",
data={"incident": {"type": "incident_reference", "status": "resolved"}},
addheaders={"From": user_email})
del current_incidents[incident['id']]
def slack_event(self, team, user, req):
me = DotMap(
pd.request(oauth_token=user["pd_token"], endpoint="users/me"))
sc = SlackClient(team["slack_bot_token"])
slack_team_id = team["slack_team_id"]
slack_userid = user["slack_userid"]
host = os.environ['SERVER_NAME'] or "localhost"
sc.api_call(
"chat.postEphemeral",
channel=req.event.channel,
text="You're currently logged in to subdomain *{}* as *{}*".format(
user["pd_subdomain"], me.user.email),
attachments=[{
"text":
"",
"color":
"#25c151",
"attachment_type":
"default",
"actions": [{
"text":
"Change PD subdomain/user",
"type":
"button",
"url":
"https://{}/me?slack_team_id={}&slack_userid={}".format(
host, slack_team_id, slack_userid)
}]
}],
user=slack_userid)
def create_pd_snow_extension(pd_service_id):
create_extension_body = {
"extension": {
"name":
f"ServiceNow ({args.snow_instance})",
"config": {
"snow_user":
args.snow_pd_user,
"snow_password":
args.snow_pd_pass,
"sync_options":
pd_incident_sync_mode,
"target":
f"https://{args.snow_instance}.service-now.com/api/x_pd_integration/pagerduty2sn"
},
"extension_schema": {
"id": pd_extension_schema_id,
"type": "extension_schema_reference"
},
"extension_objects": [{
"id": f"{pd_service_id}",
"type": "service_reference"
}]
}
}
r = pd.request(api_key=args.pd_api_key,
endpoint="extensions",
method="POST",
data=create_extension_body)
return r["extension"]["id"]
def slack_event(self, team, user, req):
me = DotMap(pd.request(oauth_token=user["pd_token"], endpoint="users/me"))
sc = SlackClient(team["slack_bot_token"])
sc.api_call("chat.postMessage",
channel=req.event.channel,
text="<@{}> is mapped to *{}* in domain *{}*".format(req.event.user, me.user.email, user["pd_subdomain"])
)
def create_integration(token, service_id):
""" create an integration in a PD service """
body = {
"type": "events_api_v2_inbound_integration",
"name": "PDprobe",
}
return pd.request(token=token,
endpoint=f"services/{service_id}/integrations",
method="POST",
data=body)
def start_zoom(req):
incident_id = req.incident.id
incident_title = req.incident.title
incident_number = req.incident.incident_number
requester_id = req.log_entries[0].agent.id
requester_name = req.log_entries[0].agent.summary
url = f"https://api.zoom.us/v2/users/{zoom_userid}/meetings"
topic = f'[{incident_number}] {incident_title}'
print(f'start zoom requested on {topic} by {requester_id} ({requester_name})')
data = {
"type": 1,
"topic": topic
}
req = requests.Request(
method='POST',
url=url,
headers={"Authorization": f"Bearer {zoom_token()}"},
json=data
)
prepped = req.prepare()
response = requests.Session().send(prepped)
res = DotMap(response.json())
if res.join_url:
join_url = res.join_url
else:
join_url = ""
if res.settings.global_dial_in_numbers[0].number:
join_tel = f"{res.settings.global_dial_in_numbers[0].number},,{res.id}#"
else:
join_tel = ""
print(f'created meeting {join_url} for incident {topic}')
add_conf = {
"requester_id": requester_id,
"incidents": [
{
"id": incident_id,
"type": "incident_reference",
"metadata": {
"conference_url": join_url,
"conference_number": join_tel
}
}
]
}
response = pd.request(api_key=pd_key, endpoint="/incidents", method="PUT", data=add_conf, addheaders={"From": from_email})
return "", 200
def slack_load_options(self, team, user, req):
endpoint = "escalation_policies"
query = req.value
r = pd.request(oauth_token=user["pd_token"], endpoint=endpoint, params={"query": query})
# Slack dialogs expect "label", interactive message select menus expect "text" :-\
options_list = [{"text": elem["name"], "label": elem["name"], "value": elem["id"]} for elem in r[endpoint]]
if len(options_list) == 0:
options_list.append({"text": "Nothing found.", "value": "nothing"})
elif len(options_list) == 25:
options_list.insert(0, {"text": "(> 25 found, please type more letters.)", "value": "nothing"})
return json.dumps({"options": options_list})
def slack_action(self, team, user, req):
if req.type == "dialog_submission":
incident = {
"incident": {
"type": "incident",
"title": req.submission.title,
"service": {
"id": req.submission.service,
"type": "service_reference"
}
}
}
description = req.submission.description or ""
description += "\n\nIncident opened by @{}: https://www.slack.com/messages/@{}".format(req.user.name, req.user.name)
incident["incident"]["body"] = {
"type": "incident_body",
"details": description
}
if req.submission.user:
incident["incident"]["assignments"] = [
{
"assignee": {
"type": "user_reference",
"id": req.submission.user
}
}
]
r = pd.request(
oauth_token=user["pd_token"],
endpoint="incidents",
method='POST',
data=incident
)
response_url = req.response_url
requests.post(response_url,
headers={
"Content-type": "application/json"
},
json={
"text": "Created an incident in domain *{}*:".format(user["pd_subdomain"]),
"attachments": slack_formatters.make_incident_attachments(r.get('incident')),
"replace_original": True
}
)
return('', 200)
def slack_action(self, team, user, req):
ep_id = req.actions[0].selected_options[0].value
response_url = req.response_url
ep = pd.request(oauth_token=user.pd_token, endpoint="/escalation_policies/{}".format(ep_id), params={"include[]": "current_oncall"})
requests.post(response_url,
headers={
"Content-type": "application/json"
},
data=json.dumps({
"text": slack_formatters.make_ep_text(ep.get('escalation_policy')),
"color": "#25c151",
"replace_original": True
})
)
def process_notes(token, incident_id, new_incident_id):
incident_notes = pd.fetch(api_key=token,
endpoint=f"incidents/{incident_id}/notes")
incident_notes.reverse()
for note in incident_notes:
note_add_data = {
"note": {
"content": f"{note['content']} ({note['created_at']})"
}
}
note_add_result = pd.request(
api_key=token,
endpoint=f"incidents/{new_incident_id}/notes",
method="POST",
data=note_add_data,
addheaders={"From": email_for_user_id(token, note["user"]["id"])})
def move_alert(token, from_email, incident_id, alert_id, new_incident_id):
alert_add_data = {
"alert": {
"type": "alert",
"incident": {
"type": "incident_reference",
"id": new_incident_id
}
}
}
alert_add_result = pd.request(
api_key=token,
endpoint=f"incidents/{incident_id}/alerts/{alert_id}",
method="PUT",
data=alert_add_data,
addheaders={"From": from_email})
def slack_load_options(self, team, user, req):
# req.pprint()
if req.name == "service":
endpoint = "services"
elif req.name == "user":
endpoint = "users"
else:
return('', 200)
query = req.value
r = pd.request(oauth_token=user["pd_token"], endpoint=endpoint, params={"query": query})
options_list = [{"text": elem["name"], "label": elem["name"], "value": elem["id"]} for elem in r[endpoint]]
if len(options_list) == 0:
options_list.append({"label": "Nothing found.", "value": "nothing"})
elif len(options_list) == 25:
options_list.insert(0, {"label": "-- More than 25 results found!. Please type more letters. --", "value": "nothing"})
return json.dumps({"options": options_list})
def createExtension(token, snow_instance, snow_api_user, snow_api_pw, sync,
int_version_key, service_id):
me = 'createExtension'
# create extension
if DEBUG: print(f' {me}: Creating extension id={int_version_key}')
body = {
"extension": {
"name":
f'ServiceNow ({snow_instance})',
"config": {
"snow_user":
snow_api_user,
"snow_password":
snow_api_pw,
"sync_options":
"sync_all" if (sync == "auto") else "manual_sync",
"target":
f'https://{snow_instance}.service-now.com/api/x_pd_integration/pagerduty2sn'
},
"extension_schema": {
"id": int_version_key,
"type": "extension_schema_reference"
},
"extension_objects": [{
"id": service_id,
"type": "service_reference",
}]
}
}
if DEBUG: print(f' {me}: Create Extension with data={body}')
create_extension_response = pd.request(token=token,
endpoint="extensions",
method="POST",
data=body)
found_webhook_id = create_extension_response['extension']['id']
if DEBUG:
print(
f' {me}: Created Extension service id={service_id} webhook id={found_webhook_id}'
)
return found_webhook_id
def create_webhook(token, name, service_id, public_url):
""" create a webhook in a PD service """
body = {
"webhook": {
"type": "webhook_reference",
"name": name,
"endpoint_url": public_url,
"webhook_object": {
"id": service_id,
"type": "service_reference"
},
"outbound_integration": {
"id": "PJFWPEP",
"type": "outbound_integration"
}
}
}
return pd.request(token=token,
endpoint=f"webhooks",
method="POST",
data=body)
def slack_command(self, team, user, form):
me = DotMap(
pd.request(oauth_token=user["pd_token"], endpoint="users/me"))
response_url = form.get('response_url')
slack_team_id = team["slack_team_id"]
slack_userid = user["slack_userid"]
slack_response = {
"response_type":
"ephemeral",
"text":
"You're currently logged in to subdomain *{}* as *{}*".format(
user["pd_subdomain"], me.user.email),
"attachments": [{
"text":
"",
"color":
"#25c151",
"attachment_type":
"default",
"actions": [{
"text":
"Change PD subdomain/user",
"type":
"button",
"url":
url_for("me",
_external=True,
_scheme="https",
slack_team_id=slack_team_id,
slack_userid=slack_userid)
}]
}]
}
requests.post(response_url,
json=slack_response,
headers={'Content-type': 'application/json'})
return ('', 200)
def slack_action(self, team, user, req):
service_id = req.actions[0].selected_options[0].value
response_url = req.response_url
if re.search(r"brief", req.callback_id):
expand_ep = False
else:
expand_ep = True
service = pd.request(oauth_token=user.pd_token,
endpoint="/services/{}".format(service_id))
r = requests.post(response_url,
headers={"Content-type": "application/json"},
json={
"text":
slack_formatters.make_service_text(
service.get('service'),
expand_ep=expand_ep,
pd_token=user["pd_token"]),
"replace_original":
True
})
def make_service_text(service, expand_ep=False, pd_token=None):
# call with service as the service body, like response.get('service')
service = DotMap(service)
service_link = "<{}|{}>".format(service.html_url, service.summary)
subdomain = re.match(r"https://([^\.]+)", service.html_url).groups()[0]
response = ":desktop_computer: Service *{}* in subdomain *{}*:\n\n".format(
service_link, subdomain)
if service.description and service.description != service.summary:
response += "Description: {}\n".format(service.description)
response += "Status: {} {}\n".format(
service_status_emoji.get(service.status), service.status.title())
ep_link = "<{}|{}>".format(service.escalation_policy.html_url,
service.escalation_policy.summary)
response += "Escalation Policy: *{}*\n".format(ep_link)
if expand_ep and pd_token:
ep = pd.request(oauth_token=pd_token,
endpoint="escalation_policies/{}".format(
service.escalation_policy.id),
method="GET",
params={"include[]": "current_oncall"})
response += make_ep_text(ep.get('escalation_policy'),
include_intro=False)
return response
def create_service(token, name, ep_id):
""" create a service in PD """
body = {
"service": {
"type": "service",
"name": name,
"escalation_policy": {
"id": ep_id,
"type": "escalation_policy_reference"
},
"incident_urgency_rule": {
"type": "constant",
"urgency": "low"
},
"alert_creation": "create_alerts_and_incidents"
}
}
if TEAM_ID:
body['teams'] = [{"type": "team_reference", "id": TEAM_ID}]
return pd.request(token=token,
endpoint="services",
method="POST",
data=body)
def slack_load_options(self, team, user, req):
endpoint = "incidents"
query = req.value
r = pd.request(oauth_token=user["pd_token"],
endpoint=endpoint,
params={
"query": query,
"statuses[]": ["triggered", "acknowledged"]
})
# Slack dialogs expect "label", interactive message select menus expect "text" :-\
options_list = [{
"text": elem["summary"],
"label": elem["summary"],
"value": elem["id"]
} for elem in r[endpoint]]
if len(options_list) == 0:
options_list.append({"text": "Nothing found.", "value": "nothing"})
elif len(options_list) == 25:
options_list.append({
"text": "See more incidents...",
"value": "more:{}".format(0)
})
return json.dumps({"options": options_list})
def destroy_service(token, ep_id):
""" destroy a service in PD """
return pd.request(token=token,
endpoint=f"services/{ep_id}",
method="DELETE")
"name": pd_webhook_name,
"type": "webhook",
"outbound_integration": {
"id": "P2JA7HV",
"type": "outbound_integration"
},
"config": {
"client_id": sfdc_client_id,
"username": sfdc_username,
"shared_key": sfdc_shared_key,
"rest_resource_url": sfdc_rest_resource_url,
"sandbox": sfdc_is_sandbox
},
"webhook_object": {
"id": pd_service_id,
"type": "service_reference"
},
}
}
print(f"Adding SFDC webhook for service {pd_service_id}... ",
end="",
flush=True)
try:
r = pd.request(token=token,
endpoint="webhooks",
method="POST",
data=body)
print(f"added webhook {r['webhook']['id']}.")
except:
print(f"failed: {r}")
print("Nothing to delete.\n")
sys.exit(0)
# Give the user a summary and confirmation
print(f"\nFound {total_objects_found} {p.plural('object', total_objects_found)} to delete:")
for obj_type, objects in things_to_delete.items():
if len(objects) > 0:
print(f"{len(objects)} {p.plural(p.singular_noun(obj_type), len(objects))}:")
for obj in objects:
print(f" {obj['id']}")
user_input = input('\nAre you sure you want to delete? [y/N] ')
if user_input.lower() in ('y', 'yes'):
# do the delete
print("OK, deleting...")
for obj_type in sorted(things_to_delete):
objects = things_to_delete[obj_type]
for obj in objects:
print(f"Deleting {p.singular_noun(obj_type)} {obj['id']}... ", end="", flush=True)
r = pd.request(token=args.token, endpoint=f"{obj_type}/{obj['id']}", method="DELETE")
if r.status_code >= 200 and r.status_code < 300:
print("Deleted.")
else:
try:
print(f"Error {r.status_code}:\n{json.dumps(r.json()['error'], indent=4)}")
except:
print(f"Error {r.status_code}")
else:
print("OK, leaving everything alone.")
endpoint = f"service_dependencies/technical_services/{pd_id}"
elif service["x_pd_integration_pagerduty_business_service"]:
pd_id = service["x_pd_integration_pagerduty_business_service"]
endpoint = f"service_dependencies/business_services/{pd_id}"
else:
print(f"Couldn't find PD ID for {service['name']}")
continue
try:
pd_service_name = pd_services_dict[pd_id]['name']
pd_service_type = pd_services_dict[pd_id]['type']
except:
pd_service_name = "(unknown service)"
pd_service_type = "(unknown type)"
r = pd.request(token=args.pd_api_key, endpoint=endpoint)
if 'relationships' in r and r['relationships']:
print(f"{pd_service_name} ({pd_service_type} {pd_id}):")
relationships = r['relationships']
for relationship in relationships:
if relationship['supporting_service']['id'] == pd_id:
verb = "Supports"
target_pd_id = relationship['dependent_service']['id']
elif relationship['dependent_service']['id'] == pd_id:
verb = "Depends on"
target_pd_id = relationship['supporting_service']['id']
else:
print(
f"PD ID is not in relationship: {json.dumps(relationship)}"
)
continue
"rotation_virtual_start":
"2019-09-17T00:00:00-05:00",
"rotation_turn_length_seconds":
86400,
"users": [{
"user": {
"id": users[-1]["id"],
"type": "user"
}
}]
}]
}
}
schedule = pd.request(api_key=api_key,
endpoint="schedules",
method="POST",
data=schedule_create_body)
schedule_id = schedule["schedule"]["id"]
print(
f"Created schedule {schedule_id} with forever on-call {users[-1]['summary']}"
)
input("Press Enter to create overrides...")
print(f"Going to create {number_of_overrides_to_create} overrides...")
dt = datetime.utcnow()
override_count = 0
while (override_count < number_of_overrides_to_create):
user = users[override_count % len(users)]
override_body = {
def email_for_user_id(token, user_id):
user = pd.request(api_key=token, endpoint=f"users/{user_id}")
return user['user']['email']
start = (parse(oncall['start'])).astimezone(
get_localzone())
end = (parse(oncall['end'])).astimezone(get_localzone())
timestr = f"{start.strftime('%c')} - {end.strftime('%c')}"
print(f" Level {oncall['level']}: {timestr}")
print("\n\nGetting Schedules... ", end="", flush=True)
schedules = pd.fetch_schedules(api_key=args.pd_api_key)
print(f"Got {len(schedules)}.\n\n")
since = datetime.now(timezone.utc).replace(microsecond=0)
until = (datetime.now(timezone.utc) +
timedelta(days=14)).replace(microsecond=0)
for s in schedules:
s_full = pd.request(api_key=args.pd_api_key,
endpoint=f"/schedules/{s['id']}",
params={
"since": since.isoformat(),
"until": until.isoformat()
})['schedule']
coverage = s_full['final_schedule']['rendered_coverage_percentage']
if coverage < 100.0:
print(
f"Schedule {s_full['summary']} ({s_full['html_url']}) has only {coverage}% coverage"
)
if coverage > 0.0:
print(" Gaps:")
entries = s_full['final_schedule']['rendered_schedule_entries']
if parse(entries[0]['start']) > since:
gap_start = since.astimezone(get_localzone())
gap_end = parse(entries[0]['start']).astimezone(
get_localzone())
print(
def merge_new_incident(token, user_id, service_id, incident_id,
integration_id):
print(
f"hi merge new incident {token} {user_id} {service_id} {incident_id}")
integration = pd.request(
api_key=token,
endpoint=f"services/{service_id}/integrations/{integration_id}")
integration_key = integration["integration"]["integration_key"]
print(f"integration key is {integration_key}")
user = pd.request(api_key=token, endpoint=f"users/{user_id}")
from_email = user['user']['email']
print(f"from email is {from_email}")
new_dedup_key = str(uuid.uuid4())
alert_body = {
"event_action": "trigger",
"routing_key": integration_key,
"dedup_key": new_dedup_key,
"payload": {
"summary": f"keepalive for {incident_id}",
"source": "PDkeepincident",
"severity": "info"
}
}
print("sending alert")
r = requests.post('https://events.pagerduty.com/v2/enqueue',
json=alert_body)
print(r.json())
tries = 0
time.sleep(1)
r = pd.request(api_key=token,
endpoint='incidents',
params={'incident_key': new_dedup_key})
while (r.get("incidents") == None or len(r["incidents"]) < 1
or tries < 30):
print(f"no incident yet; sleeping...")
tries += 1
time.sleep(1)
r = pd.request(api_key=token,
endpoint='incidents',
params={'incident_key': new_dedup_key})
new_incident_id = r["incidents"][0]["id"]
print(f"new incident id is {new_incident_id}")
merge_body = {
"source_incidents": [{
"id": new_incident_id,
"type": "incident_reference"
}]
}
r = pd.request(api_key=token,
endpoint=f"incidents/{incident_id}/merge",
method="PUT",
addheaders={"From": from_email},
data=merge_body)
print(r)
def destroy_escalation_policy(token, ep_id):
""" destroy an escalation policy in PD """
return pd.request(token=token,
endpoint=f"escalation_policies/{ep_id}",
method="DELETE")
