def _statuspage_send_email_on_error(subject, content, incident_id=None):
page_id = flask.current_app.config.get('STATUSPAGE_PAGE_ID')
address = flask.current_app.config.get('STATUSPAGE_NOTIFY_ON_ERROR')
if not address or not page_id:
log.error(
'STATUSPAGE_NOTIFY_ON_ERROR and/or STATUSPAGE_PAGE_ID not defined in app config.'
)
return
link = {
'href': f'https://manage.statuspage.io/pages/{page_id}',
'text': 'Visit statuspage',
}
if incident_id:
link = {
'href':
f'https://manage.statuspage.io/pages/{page_id}/incidents/{incident_id}',
'text': 'Visit statuspage incident',
}
flask.current_app.notify.email({
'address': address,
'subject': subject,
'content': content,
'link': link,
})
def _notify_status_change(trees_changes, tags=[]):
if current_app.config.get('PULSE_TREESTATUS_ENABLE'):
routing_key_pattern = 'tree/{0}/status_change'
exchange = current_app.config.get('PULSE_TREESTATUS_EXCHANGE')
for tree_change in trees_changes:
tree, status_from, status_to = tree_change
payload = {'status_from': status_from,
'status_to': status_to,
'tree': tree.to_dict(),
'tags': tags}
routing_key = routing_key_pattern.format(tree.tree)
log.info(
'Sending pulse to {} for tree: {}'.format(
exchange,
tree.tree,
))
try:
current_app.pulse.publish(exchange, routing_key, payload)
except Exception as e:
import traceback
msg = 'Can\'t send notification to pulse.'
trace = traceback.format_exc()
log.error('{0}\nException:{1}\nTraceback: {2}'.format(msg, e, trace)) # noqa
def _statuspage_create_incident(
headers,
component_id,
tree,
status_from,
status_to,
):
page_id = flask.current_app.config.get('STATUSPAGE_PAGE_ID')
if not page_id:
log.error('STATUSPAGE_PAGE_ID not defined in app config.')
return
data = _statuspage_data(
False,
component_id,
tree,
status_from,
status_to,
)
log.debug(
f'Create statuspage incident for tree `{tree.tree}` under page `{page_id}`',
data=data)
response = requests.post(
f'{STATUSPAGE_URL}/pages/{page_id}/incidents',
headers=headers,
json=data,
)
try:
response.raise_for_status()
except Exception as e:
log.exception(e)
_statuspage_send_email_on_error(
subject=f'[treestatus] Error when creating statuspage incident',
content=STATUSPAGE_ERROR_ON_CREATE.format(tree=tree.tree),
)
def get_queue_group_state(task_group_id):
# TODO
# the python taskcluster package doesn't actually support the limit or
# continuationToken query strings, so we can't get a subset or more than
# 1000 tasks
taskgroup = TC_QUEUE.listTaskGroup(task_group_id)
try:
states = Counter(task['status']['state']
for task in taskgroup['tasks'])
except KeyError:
log.error('Could not parse task states from task graph')
return 'exception'
# Example:
# Counter({'completed': 689, 'unscheduled': 161, 'pending': 41, 'running': 19})
if states['exception'] > 0 or states['failed'] > 0:
# failed, exception > 0
return 'failed'
elif list(states.keys()) == ['completed']:
# Only 'completed' states
return 'completed'
else:
# unscheduled, running, pending > 0
return 'running'
def hg_run(cmd):
"""
Run a mercurial command without an hglib instance
Useful for initial custom clones
Redirects stdout & stderr to python's logger
"""
def _log_process(output, name):
# Read and display every line
out = output.read()
if out is None:
return
text = filter(None, out.decode("utf-8").splitlines())
for line in text:
log.info("{}: {}".format(name, line))
# Start process
main_cmd = cmd[0]
proc = hglib.util.popen([hglib.HGPATH] + cmd)
# Set process outputs as non blocking
for output in (proc.stdout, proc.stderr):
fcntl.fcntl(output.fileno(), fcntl.F_SETFL, fcntl.fcntl(output, fcntl.F_GETFL) | os.O_NONBLOCK)
while proc.poll() is None:
_log_process(proc.stdout, main_cmd)
_log_process(proc.stderr, "{} (err)".format(main_cmd))
time.sleep(2)
out, err = proc.communicate()
if proc.returncode != 0:
log.error("Mercurial {} failure".format(main_cmd), out=out, err=err)
raise hglib.error.CommandError(cmd, proc.returncode, out, err)
return out
def _notify_status_change(trees_changes, tags=[]):
if flask.current_app.config.get('PULSE_TREESTATUS_ENABLE'):
routing_key_pattern = 'tree/{0}/status_change'
exchange = flask.current_app.config.get('PULSE_TREESTATUS_EXCHANGE')
for tree_change in trees_changes:
tree, status_from, status_to = tree_change
payload = {'status_from': status_from,
'status_to': status_to,
'tree': tree.to_dict(),
'tags': tags}
routing_key = routing_key_pattern.format(tree.tree)
log.info(
'Sending pulse to {} for tree: {}'.format(
exchange,
tree.tree,
))
try:
flask.current_app.pulse.publish(exchange, routing_key, payload)
except Exception as e:
import traceback
msg = 'Can\'t send notification to pulse.'
trace = traceback.format_exc()
log.error('{0}\nException:{1}\nTraceback: {2}'.format(msg, e, trace)) # noqa
def hg_run(cmd):
'''
Run a mercurial command without an hglib instance
Useful for initial custom clones
Redirects stdout & stderr to python's logger
'''
def _log_process(output, name):
# Read and display every line
out = output.read()
if out is None:
return
text = filter(None, out.decode('utf-8').splitlines())
for line in text:
log.info('{}: {}'.format(name, line))
# Start process
main_cmd = cmd[0]
proc = hglib.util.popen([hglib.HGPATH] + cmd)
# Set process outputs as non blocking
for output in (proc.stdout, proc.stderr):
fcntl.fcntl(
output.fileno(),
fcntl.F_SETFL,
fcntl.fcntl(output, fcntl.F_GETFL) | os.O_NONBLOCK,
)
while proc.poll() is None:
_log_process(proc.stdout, main_cmd)
_log_process(proc.stderr, '{} (err)'.format(main_cmd))
time.sleep(2)
out, err = proc.communicate()
if proc.returncode != 0:
log.error('Mercurial {} failure'.format(main_cmd), out=out, err=err)
raise hglib.error.CommandError(cmd, proc.returncode, out, err)
return out
def _notify_status_change(trees_changes, tags=[]):
if flask.current_app.config.get('STATUSPAGE_ENABLE'):
log.debug('Notify statuspage about trees changes.')
components = flask.current_app.config.get('STATUSPAGE_COMPONENTS', {})
token = flask.current_app.config.get('STATUSPAGE_TOKEN')
if not token:
log.error('STATUSPAGE_PAGE_ID not defined in app config.')
else:
headers = {'Authorization': f'OAuth {token}'}
for tree_change in trees_changes:
tree, status_from, status_to = tree_change
if tree.tree not in components.keys():
continue
log.debug(f'Notify statuspage about: {tree.tree}')
component_id = components[tree.tree]
# create an accident
if status_from in ['open', 'approval required'
] and status_to == 'closed':
_statuspage_create_incident(
headers,
component_id,
tree,
status_from,
status_to,
)
# close an accident
elif status_from == 'closed' and status_to in [
'open', 'approval required'
]:
_statuspage_resolve_incident(
headers,
component_id,
tree,
status_from,
status_to,
)
if flask.current_app.config.get('PULSE_TREESTATUS_ENABLE'):
routing_key_pattern = 'tree/{0}/status_change'
exchange = flask.current_app.config.get('PULSE_TREESTATUS_EXCHANGE')
for tree_change in trees_changes:
tree, status_from, status_to = tree_change
payload = {
'status_from': status_from,
'status_to': status_to,
'tree': tree.to_dict(),
'tags': tags
}
routing_key = routing_key_pattern.format(tree.tree)
log.info('Sending pulse to {} for tree: {}'.format(
exchange,
tree.tree,
))
try:
flask.current_app.pulse.publish(exchange, routing_key, payload)
except Exception as e:
import traceback
msg = 'Can\'t send notification to pulse.'
trace = traceback.format_exc()
log.error('{0}\nException:{1}\nTraceback: {2}'.format(
msg, e, trace)) # noqa
def _statuspage_resolve_incident(
headers,
component_id,
tree,
status_from,
status_to,
):
page_id = flask.current_app.config.get('STATUSPAGE_PAGE_ID')
response = requests.get(
f'{STATUSPAGE_URL}/pages/{page_id}/incidents/unresolved',
headers=headers,
)
try:
response.raise_for_status()
except Exception as e:
log.exception(e)
_statuspage_send_email_on_error(
subject=f'[treestatus] Error when closing statuspage incident',
content=STATUSPAGE_ERROR_ON_CLOSE.format(tree=tree.tree),
)
return
# last incident with meta.treestatus.tree == tree.tree
incident_id = None
incidents = sorted(response.json(), key=lambda x: x['created_at'])
for incident in incidents:
if 'id' in incident and \
'metadata' in incident and \
'treestatus' in incident['metadata'] and \
'tree' in incident['metadata']['treestatus'] and \
incident['metadata']['treestatus']['tree'] == tree.tree:
incident_id = incident['id']
break
if incident_id is None:
log.error(f'No incident found when closing tree `{tree.tree}`')
_statuspage_send_email_on_error(
subject=f'[treestatus] Error when closing statuspage incident',
content=STATUSPAGE_ERROR_ON_CLOSE.format(tree=tree.tree),
)
return
response = requests.patch(
f'{STATUSPAGE_URL}/pages/{page_id}/incidents/{incident_id}',
headers=headers,
json=_statuspage_data(
True,
component_id,
tree,
status_from,
status_to,
),
)
try:
response.raise_for_status()
except Exception as e:
log.exception(e)
_statuspage_send_email_on_error(
subject=f'[treestatus] Error when closing statuspage incident',
content=STATUSPAGE_ERROR_ON_CLOSE.format(tree=tree.tree),
incident_id=incident_id,
)