def _all_zeros(data, agg_level):
values = [(not kpi['value'] and not kpi['all']) for row in data['records']
for kpi in row]
retry = False
if agg_level <= 1:
retry = any(values)
else:
retry = all(values)
if retry:
create_metrics_event('ICDS 0s',
'All indicators in program summary equals 0',
aggregation_key='icds_0')
return retry
def server_up(req):
"""
Health check view which can be hooked into server monitoring tools like 'pingdom'
Returns:
HttpResponse("success", status_code=200)
HttpResponse(error_message, status_code=500)
Hit serverup.txt to check all the default enabled services (always_check=True)
Hit serverup.txt?only={check_name} to only check a specific service
Hit serverup.txt?{check_name} to include a non-default check (currently only ``heartbeat``)
"""
only = req.GET.get('only', None)
if only and only in CHECKS:
checks_to_do = [only]
else:
checks_to_do = [
check for check, check_info in CHECKS.items() if
check_info['always_check'] or req.GET.get(check, None) is not None
]
statuses = run_checks(checks_to_do)
failed_checks = [(check, status) for check, status in statuses
if not status.success]
for check_name, status in statuses:
tags = {
'status': 'failed' if not status.success else 'ok',
'check': check_name
}
metrics_gauge('commcare.serverup.check',
status.duration,
tags=tags,
multiprocess_mode=MPM_MAX)
if failed_checks and not is_deploy_in_progress():
status_messages = [
html.linebreaks('<strong>{}</strong>: {}'.format(
check, html.escape(status.msg)).strip())
for check, status in failed_checks
]
create_metrics_event(
'Serverup check failed',
'\n'.join(status_messages),
alert_type='error',
aggregation_key='serverup',
)
status_messages.insert(0, 'Failed Checks (%s):' % os.uname()[1])
return HttpResponse(''.join(status_messages), status=500)
else:
return HttpResponse("success")
def _all_zeros_graph(step, data, agg_level):
if step == 'map':
if agg_level <= 3:
map_data_by_location = data['data']
else:
map_data_by_location = data['tooltips_data']
values = [
not all(map_data_by_location[key].values())
for key in map_data_by_location
if key not in ['original_name', 'fillKey']
]
else:
values = [(not location['value'])
for location in data['all_locations']]
retry = all(values)
if retry:
create_metrics_event('ICDS 0s',
'All indicators in awc_covered equals 0',
aggregation_key='icds_0')
return retry
def handle(self, **options):
compare_url = options.get('url', None)
minutes = options.get('minutes', None)
deploy = HqDeploy(
date=datetime.utcnow(),
user=options['user'],
environment=options['environment'],
diff_url=compare_url,
commit=options['commit']
)
deploy.save()
# reset PillowTop errors in the hope that a fix has been deployed
rows_updated = PillowError.bulk_reset_attempts(datetime.utcnow())
if rows_updated:
print("\n---------------- Pillow Errors Reset ----------------\n" \
"{} pillow errors queued for retry\n".format(rows_updated))
deploy_notification_text = (
"CommCareHQ has been successfully deployed to *{}* by *{}* in *{}* minutes. ".format(
options['environment'],
options['user'],
minutes or '?',
)
)
if options['environment'] == 'production':
deploy_notification_text += "Monitor the {dashboard_link}. "
if settings.MOBILE_INTEGRATION_TEST_TOKEN:
deploy_notification_text += "Check the integration {integration_tests_link}. "
requests.get(
'https://jenkins.dimagi.com/job/integration-tests/build',
params={'token': settings.MOBILE_INTEGRATION_TEST_TOKEN},
)
requests.get(
'https://jenkins.dimagi.com/job/integration-tests-pipeline/build',
params={'token': settings.MOBILE_INTEGRATION_TEST_TOKEN},
)
deploy_notification_text += "Find the diff {diff_link}"
if settings.DATADOG_API_KEY:
link = diff_link(compare_url)
create_metrics_event(
title="Deploy Success",
text=deploy_notification_text.format(
dashboard_link=dashboard_link(DASHBOARD_URL),
diff_link=link,
integration_tests_link=integration_tests_link(INTEGRATION_TEST_URL)
),
tags={'environment': options['environment']},
alert_type="success"
)
print(
"\n=============================================================\n"
"Congratulations! Deploy Complete.\n\n"
"Don't forget to keep an eye on the deploy dashboard to "
"make sure everything is running smoothly.\n\n"
"https://app.datadoghq.com/dashboard/xch-zwt-vzv/hq-deploy-dashboard?tpl_var_environment={}"
"\n=============================================================\n".format(
settings.SERVER_ENVIRONMENT
)
)
if options['mail_admins']:
message_body = get_deploy_email_message_body(user=options['user'], compare_url=compare_url)
subject = 'Deploy Successful - {}'.format(options['environment'])
call_command('mail_admins', message_body, **{'subject': subject, 'html': True})
if settings.DAILY_DEPLOY_EMAIL:
recipient = settings.DAILY_DEPLOY_EMAIL
send_HTML_email(subject=subject,
recipient=recipient,
html_content=message_body)
if settings.SENTRY_CONFIGURED and settings.SENTRY_API_KEY:
create_update_sentry_release()
notify_sentry_deploy(minutes)