def _all_zeros(data, agg_level): values = [(not kpi['value'] and not kpi['all']) for row in data['records'] for kpi in row] retry = False if agg_level <= 1: retry = any(values) else: retry = all(values) if retry: create_metrics_event('ICDS 0s', 'All indicators in program summary equals 0', aggregation_key='icds_0') return retry
def server_up(req): """ Health check view which can be hooked into server monitoring tools like 'pingdom' Returns: HttpResponse("success", status_code=200) HttpResponse(error_message, status_code=500) Hit serverup.txt to check all the default enabled services (always_check=True) Hit serverup.txt?only={check_name} to only check a specific service Hit serverup.txt?{check_name} to include a non-default check (currently only ``heartbeat``) """ only = req.GET.get('only', None) if only and only in CHECKS: checks_to_do = [only] else: checks_to_do = [ check for check, check_info in CHECKS.items() if check_info['always_check'] or req.GET.get(check, None) is not None ] statuses = run_checks(checks_to_do) failed_checks = [(check, status) for check, status in statuses if not status.success] for check_name, status in statuses: tags = { 'status': 'failed' if not status.success else 'ok', 'check': check_name } metrics_gauge('commcare.serverup.check', status.duration, tags=tags, multiprocess_mode=MPM_MAX) if failed_checks and not is_deploy_in_progress(): status_messages = [ html.linebreaks('<strong>{}</strong>: {}'.format( check, html.escape(status.msg)).strip()) for check, status in failed_checks ] create_metrics_event( 'Serverup check failed', '\n'.join(status_messages), alert_type='error', aggregation_key='serverup', ) status_messages.insert(0, 'Failed Checks (%s):' % os.uname()[1]) return HttpResponse(''.join(status_messages), status=500) else: return HttpResponse("success")
def _all_zeros_graph(step, data, agg_level): if step == 'map': if agg_level <= 3: map_data_by_location = data['data'] else: map_data_by_location = data['tooltips_data'] values = [ not all(map_data_by_location[key].values()) for key in map_data_by_location if key not in ['original_name', 'fillKey'] ] else: values = [(not location['value']) for location in data['all_locations']] retry = all(values) if retry: create_metrics_event('ICDS 0s', 'All indicators in awc_covered equals 0', aggregation_key='icds_0') return retry
def handle(self, **options): compare_url = options.get('url', None) minutes = options.get('minutes', None) deploy = HqDeploy( date=datetime.utcnow(), user=options['user'], environment=options['environment'], diff_url=compare_url, commit=options['commit'] ) deploy.save() # reset PillowTop errors in the hope that a fix has been deployed rows_updated = PillowError.bulk_reset_attempts(datetime.utcnow()) if rows_updated: print("\n---------------- Pillow Errors Reset ----------------\n" \ "{} pillow errors queued for retry\n".format(rows_updated)) deploy_notification_text = ( "CommCareHQ has been successfully deployed to *{}* by *{}* in *{}* minutes. ".format( options['environment'], options['user'], minutes or '?', ) ) if options['environment'] == 'production': deploy_notification_text += "Monitor the {dashboard_link}. " if settings.MOBILE_INTEGRATION_TEST_TOKEN: deploy_notification_text += "Check the integration {integration_tests_link}. " requests.get( 'https://jenkins.dimagi.com/job/integration-tests/build', params={'token': settings.MOBILE_INTEGRATION_TEST_TOKEN}, ) requests.get( 'https://jenkins.dimagi.com/job/integration-tests-pipeline/build', params={'token': settings.MOBILE_INTEGRATION_TEST_TOKEN}, ) deploy_notification_text += "Find the diff {diff_link}" if settings.DATADOG_API_KEY: link = diff_link(compare_url) create_metrics_event( title="Deploy Success", text=deploy_notification_text.format( dashboard_link=dashboard_link(DASHBOARD_URL), diff_link=link, integration_tests_link=integration_tests_link(INTEGRATION_TEST_URL) ), tags={'environment': options['environment']}, alert_type="success" ) print( "\n=============================================================\n" "Congratulations! Deploy Complete.\n\n" "Don't forget to keep an eye on the deploy dashboard to " "make sure everything is running smoothly.\n\n" "https://app.datadoghq.com/dashboard/xch-zwt-vzv/hq-deploy-dashboard?tpl_var_environment={}" "\n=============================================================\n".format( settings.SERVER_ENVIRONMENT ) ) if options['mail_admins']: message_body = get_deploy_email_message_body(user=options['user'], compare_url=compare_url) subject = 'Deploy Successful - {}'.format(options['environment']) call_command('mail_admins', message_body, **{'subject': subject, 'html': True}) if settings.DAILY_DEPLOY_EMAIL: recipient = settings.DAILY_DEPLOY_EMAIL send_HTML_email(subject=subject, recipient=recipient, html_content=message_body) if settings.SENTRY_CONFIGURED and settings.SENTRY_API_KEY: create_update_sentry_release() notify_sentry_deploy(minutes)