def handle(self, *args, **options): root_dir = settings.FILEPATH git_snapshot = gitinfo.get_project_snapshot(root_dir, submodules=True) git_snapshot['diff_url'] = options.get('url', None) deploy = HqDeploy( date=datetime.utcnow(), user=options['user'], environment=options['environment'], code_snapshot=git_snapshot, ) deploy.save() # reset PillowTop errors in the hope that a fix has been deployed rows_updated = PillowError.bulk_reset_attempts(datetime.utcnow()) if rows_updated: print "\n---------------- Pillow Errors Reset ----------------\n" \ "{} pillow errors queued for retry\n".format(rows_updated) if options['mail_admins']: snapshot_table = render_to_string('hqadmin/partials/project_snapshot.html', dictionary={'snapshot': git_snapshot}) message = "Deployed by %s, cheers!" % options['user'] snapshot_body = "<html><head><title>Deploy Snapshot</title></head><body><h2>%s</h2>%s</body></html>" % (message, snapshot_table) call_command('mail_admins', snapshot_body, **{'subject': 'Deploy successful', 'html': True})
def handle(self, *args, **options): deploy = HqDeploy( date=datetime.utcnow(), user=options['user'] ) deploy.save()
def handle(self, *args, **options): root_dir = settings.FILEPATH git_snapshot = gitinfo.get_project_snapshot(root_dir, submodules=True) compare_url = git_snapshot['diff_url'] = options.get('url', None) deploy = HqDeploy( date=datetime.utcnow(), user=options['user'], environment=options['environment'], code_snapshot=git_snapshot, ) deploy.save() # reset PillowTop errors in the hope that a fix has been deployed rows_updated = PillowError.bulk_reset_attempts(datetime.utcnow()) if rows_updated: print "\n---------------- Pillow Errors Reset ----------------\n" \ "{} pillow errors queued for retry\n".format(rows_updated) deploy_notification_text = ( "CommCareHQ has been successfully deployed to *{}* by *{}*. " "Find the diff {{diff_link}}".format( options['environment'], options['user'], ) ) if hasattr(settings, 'MIA_THE_DEPLOY_BOT_API'): link = diff_link(STYLE_SLACK, compare_url) requests.post(settings.MIA_THE_DEPLOY_BOT_API, data=json.dumps({ "username": "******", "text": deploy_notification_text.format(diff_link=link), })) if settings.DATADOG_API_KEY: tags = ['environment:{}'.format(options['environment'])] link = diff_link(STYLE_MARKDOWN, compare_url) datadog_api.Event.create( title="Deploy Success", text=deploy_notification_text.format(diff_link=link), tags=tags, alert_type="success" ) print "\n=============================================================\n" \ "Congratulations! Deploy Complete.\n\n" \ "Don't forget to keep an eye on the deploy dashboard to " \ "make sure everything is running smoothly.\n\n" \ "https://p.datadoghq.com/sb/5c4af2ac8-1f739e93ef" \ "\n=============================================================\n" if options['mail_admins']: message_body = get_deploy_email_message_body( environment=options['environment'], user=options['user'], compare_url=compare_url) call_command('mail_admins', message_body, **{'subject': 'Deploy successful', 'html': True})
def handle(self, *args, **options): root_dir = settings.FILEPATH git_snapshot = gitinfo.get_project_snapshot(root_dir, submodules=True) git_snapshot['diff_url'] = options.get('url', None) deploy = HqDeploy( date=datetime.utcnow(), user=options['user'], environment=options['environment'], code_snapshot=git_snapshot, ) deploy.save() # reset PillowTop errors in the hope that a fix has been deployed rows_updated = PillowError.bulk_reset_attempts(datetime.utcnow()) if rows_updated: print "\n---------------- Pillow Errors Reset ----------------\n" \ "{} pillow errors queued for retry\n".format(rows_updated) deploy_notification_text = ( "CommCareHQ has been successfully deployed to *{}* by *{}*. " "Find the diff {{diff_link}}".format( options['environment'], options['user'], ) ) if hasattr(settings, 'MIA_THE_DEPLOY_BOT_API'): link = diff_link(STYLE_SLACK, git_snapshot['diff_url']) requests.post(settings.MIA_THE_DEPLOY_BOT_API, data=json.dumps({ "channel": "#dev", "username": "******", "text": deploy_notification_text.format(diff_link=link), "icon_emoji": ":see_no_evil:" })) if settings.DATADOG_API_KEY: tags = ['environment:{}'.format(options['environment'])] link = diff_link(STYLE_MARKDOWN, git_snapshot['diff_url']) datadog_api.Event.create( title="Deploy Success", text=deploy_notification_text.format(diff_link=link), tags=tags ) if options['mail_admins']: snapshot_table = render_to_string('hqadmin/partials/project_snapshot.html', dictionary={'snapshot': git_snapshot}) message = "Deployed by %s, cheers!" % options['user'] snapshot_body = "<html><head><title>Deploy Snapshot</title></head><body><h2>%s</h2>%s</body></html>" % (message, snapshot_table) call_command('mail_admins', snapshot_body, **{'subject': 'Deploy successful', 'html': True})
def system_info(request): environment = settings.SERVER_ENVIRONMENT context = get_hqadmin_base_context(request) context['couch_update'] = request.GET.get('couch_update', 5000) context['celery_update'] = request.GET.get('celery_update', 10000) context['celery_flower_url'] = getattr(settings, 'CELERY_FLOWER_URL', None) # recent changes recent_changes = int(request.GET.get('changes', 50)) context['recent_changes'] = get_recent_changes(get_db(), recent_changes) context['rabbitmq_url'] = get_rabbitmq_management_url() context['hide_filters'] = True context['current_system'] = socket.gethostname() context['last_deploy'] = HqDeploy.get_latest(environment) context['snapshot'] = context['last_deploy'].code_snapshot if context[ 'last_deploy'] else {} context.update(check_redis()) context.update(check_rabbitmq()) context.update(check_celery_health()) context.update(check_memcached()) context.update(check_es_cluster_health()) return render(request, "hqadmin/system_info.html", context)
def page_context(self): environment = settings.SERVER_ENVIRONMENT context = get_hqadmin_base_context(self.request) context['couch_update'] = self.request.GET.get('couch_update', 5000) context['celery_update'] = self.request.GET.get('celery_update', 10000) context['db_update'] = self.request.GET.get('db_update', 30000) context['self.request'] = getattr(settings, 'CELERY_FLOWER_URL', None) context['is_bigcouch'] = is_bigcouch() context['rabbitmq_url'] = get_rabbitmq_management_url() context['hide_filters'] = True context['current_system'] = socket.gethostname() context['deploy_history'] = HqDeploy.get_latest(environment, limit=5) context['user_is_support'] = hasattr(self.request, 'user') and SUPPORT.enabled( self.request.user.username) context['redis'] = service_checks.check_redis() context['rabbitmq'] = service_checks.check_rabbitmq() context['celery_stats'] = get_celery_stats() context['heartbeat'] = service_checks.check_heartbeat() context['cluster_health'] = escheck.check_es_cluster_health() return context
def page_context(self): environment = settings.SERVER_ENVIRONMENT context = get_hqadmin_base_context(self.request) context['couch_update'] = self.request.GET.get('couch_update', 5000) context['celery_update'] = self.request.GET.get('celery_update', 10000) context['db_update'] = self.request.GET.get('db_update', 30000) context['self.request'] = getattr(settings, 'CELERY_FLOWER_URL', None) context['is_bigcouch'] = is_bigcouch() context['rabbitmq_url'] = get_rabbitmq_management_url() context['hide_filters'] = True context['current_system'] = socket.gethostname() context['deploy_history'] = HqDeploy.get_latest(environment, limit=5) context['user_is_support'] = hasattr(self.request, 'user') and SUPPORT.enabled(self.request.user.username) context['redis'] = service_checks.check_redis() context['rabbitmq'] = service_checks.check_rabbitmq() context['celery_stats'] = get_celery_stats() context['heartbeat'] = service_checks.check_heartbeat() context['cluster_health'] = escheck.check_es_cluster_health() return context
def handle(self, *args, **options): start = parser.parse(options['startdate']) enddate = options['enddate'] end = parser.parse(enddate) if enddate else datetime.now() ds = HqDeploy.get_list('production', start, end) ids = [d['id'] for d in ds] sha_prev = None print_row('Deploy Date', "Commit Date", "Diff") for id in ids: d = HqDeploy.get(id) s = d.code_snapshot['commits'][0] sha = s['sha'] url = "https://github.com/dimagi/commcare-hq/compare/{sha_prev}...{sha}".format( sha=sha, sha_prev=sha_prev) print_row(d.date, s['date'], url) sha_prev = sha
def handle(self, *args, **options): root_dir = settings.FILEPATH git_snapshot = gitinfo.get_project_snapshot(root_dir, submodules=True) deploy = HqDeploy( date=datetime.utcnow(), user=options['user'], environment=options['environment'], code_snapshot=git_snapshot, ) deploy.save() if options['mail_admins']: snapshot_table = render_to_string('hqadmin/partials/project_snapshot.html', dictionary={'snapshot': git_snapshot}) message = "Deployed by %s, cheers!" % options['user'] snapshot_body = "<html><head><title>Deploy Snapshot</title></head><body><h2>%s</h2>%s</body></html>" % (message, snapshot_table) call_command('mail_admins', snapshot_body, **{'subject': 'Deploy successful', 'html': True})
def loadtest(request): # The multimech results api is kinda all over the place. # the docs are here: http://testutils.org/multi-mechanize/datastore.html scripts = ["submit_form.py", "ota_restore.py"] tests = [] # datetime info seems to be buried in GlobalConfig.results[0].run_id, # which makes ORM-level sorting problematic for gc in Session.query(GlobalConfig).all()[::-1]: gc.scripts = dict((uc.script, uc) for uc in gc.user_group_configs) if gc.results: for script, uc in gc.scripts.items(): uc.results = filter(lambda res: res.user_group_name == uc.user_group, gc.results) test = {"datetime": gc.results[0].run_id, "run_time": gc.run_time, "results": gc.results} for script in scripts: test[script.split(".")[0]] = gc.scripts.get(script) tests.append(test) context = get_hqadmin_base_context(request) context.update({"tests": tests, "hide_filters": True}) date_axis = Axis(label="Date", dateFormat="%m/%d/%Y") tests_axis = Axis(label="Number of Tests in 30s") chart = LineChart("HQ Load Test Performance", date_axis, tests_axis) submit_data = [] ota_data = [] total_data = [] max_val = 0 max_date = None min_date = None for test in tests: date = test["datetime"] total = len(test["results"]) max_val = total if total > max_val else max_val max_date = date if not max_date or date > max_date else max_date min_date = date if not min_date or date < min_date else min_date submit_data.append({"x": date, "y": len(test["submit_form"].results)}) ota_data.append({"x": date, "y": len(test["ota_restore"].results)}) total_data.append({"x": date, "y": total}) deployments = [row["key"][1] for row in HqDeploy.get_list(settings.SERVER_ENVIRONMENT, min_date, max_date)] deploy_data = [{"x": min_date, "y": 0}] for date in deployments: deploy_data.extend([{"x": date, "y": 0}, {"x": date, "y": max_val}, {"x": date, "y": 0}]) deploy_data.append({"x": max_date, "y": 0}) chart.add_dataset("Deployments", deploy_data) chart.add_dataset("Form Submission Count", submit_data) chart.add_dataset("OTA Restore Count", ota_data) chart.add_dataset("Total Count", total_data) context["charts"] = [chart] template = "hqadmin/loadtest.html" return render(request, template, context)
def handle(self, *args, **options): start = parser.parse(options['startdate']) enddate = options['enddate'] end = parser.parse(enddate) if enddate else datetime.utcnow() ds = HqDeploy.get_list('production', start, end) ids = [d['id'] for d in ds] sha_prev = None print_row('Deploy Date', "Commit Date", "Diff") for id in ids: d = HqDeploy.get(id) s = d.code_snapshot['commits'][0] sha = s['sha'] url = "https://github.com/dimagi/commcare-hq/compare/{sha_prev}...{sha}".format( sha=sha, sha_prev=sha_prev ) print_row(d.date, s['date'], url) sha_prev = sha
def system_info(request): def human_bytes(bytes): #source: https://github.com/bartTC/django-memcache-status bytes = float(bytes) if bytes >= 1073741824: gigabytes = bytes / 1073741824 size = '%.2fGB' % gigabytes elif bytes >= 1048576: megabytes = bytes / 1048576 size = '%.2fMB' % megabytes elif bytes >= 1024: kilobytes = bytes / 1024 size = '%.2fKB' % kilobytes else: size = '%.2fB' % bytes return size context = get_hqadmin_base_context(request) context['couch_update'] = request.GET.get('couch_update', 5000) context['celery_update'] = request.GET.get('celery_update', 10000) context['celery_flower_url'] = getattr(settings, 'CELERY_FLOWER_URL', None) # recent changes recent_changes = int(request.GET.get('changes', 50)) context['recent_changes'] = get_recent_changes(get_db(), recent_changes) context['rabbitmq_url'] = get_rabbitmq_management_url() context['hide_filters'] = True context['current_system'] = socket.gethostname() environment = settings.SERVER_ENVIRONMENT context['last_deploy'] = HqDeploy.get_latest(environment) context['snapshot'] = context['last_deploy'].code_snapshot if context['last_deploy'] else {} #redis status redis_status = "" redis_results = "" if 'redis' in settings.CACHES: rc = cache.get_cache('redis') try: import redis redis_api = redis.StrictRedis.from_url('redis://%s' % rc._server) info_dict = redis_api.info() redis_status = "Online" redis_results = "Used Memory: %s" % info_dict['used_memory_human'] except Exception, ex: redis_status = "Offline" redis_results = "Redis connection error: %s" % ex
def handle(self, *args, **options): root_dir = settings.FILEPATH git_snapshot = gitinfo.get_project_snapshot(root_dir, submodules=True) deploy = HqDeploy( date=datetime.utcnow(), user=options['user'], environment=options['environment'], code_snapshot=git_snapshot, ) deploy.save() if options['mail_admins']: snapshot_table = render_to_string( 'hqadmin/partials/project_snapshot.html', dictionary={'snapshot': git_snapshot}) message = "Deployed by %s, cheers!" % options['user'] snapshot_body = "<html><head><title>Deploy Snapshot</title></head><body><h2>%s</h2>%s</body></html>" % ( message, snapshot_table) call_command('mail_admins', snapshot_body, **{ 'subject': 'Deploy successful', 'html': True })
def system_info(request): environment = settings.SERVER_ENVIRONMENT context = get_hqadmin_base_context(request) context['couch_update'] = request.GET.get('couch_update', 5000) context['celery_update'] = request.GET.get('celery_update', 10000) context['db_update'] = request.GET.get('db_update', 30000) context['celery_flower_url'] = getattr(settings, 'CELERY_FLOWER_URL', None) context['is_bigcouch'] = is_bigcouch() context['rabbitmq_url'] = get_rabbitmq_management_url() context['hide_filters'] = True context['current_system'] = socket.gethostname() context['deploy_history'] = HqDeploy.get_latest(environment, limit=5) context.update(check_redis()) context.update(check_rabbitmq()) context.update(check_celery_health()) context.update(check_es_cluster_health()) return render(request, "hqadmin/system_info.html", context)
def system_info(request): environment = settings.SERVER_ENVIRONMENT context = get_hqadmin_base_context(request) context['couch_update'] = request.GET.get('couch_update', 5000) context['celery_update'] = request.GET.get('celery_update', 10000) context['celery_flower_url'] = getattr(settings, 'CELERY_FLOWER_URL', None) # recent changes recent_changes = int(request.GET.get('changes', 50)) context['recent_changes'] = get_recent_changes(get_db(), recent_changes) context['rabbitmq_url'] = get_rabbitmq_management_url() context['hide_filters'] = True context['current_system'] = socket.gethostname() context['last_deploy'] = HqDeploy.get_latest(environment) context['snapshot'] = context['last_deploy'].code_snapshot if context['last_deploy'] else {} context.update(check_redis()) context.update(check_rabbitmq()) context.update(check_celery_health()) context.update(check_memcached()) context.update(check_es_cluster_health()) return render(request, "hqadmin/system_info.html", context)
def handle(self, **options): compare_url = options.get('url', None) minutes = options.get('minutes', None) deploy = HqDeploy(date=datetime.utcnow(), user=options['user'], environment=options['environment'], diff_url=compare_url) deploy.save() # reset PillowTop errors in the hope that a fix has been deployed rows_updated = PillowError.bulk_reset_attempts(datetime.utcnow()) if rows_updated: print("\n---------------- Pillow Errors Reset ----------------\n" \ "{} pillow errors queued for retry\n".format(rows_updated)) deploy_notification_text = ( "CommCareHQ has been successfully deployed to *{}* by *{}* in *{}* minutes. " .format( options['environment'], options['user'], minutes or '?', )) if options['environment'] == 'production': deploy_notification_text += "Monitor the {dashboard_link}. " if settings.MOBILE_INTEGRATION_TEST_TOKEN: deploy_notification_text += "Check the integration {integration_tests_link}. " requests.get( 'https://jenkins.dimagi.com/job/integration-tests/build', params={'token': settings.MOBILE_INTEGRATION_TEST_TOKEN}, ) requests.get( 'https://jenkins.dimagi.com/job/integration-tests-pipeline/build', params={'token': settings.MOBILE_INTEGRATION_TEST_TOKEN}, ) deploy_notification_text += "Find the diff {diff_link}" if hasattr(settings, 'MIA_THE_DEPLOY_BOT_API'): link = diff_link(STYLE_SLACK, compare_url) if options['environment'] == 'staging': channel = '#staging' elif options['environment'] == 'icds': channel = '#nic-server-standup' else: channel = '#hq-ops' requests.post( settings.MIA_THE_DEPLOY_BOT_API, data=json.dumps({ "username": "******", "channel": channel, "text": deploy_notification_text.format( dashboard_link=dashboard_link(STYLE_SLACK, DASHBOARD_URL), diff_link=link, integration_tests_link=integration_tests_link( STYLE_SLACK, INTEGRATION_TEST_URL)), })) if settings.DATADOG_API_KEY: tags = ['environment:{}'.format(options['environment'])] link = diff_link(STYLE_MARKDOWN, compare_url) datadog_api.Event.create( title="Deploy Success", text=deploy_notification_text.format( dashboard_link=dashboard_link(STYLE_MARKDOWN, DASHBOARD_URL), diff_link=link, integration_tests_link=integration_tests_link( STYLE_MARKDOWN, INTEGRATION_TEST_URL)), tags=tags, alert_type="success") print("\n=============================================================\n" \ "Congratulations! Deploy Complete.\n\n" \ "Don't forget to keep an eye on the deploy dashboard to " \ "make sure everything is running smoothly.\n\n" \ "https://p.datadoghq.com/sb/5c4af2ac8-1f739e93ef" \ "\n=============================================================\n") if options['mail_admins']: message_body = get_deploy_email_message_body( user=options['user'], compare_url=compare_url) call_command('mail_admins', message_body, **{ 'subject': 'Deploy successful', 'html': True }) if settings.DAILY_DEPLOY_EMAIL: recipient = settings.DAILY_DEPLOY_EMAIL subject = 'Deploy Successful - {}'.format( options['environment']) send_HTML_email(subject=subject, recipient=recipient, html_content=message_body) if settings.SENTRY_CONFIGURED and settings.SENTRY_API_KEY: create_update_sentry_release() notify_sentry_deploy(minutes)
if len(mc_stats) > 0: mc_status = "Online" stats_dict = mc_stats[0][1] bytes = stats_dict['bytes'] max_bytes = stats_dict['limit_maxbytes'] curr_items = stats_dict['curr_items'] mc_results = "%s Items %s out of %s" % (curr_items, human_bytes(bytes), human_bytes(max_bytes)) except Exception, ex: mc_status = "Offline" mc_results = "%s" % ex context['memcached_status'] = mc_status context['memcached_results'] = mc_results context['last_deploy'] = HqDeploy.get_latest() #elasticsearch status #node status context.update(check_cluster_health()) context.update(check_case_index()) context.update(check_xform_index()) context.update(check_exchange_index()) return render(request, "hqadmin/system_info.html", context) @require_superuser def noneulized_users(request, template="hqadmin/noneulized_users.html"): context = get_hqadmin_base_context(request) days = request.GET.get("days", None)
mc_stats = mc._cache.get_stats() if len(mc_stats) > 0: mc_status = "Online" stats_dict = mc_stats[0][1] bytes = stats_dict["bytes"] max_bytes = stats_dict["limit_maxbytes"] curr_items = stats_dict["curr_items"] mc_results = "%s Items %s out of %s" % (curr_items, human_bytes(bytes), human_bytes(max_bytes)) except Exception, ex: mc_status = "Offline" mc_results = "%s" % ex context["memcached_status"] = mc_status context["memcached_results"] = mc_results context["last_deploy"] = HqDeploy.get_latest() # elasticsearch status # node status context.update(check_cluster_health()) context.update(check_case_index()) context.update(check_xform_index()) context.update(check_exchange_index()) return render(request, "hqadmin/system_info.html", context) @require_superuser def noneulized_users(request, template="hqadmin/noneulized_users.html"): context = get_hqadmin_base_context(request)
def handle(self, *args, **options): deploy = HqDeploy(date=datetime.utcnow(), user=options['user']) deploy.save()
def handle(self, *args, **options): compare_url = options.get('url', None) minutes = options.get('minutes', None) deploy = HqDeploy(date=datetime.utcnow(), user=options['user'], environment=options['environment'], diff_url=compare_url) deploy.save() # reset PillowTop errors in the hope that a fix has been deployed rows_updated = PillowError.bulk_reset_attempts(datetime.utcnow()) if rows_updated: print "\n---------------- Pillow Errors Reset ----------------\n" \ "{} pillow errors queued for retry\n".format(rows_updated) deploy_notification_text = ( "CommCareHQ has been successfully deployed to *{}* by *{}* in *{}* minutes. " "Monitor the {{dashboard_link}}. " "Check the integration {{integration_tests_link}}. " "Find the diff {{diff_link}}".format( options['environment'], options['user'], minutes or '?', )) if settings.MOBILE_INTEGRATION_TEST_TOKEN: requests.get( 'https://jenkins.dimagi.com/job/integration-tests/build', params={'token': settings.MOBILE_INTEGRATION_TEST_TOKEN}, ) if hasattr(settings, 'MIA_THE_DEPLOY_BOT_API'): link = diff_link(STYLE_SLACK, compare_url) requests.post( settings.MIA_THE_DEPLOY_BOT_API, data=json.dumps({ "username": "******", "text": deploy_notification_text.format( dashboard_link=dashboard_link(STYLE_SLACK, DASHBOARD_URL), diff_link=link, integration_tests_link=integration_tests_link( STYLE_SLACK, INTEGRATION_TEST_URL)), })) if settings.DATADOG_API_KEY: tags = ['environment:{}'.format(options['environment'])] link = diff_link(STYLE_MARKDOWN, compare_url) datadog_api.Event.create( title="Deploy Success", text=deploy_notification_text.format( dashboard_link=dashboard_link(STYLE_MARKDOWN, DASHBOARD_URL), diff_link=link, integration_tests_link=integration_tests_link( STYLE_MARKDOWN, INTEGRATION_TEST_URL)), tags=tags, alert_type="success") print "\n=============================================================\n" \ "Congratulations! Deploy Complete.\n\n" \ "Don't forget to keep an eye on the deploy dashboard to " \ "make sure everything is running smoothly.\n\n" \ "https://p.datadoghq.com/sb/5c4af2ac8-1f739e93ef" \ "\n=============================================================\n" if options['mail_admins']: message_body = get_deploy_email_message_body( environment=options['environment'], user=options['user'], compare_url=compare_url) call_command('mail_admins', message_body, **{ 'subject': 'Deploy successful', 'html': True })
def handle(self, **options): compare_url = options.get('url', None) minutes = options.get('minutes', None) deploy = HqDeploy( date=datetime.utcnow(), user=options['user'], environment=options['environment'], diff_url=compare_url, commit=options['commit'] ) deploy.save() # reset PillowTop errors in the hope that a fix has been deployed rows_updated = PillowError.bulk_reset_attempts(datetime.utcnow()) if rows_updated: print("\n---------------- Pillow Errors Reset ----------------\n" \ "{} pillow errors queued for retry\n".format(rows_updated)) deploy_notification_text = ( "CommCareHQ has been successfully deployed to *{}* by *{}* in *{}* minutes. ".format( options['environment'], options['user'], minutes or '?', ) ) if options['environment'] == 'production': deploy_notification_text += "Monitor the {dashboard_link}. " if settings.MOBILE_INTEGRATION_TEST_TOKEN: deploy_notification_text += "Check the integration {integration_tests_link}. " requests.get( 'https://jenkins.dimagi.com/job/integration-tests/build', params={'token': settings.MOBILE_INTEGRATION_TEST_TOKEN}, ) requests.get( 'https://jenkins.dimagi.com/job/integration-tests-pipeline/build', params={'token': settings.MOBILE_INTEGRATION_TEST_TOKEN}, ) deploy_notification_text += "Find the diff {diff_link}" if settings.DATADOG_API_KEY: link = diff_link(compare_url) create_metrics_event( title="Deploy Success", text=deploy_notification_text.format( dashboard_link=dashboard_link(DASHBOARD_URL), diff_link=link, integration_tests_link=integration_tests_link(INTEGRATION_TEST_URL) ), tags={'environment': options['environment']}, alert_type="success" ) print( "\n=============================================================\n" "Congratulations! Deploy Complete.\n\n" "Don't forget to keep an eye on the deploy dashboard to " "make sure everything is running smoothly.\n\n" "https://app.datadoghq.com/dashboard/xch-zwt-vzv/hq-deploy-dashboard?tpl_var_environment={}" "\n=============================================================\n".format( settings.SERVER_ENVIRONMENT ) ) if options['mail_admins']: message_body = get_deploy_email_message_body(user=options['user'], compare_url=compare_url) subject = 'Deploy Successful - {}'.format(options['environment']) call_command('mail_admins', message_body, **{'subject': subject, 'html': True}) if settings.DAILY_DEPLOY_EMAIL: recipient = settings.DAILY_DEPLOY_EMAIL send_HTML_email(subject=subject, recipient=recipient, html_content=message_body) if settings.SENTRY_CONFIGURED and settings.SENTRY_API_KEY: create_update_sentry_release() notify_sentry_deploy(minutes)
def loadtest(request): # The multimech results api is kinda all over the place. # the docs are here: http://testutils.org/multi-mechanize/datastore.html db_settings = settings.DATABASES["default"] db_settings['PORT'] = db_settings.get('PORT', '') or '5432' db_url = "postgresql://{USER}:{PASSWORD}@{HOST}:{PORT}/{NAME}".format( **db_settings) engine = create_engine(db_url) session = sessionmaker(bind=engine) current = session() scripts = ['submit_form.py', 'ota_restore.py'] tests = [] # datetime info seems to be buried in GlobalConfig.results[0].run_id, # which makes ORM-level sorting problematic for gc in current.query(GlobalConfig).all()[::-1]: gc.scripts = dict((uc.script, uc) for uc in gc.user_group_configs) if gc.results: for script, uc in gc.scripts.items(): uc.results = filter( lambda res: res.user_group_name == uc.user_group, gc.results) test = { 'datetime': gc.results[0].run_id, 'run_time': gc.run_time, 'results': gc.results, } for script in scripts: test[script.split('.')[0]] = gc.scripts.get(script) tests.append(test) context = get_hqadmin_base_context(request) context.update({ "tests": tests, "hide_filters": True, }) date_axis = Axis(label="Date", dateFormat="%m/%d/%Y") tests_axis = Axis(label="Number of Tests in 30s") chart = LineChart("HQ Load Test Performance", date_axis, tests_axis) submit_data = [] ota_data = [] total_data = [] max_val = 0 max_date = None min_date = None for test in tests: date = test['datetime'] total = len(test['results']) max_val = total if total > max_val else max_val max_date = date if not max_date or date > max_date else max_date min_date = date if not min_date or date < min_date else min_date submit_data.append({'x': date, 'y': len(test['submit_form'].results)}) ota_data.append({'x': date, 'y': len(test['ota_restore'].results)}) total_data.append({'x': date, 'y': total}) deployments = [ row['key'][1] for row in HqDeploy.get_list(settings.SERVER_ENVIRONMENT, min_date, max_date) ] deploy_data = [{'x': min_date, 'y': 0}] for date in deployments: deploy_data.extend([{ 'x': date, 'y': 0 }, { 'x': date, 'y': max_val }, { 'x': date, 'y': 0 }]) deploy_data.append({'x': max_date, 'y': 0}) chart.add_dataset("Deployments", deploy_data) chart.add_dataset("Form Submission Count", submit_data) chart.add_dataset("OTA Restore Count", ota_data) chart.add_dataset("Total Count", total_data) context['charts'] = [chart] template = "hqadmin/loadtest.html" return render(request, template, context)
def loadtest(request): # The multimech results api is kinda all over the place. # the docs are here: http://testutils.org/multi-mechanize/datastore.html db_settings = settings.DATABASES["default"] db_settings['PORT'] = db_settings.get('PORT', '') or '5432' db_url = "postgresql://{USER}:{PASSWORD}@{HOST}:{PORT}/{NAME}".format( **db_settings ) engine = create_engine(db_url) session = sessionmaker(bind=engine) current = session() scripts = ['submit_form.py', 'ota_restore.py'] tests = [] # datetime info seems to be buried in GlobalConfig.results[0].run_id, # which makes ORM-level sorting problematic for gc in current.query(GlobalConfig).all()[::-1]: gc.scripts = dict((uc.script, uc) for uc in gc.user_group_configs) if gc.results: for script, uc in gc.scripts.items(): uc.results = filter( lambda res: res.user_group_name == uc.user_group, gc.results ) test = { 'datetime': gc.results[0].run_id, 'run_time': gc.run_time, 'results': gc.results, } for script in scripts: test[script.split('.')[0]] = gc.scripts.get(script) tests.append(test) context = get_hqadmin_base_context(request) context.update({ "tests": tests, "hide_filters": True, }) date_axis = Axis(label="Date", dateFormat="%m/%d/%Y") tests_axis = Axis(label="Number of Tests in 30s") chart = LineChart("HQ Load Test Performance", date_axis, tests_axis) submit_data = [] ota_data = [] total_data = [] max_val = 0 max_date = None min_date = None for test in tests: date = test['datetime'] total = len(test['results']) max_val = total if total > max_val else max_val max_date = date if not max_date or date > max_date else max_date min_date = date if not min_date or date < min_date else min_date submit_data.append({'x': date, 'y': len(test['submit_form'].results)}) ota_data.append({'x': date, 'y': len(test['ota_restore'].results)}) total_data.append({'x': date, 'y': total}) deployments = [row['key'][1] for row in HqDeploy.get_list(settings.SERVER_ENVIRONMENT, min_date, max_date)] deploy_data = [{'x': min_date, 'y': 0}] for date in deployments: deploy_data.extend([{'x': date, 'y': 0}, {'x': date, 'y': max_val}, {'x': date, 'y': 0}]) deploy_data.append({'x': max_date, 'y': 0}) chart.add_dataset("Deployments", deploy_data) chart.add_dataset("Form Submission Count", submit_data) chart.add_dataset("OTA Restore Count", ota_data) chart.add_dataset("Total Count", total_data) context['charts'] = [chart] template = "hqadmin/loadtest.html" return render(request, template, context)
if len(mc_stats) > 0: mc_status = "Online" stats_dict = mc_stats[0][1] bytes = stats_dict['bytes'] max_bytes = stats_dict['limit_maxbytes'] curr_items = stats_dict['curr_items'] mc_results = "%s Items %s out of %s" % (curr_items, human_bytes(bytes), human_bytes(max_bytes)) except Exception, ex: mc_status = "Offline" mc_results = "%s" % ex context['memcached_status'] = mc_status context['memcached_results'] = mc_results context['last_deploy'] = HqDeploy.get_latest() #elasticsearch status #node status context.update(check_cluster_health()) context.update(check_case_index()) context.update(check_xform_index()) context.update(check_exchange_index()) return render(request, "hqadmin/system_info.html", context) @require_superuser def noneulized_users(request, template="hqadmin/noneulized_users.html"): context = get_hqadmin_base_context(request) days = request.GET.get("days", None)
def handle(self, **options): compare_url = options.get('url', None) minutes = options.get('minutes', None) deploy = HqDeploy( date=datetime.utcnow(), user=options['user'], environment=options['environment'], diff_url=compare_url ) deploy.save() # reset PillowTop errors in the hope that a fix has been deployed rows_updated = PillowError.bulk_reset_attempts(datetime.utcnow()) if rows_updated: print("\n---------------- Pillow Errors Reset ----------------\n" \ "{} pillow errors queued for retry\n".format(rows_updated)) deploy_notification_text = ( "CommCareHQ has been successfully deployed to *{}* by *{}* in *{}* minutes. ".format( options['environment'], options['user'], minutes or '?', ) ) if options['environment'] == 'production': deploy_notification_text += "Monitor the {dashboard_link}. " if settings.MOBILE_INTEGRATION_TEST_TOKEN: deploy_notification_text += "Check the integration {integration_tests_link}. " requests.get( 'https://jenkins.dimagi.com/job/integration-tests/build', params={'token': settings.MOBILE_INTEGRATION_TEST_TOKEN}, ) requests.get( 'https://jenkins.dimagi.com/job/integration-tests-pipeline/build', params={'token': settings.MOBILE_INTEGRATION_TEST_TOKEN}, ) deploy_notification_text += "Find the diff {diff_link}" if hasattr(settings, 'MIA_THE_DEPLOY_BOT_API'): link = diff_link(STYLE_SLACK, compare_url) if options['environment'] == 'staging': channel = '#staging' elif options['environment'] == 'icds': channel = '#nic-server-standup' else: channel = '#hq-ops' requests.post(settings.MIA_THE_DEPLOY_BOT_API, data=json.dumps({ "username": "******", "channel": channel, "text": deploy_notification_text.format( dashboard_link=dashboard_link(STYLE_SLACK, DASHBOARD_URL), diff_link=link, integration_tests_link=integration_tests_link(STYLE_SLACK, INTEGRATION_TEST_URL) ), })) if settings.DATADOG_API_KEY: tags = ['environment:{}'.format(options['environment'])] link = diff_link(STYLE_MARKDOWN, compare_url) datadog_api.Event.create( title="Deploy Success", text=deploy_notification_text.format( dashboard_link=dashboard_link(STYLE_MARKDOWN, DASHBOARD_URL), diff_link=link, integration_tests_link=integration_tests_link(STYLE_MARKDOWN, INTEGRATION_TEST_URL) ), tags=tags, alert_type="success" ) print("\n=============================================================\n" \ "Congratulations! Deploy Complete.\n\n" \ "Don't forget to keep an eye on the deploy dashboard to " \ "make sure everything is running smoothly.\n\n" \ "https://p.datadoghq.com/sb/5c4af2ac8-1f739e93ef" \ "\n=============================================================\n") if options['mail_admins']: message_body = get_deploy_email_message_body(user=options['user'], compare_url=compare_url) call_command('mail_admins', message_body, **{'subject': 'Deploy successful', 'html': True}) if settings.DAILY_DEPLOY_EMAIL: recipient = settings.DAILY_DEPLOY_EMAIL subject = 'Deploy Successful - {}'.format(options['environment']) send_HTML_email(subject=subject, recipient=recipient, html_content=message_body) if settings.SENTRY_CONFIGURED and settings.SENTRY_API_KEY: create_update_sentry_release() notify_sentry_deploy(minutes)