def dashboard(request): """ Render HTML for dashboard/metrics view. """ # Datasets dataset_count = Dataset.objects.all().count() datasets_without_descriptions = [ (unquote(dataset['name']), dataset['slug']) for dataset in Dataset.objects.filter( description='').values('name', 'slug') ] datasets_without_categories = [(unquote(dataset['name']), dataset['slug']) for dataset in Dataset.objects.filter( categories=None).values('name', 'slug')] # Users user_count = UserProxy.objects.all().count() activated_user_count = UserProxy.objects.filter(is_active=True).count() today = now().date() thirty_days_ago = today - datetime.timedelta(days=30) active_users = list( UserProxy.objects.raw( 'SELECT auth_user.*, count(panda_activitylog.id) AS activity_logs__count FROM auth_user LEFT JOIN panda_activitylog ON panda_activitylog.user_id = auth_user.id WHERE auth_user.is_active = True AND panda_activitylog.when > %s GROUP BY auth_user.id ORDER BY activity_logs__count DESC, auth_user.id ASC', [thirty_days_ago])) most_active_users = active_users[:10] if len(active_users) > 10: least_active_users = active_users[-10:] least_active_users.reverse() else: least_active_users = [] inactive_users = UserProxy.objects.all() \ .annotate(Count('activity_logs')) \ .filter(activity_logs__count=0) _active_users_by_day = \ list(ActivityLog.objects.filter(when__gt=thirty_days_ago) \ .values('when') \ .annotate(Count('id')) \ .order_by('when')) dates = [ thirty_days_ago + datetime.timedelta(days=x) for x in range(0, 31) ] active_users_by_day = [] for d in dates: if _active_users_by_day and _active_users_by_day[0]['when'] == d: _d = _active_users_by_day.pop(0) active_users_by_day.append(_d) else: active_users_by_day.append({'when': d, 'id__count': 0}) # Searches total_searches = SearchLog.objects.count() most_searched_datasets = [(unquote(dataset['name']), dataset['slug'], dataset['searches__count']) for dataset in \ Dataset.objects.all() \ .annotate(Count('searches')) \ .filter(searches__count__gt=0) \ .order_by('-searches__count') \ .values('name', 'slug', 'searches__count')[:10]] _searches_by_day = \ list(SearchLog.objects.filter(when__gt=thirty_days_ago) \ .extra(select={ 'day': '"when"::date' }) \ .values('day') \ .annotate(Count('when')) \ .order_by('day')) dates = [ thirty_days_ago + datetime.timedelta(days=x) for x in range(0, 31) ] searches_by_day = [] for d in dates: if _searches_by_day and _searches_by_day[0]['day'] == d: _d = _searches_by_day.pop(0) searches_by_day.append(_d) else: searches_by_day.append({'day': d, 'when__count': 0}) # Disk space root_disk = os.stat('/').st_dev upload_disk = os.stat(settings.MEDIA_ROOT).st_dev indices_disk = os.stat(settings.SOLR_DIRECTORY).st_dev root_disk_total = utils.get_total_disk_space('/') root_disk_free = utils.get_free_disk_space('/') root_disk_percent_used = 100 - (float(root_disk_free) / root_disk_total * 100) if upload_disk != root_disk: upload_disk_total = utils.get_total_disk_space(settings.MEDIA_ROOT) upload_disk_free = utils.get_free_disk_space(settings.MEDIA_ROOT) upload_disk_percent_used = 100 - (float(upload_disk_free) / upload_disk_total * 100) else: upload_disk_total = None upload_disk_free = None upload_disk_percent_used = None if indices_disk != root_disk: indices_disk_total = utils.get_total_disk_space( settings.SOLR_DIRECTORY) indices_disk_free = utils.get_free_disk_space(settings.SOLR_DIRECTORY) indices_disk_percent_used = 100 - (float(indices_disk_free) / indices_disk_total * 100) else: indices_disk_total = None indices_disk_free = None indices_disk_percent_used = None return render_to_response( 'dashboard.html', { 'settings': settings, 'dataset_count': dataset_count, 'datasets_without_descriptions': datasets_without_descriptions, 'datasets_without_categories': datasets_without_categories, 'user_count': user_count, 'activated_user_count': activated_user_count, 'most_active_users': most_active_users, 'least_active_users': least_active_users, 'inactive_users': inactive_users, 'active_users_by_day': active_users_by_day, 'total_searches': total_searches, 'most_searched_datasets': most_searched_datasets, 'searches_by_day': searches_by_day, 'root_disk_total': root_disk_total, 'root_disk_free': root_disk_free, 'root_disk_percent_used': root_disk_percent_used, 'upload_disk_total': upload_disk_total, 'upload_disk_free': upload_disk_free, 'upload_disk_percent_used': upload_disk_percent_used, 'indices_disk_total': indices_disk_total, 'indices_disk_free': indices_disk_free, 'indices_disk_percent_used': indices_disk_percent_used, 'storage_documentation_url': 'http://panda.readthedocs.org/en/%s/storage.html' % settings.PANDA_VERSION })
def dashboard(request): """ Render HTML for dashboard/metrics view. """ # Datasets dataset_count = Dataset.objects.all().count() datasets_without_descriptions = [(unquote(dataset['name']), dataset['slug']) for dataset in Dataset.objects.filter(description='').values('name', 'slug')] datasets_without_categories = [(unquote(dataset['name']), dataset['slug']) for dataset in Dataset.objects.filter(categories=None).values('name', 'slug')] # Users user_count = UserProxy.objects.all().count() activated_user_count = UserProxy.objects.filter(is_active=True).count() today = now().date() thirty_days_ago = today - datetime.timedelta(days=30) active_users = list(UserProxy.objects.raw('SELECT auth_user.*, count(panda_activitylog.id) AS activity_logs__count FROM auth_user LEFT JOIN panda_activitylog ON panda_activitylog.user_id = auth_user.id WHERE auth_user.is_active = True AND panda_activitylog.when > %s GROUP BY auth_user.id ORDER BY activity_logs__count DESC, auth_user.id ASC', [thirty_days_ago])) most_active_users = active_users[:10] if len(active_users) > 10: least_active_users = active_users[-10:] least_active_users.reverse() else: least_active_users = [] inactive_users = UserProxy.objects.all() \ .annotate(Count('activity_logs')) \ .filter(activity_logs__count=0) _active_users_by_day = \ list(ActivityLog.objects.filter(when__gt=thirty_days_ago) \ .values('when') \ .annotate(Count('id')) \ .order_by('when')) dates = [thirty_days_ago + datetime.timedelta(days=x) for x in range(0, 31)] active_users_by_day = [] for d in dates: if _active_users_by_day and _active_users_by_day[0]['when'] == d: _d = _active_users_by_day.pop(0) active_users_by_day.append(_d) else: active_users_by_day.append({ 'when': d, 'id__count': 0 }) # Searches total_searches = SearchLog.objects.count() most_searched_datasets = [(unquote(dataset['name']), dataset['slug'], dataset['searches__count']) for dataset in \ Dataset.objects.all() \ .annotate(Count('searches')) \ .filter(searches__count__gt=0) \ .order_by('-searches__count') \ .values('name', 'slug', 'searches__count')[:10]] _searches_by_day = \ list(SearchLog.objects.filter(when__gt=thirty_days_ago) \ .extra(select={ 'day': '"when"::date' }) \ .values('day') \ .annotate(Count('when')) \ .order_by('day')) dates = [thirty_days_ago + datetime.timedelta(days=x) for x in range(0, 31)] searches_by_day = [] for d in dates: if _searches_by_day and _searches_by_day[0]['day'] == d: _d = _searches_by_day.pop(0) searches_by_day.append(_d) else: searches_by_day.append({ 'day': d, 'when__count': 0 }) # Disk space root_disk = os.stat('/').st_dev upload_disk = os.stat(settings.MEDIA_ROOT).st_dev indices_disk = os.stat(settings.SOLR_DIRECTORY).st_dev root_disk_total = utils.get_total_disk_space('/') root_disk_free = utils.get_free_disk_space('/') root_disk_percent_used = 100 - (float(root_disk_free) / root_disk_total * 100) if upload_disk != root_disk: upload_disk_total = utils.get_total_disk_space(settings.MEDIA_ROOT) upload_disk_free = utils.get_free_disk_space(settings.MEDIA_ROOT) upload_disk_percent_used = 100 - (float(upload_disk_free) / upload_disk_total * 100) else: upload_disk_total = None upload_disk_free = None upload_disk_percent_used = None if indices_disk != root_disk: indices_disk_total = utils.get_total_disk_space(settings.SOLR_DIRECTORY) indices_disk_free = utils.get_free_disk_space(settings.SOLR_DIRECTORY) indices_disk_percent_used = 100 - (float(indices_disk_free) / indices_disk_total * 100) else: indices_disk_total = None indices_disk_free = None indices_disk_percent_used = None return render_to_response('dashboard.html', { 'settings': settings, 'dataset_count': dataset_count, 'datasets_without_descriptions': datasets_without_descriptions, 'datasets_without_categories': datasets_without_categories, 'user_count': user_count, 'activated_user_count': activated_user_count, 'most_active_users': most_active_users, 'least_active_users': least_active_users, 'inactive_users': inactive_users, 'active_users_by_day': active_users_by_day, 'total_searches': total_searches, 'most_searched_datasets': most_searched_datasets, 'searches_by_day': searches_by_day, 'root_disk_total': root_disk_total, 'root_disk_free': root_disk_free, 'root_disk_percent_used': root_disk_percent_used, 'upload_disk_total': upload_disk_total, 'upload_disk_free': upload_disk_free, 'upload_disk_percent_used': upload_disk_percent_used, 'indices_disk_total': indices_disk_total, 'indices_disk_free': indices_disk_free, 'indices_disk_percent_used': indices_disk_percent_used, 'storage_documentation_url': 'http://panda.readthedocs.org/en/%s/storage.html' % settings.PANDA_VERSION })
def run(self, *args, **kwargs): from panda.models import UserProxy log = logging.getLogger(self.name) log.info('Running admin alerts') # Disk space root_disk = os.stat('/').st_dev upload_disk = os.stat(settings.MEDIA_ROOT).st_dev indices_disk = os.stat(settings.SOLR_DIRECTORY).st_dev root_disk_total = get_total_disk_space('/') root_disk_free = get_free_disk_space('/') root_disk_percent_used = 100 - (float(root_disk_free) / root_disk_total * 100) if upload_disk != root_disk: upload_disk_total = get_total_disk_space(settings.MEDIA_ROOT) upload_disk_free = get_free_disk_space(settings.MEDIA_ROOT) upload_disk_percent_used = 100 - (float(upload_disk_free) / upload_disk_total * 100) else: upload_disk_total = None upload_disk_free = None upload_disk_percent_used = None if indices_disk != root_disk: indices_disk_total = get_total_disk_space(settings.SOLR_DIRECTORY) indices_disk_free = get_free_disk_space(settings.SOLR_DIRECTORY) indices_disk_percent_used = 100 - (float(indices_disk_free) / indices_disk_total * 100) else: indices_disk_total = None indices_disk_free = None indices_disk_percent_used = None notify = False for free in (root_disk_free, upload_disk_free, indices_disk_free): if free is None: continue if free < settings.PANDA_AVAILABLE_SPACE_WARN: notify = True if notify: context = Context({ 'root_disk': root_disk, 'upload_disk': upload_disk, 'indices_disk': indices_disk, 'root_disk_total': root_disk_total, 'root_disk_free': root_disk_free, 'root_disk_percent_used': root_disk_percent_used, 'upload_disk_total': upload_disk_total, 'upload_disk_free': upload_disk_free, 'upload_disk_percent_used': upload_disk_percent_used, 'indices_disk_total': indices_disk_total, 'indices_disk_free': indices_disk_free, 'indices_disk_percent_used': indices_disk_percent_used, 'settings': settings, 'site_domain': config_value('DOMAIN', 'SITE_DOMAIN') }) # Don't HTML escape plain-text emails context.autoescape = False email_subject = get_email_subject_template('disk_space_alert').render(context) email_message = get_email_body_template('disk_space_alert').render(context) recipients = UserProxy.objects.filter(is_superuser=True, is_active=True) send_mail(email_subject.strip(), email_message, [r.email for r in recipients]) log.info('Finished running admin alerts')
def dashboard(request): """ Render HTML for dashboard/metrics view. """ # Datasets dataset_count = Dataset.objects.all().count() datasets_without_descriptions = [(unquote(dataset['name']), dataset['slug']) for dataset in Dataset.objects.filter(description='').values('name', 'slug')] datasets_without_categories = [(unquote(dataset['name']), dataset['slug']) for dataset in Dataset.objects.filter(categories=None).values('name', 'slug')] # Users user_count = UserProxy.objects.all().count() inactive_user_count = UserProxy.objects.filter(is_active=False).count() most_active_users = \ UserProxy.objects.all() \ .annotate(Count('activity_logs')) \ .filter(activity_logs__count__gt=0) \ .order_by('-activity_logs__count')[:10] least_active_users = \ UserProxy.objects.all() \ .annotate(Count('activity_logs')) \ .exclude(id__in=[user.id for user in most_active_users]) \ .order_by('activity_logs__count')[:10] today = now().date() thirty_days_ago = today - datetime.timedelta(days=30) _active_users_by_day = \ list(ActivityLog.objects.filter(when__gt=thirty_days_ago) \ .values('when') \ .annotate(Count('id')) \ .order_by('when')) dates = [thirty_days_ago + datetime.timedelta(days=x) for x in range(0, 31)] active_users_by_day = [] for d in dates: if _active_users_by_day and _active_users_by_day[0]['when'] == d: _d = _active_users_by_day.pop(0) active_users_by_day.append(_d) else: active_users_by_day.append({ 'when': d, 'id__count': 0 }) # Searches total_searches = SearchLog.objects.count() most_searched_datasets = [(unquote(dataset['name']), dataset['slug'], dataset['searches__count']) for dataset in \ Dataset.objects.all() \ .annotate(Count('searches')) \ .filter(searches__count__gt=0) \ .order_by('-searches__count') \ .values('name', 'slug', 'searches__count')[:10]] _searches_by_day = \ list(SearchLog.objects.filter(when__gt=thirty_days_ago) \ .extra(select={ 'day': '"when"::date' }) \ .values('day') \ .annotate(Count('when')) \ .order_by('day')) dates = [thirty_days_ago + datetime.timedelta(days=x) for x in range(0, 31)] searches_by_day = [] for d in dates: if _searches_by_day and _searches_by_day[0]['day'] == d: _d = _searches_by_day.pop(0) searches_by_day.append(_d) else: searches_by_day.append({ 'day': d, 'when__count': 0 }) # Disk space root_disk = os.stat('/').st_dev upload_disk = os.stat(settings.MEDIA_ROOT).st_dev indices_disk = os.stat(settings.SOLR_DIRECTORY).st_dev root_disk_total = utils.get_total_disk_space('/') root_disk_free = utils.get_free_disk_space('/') root_disk_percent_used = 100 - (float(root_disk_free) / root_disk_total * 100) if upload_disk != root_disk: upload_disk_total = utils.get_total_disk_space(settings.MEDIA_ROOT) upload_disk_free = utils.get_free_disk_space(settings.MEDIA_ROOT) upload_disk_percent_used = 100 - (float(upload_disk_free) / upload_disk_total * 100) else: upload_disk_total = None upload_disk_free = None upload_disk_percent_used = None if indices_disk != root_disk: indices_disk_total = utils.get_total_disk_space(settings.SOLR_DIRECTORY) indices_disk_free = utils.get_free_disk_space(settings.SOLR_DIRECTORY) indices_disk_percent_used = 100 - (float(indices_disk_free) / indices_disk_total * 100) else: indices_disk_total = None indices_disk_free = None indices_disk_percent_used = None return render_to_response('dashboard.html', { 'settings': settings, 'dataset_count': dataset_count, 'datasets_without_descriptions': datasets_without_descriptions, 'datasets_without_categories': datasets_without_categories, 'user_count': user_count, 'inactive_user_count': inactive_user_count, 'most_active_users': most_active_users, 'least_active_users': least_active_users, 'active_users_by_day': active_users_by_day, 'total_searches': total_searches, 'most_searched_datasets': most_searched_datasets, 'searches_by_day': searches_by_day, 'root_disk_total': root_disk_total, 'root_disk_free': root_disk_free, 'root_disk_percent_used': root_disk_percent_used, 'upload_disk_total': upload_disk_total, 'upload_disk_free': upload_disk_free, 'upload_disk_percent_used': upload_disk_percent_used, 'indices_disk_total': indices_disk_total, 'indices_disk_free': indices_disk_free, 'indices_disk_percent_used': indices_disk_percent_used })
def run(self, *args, **kwargs): from panda.models import UserProxy log = logging.getLogger(self.name) log.info('Running admin alerts') # Disk space root_disk = os.stat('/').st_dev upload_disk = os.stat(settings.MEDIA_ROOT).st_dev indices_disk = os.stat(settings.SOLR_DIRECTORY).st_dev root_disk_total = get_total_disk_space('/') root_disk_free = get_free_disk_space('/') root_disk_percent_used = 100 - (float(root_disk_free) / root_disk_total * 100) if upload_disk != root_disk: upload_disk_total = get_total_disk_space(settings.MEDIA_ROOT) upload_disk_free = get_free_disk_space(settings.MEDIA_ROOT) upload_disk_percent_used = 100 - (float(upload_disk_free) / upload_disk_total * 100) else: upload_disk_total = None upload_disk_free = None upload_disk_percent_used = None if indices_disk != root_disk: indices_disk_total = get_total_disk_space(settings.SOLR_DIRECTORY) indices_disk_free = get_free_disk_space(settings.SOLR_DIRECTORY) indices_disk_percent_used = 100 - (float(indices_disk_free) / indices_disk_total * 100) else: indices_disk_total = None indices_disk_free = None indices_disk_percent_used = None notify = False for free in (root_disk_free, upload_disk_free, indices_disk_free): if free is None: continue if free < settings.PANDA_AVAILABLE_SPACE_WARN: notify = True if notify: context = Context({ 'root_disk': root_disk, 'upload_disk': upload_disk, 'indices_disk': indices_disk, 'root_disk_total': root_disk_total, 'root_disk_free': root_disk_free, 'root_disk_percent_used': root_disk_percent_used, 'upload_disk_total': upload_disk_total, 'upload_disk_free': upload_disk_free, 'upload_disk_percent_used': upload_disk_percent_used, 'indices_disk_total': indices_disk_total, 'indices_disk_free': indices_disk_free, 'indices_disk_percent_used': indices_disk_percent_used, 'settings': settings, 'site_domain': config_value('DOMAIN', 'SITE_DOMAIN') }) # Don't HTML escape plain-text emails context.autoescape = False email_subject = get_email_subject_template( 'disk_space_alert').render(context) email_message = get_email_body_template('disk_space_alert').render( context) recipients = UserProxy.objects.filter(is_superuser=True, is_active=True) send_mail(email_subject.strip(), email_message, [r.email for r in recipients]) log.info('Finished running admin alerts')