def record_queue_size(): """Records the rabbitmq size in statsd""" statsd.gauge('rabbitmq.size', rabbitmq_queue_size())
def monitor(request): """View for services monitor.""" status = {} # Note: To add a new component to the services monitor, do your # testing and then add a name -> list of output tuples map to # status. # Check memcached. memcache_results = [] try: for cache_name, cache_props in settings.CACHES.items(): result = True backend = cache_props['BACKEND'] location = cache_props['LOCATION'] # LOCATION can be a string or a list of strings if isinstance(location, basestring): location = location.split(';') if 'memcache' in backend: for loc in location: # TODO: this doesn't handle unix: variant ip, port = loc.split(':') result = test_memcached(ip, int(port)) memcache_results.append( (INFO, '%s:%s %s' % (ip, port, result))) if not memcache_results: memcache_results.append((ERROR, 'memcache is not configured.')) elif len(memcache_results) < 2: memcache_results.append( (ERROR, ('You should have at least 2 memcache servers. ' 'You have %s.' % len(memcache_results)))) else: memcache_results.append((INFO, 'memcached servers look good.')) except Exception as exc: memcache_results.append( (ERROR, 'Exception while looking at memcached: %s' % str(exc))) status['memcached'] = memcache_results # Check Libraries and versions libraries_results = [] try: Image.new('RGB', (16, 16)).save(StringIO.StringIO(), 'JPEG') libraries_results.append((INFO, 'PIL+JPEG: Got it!')) except Exception as exc: libraries_results.append( (ERROR, 'PIL+JPEG: Probably missing: ' 'Failed to create a jpeg image: %s' % exc)) status['libraries'] = libraries_results # Check file paths. msg = 'We want read + write.' filepaths = ( (settings.USER_AVATAR_PATH, os.R_OK | os.W_OK, msg), (settings.IMAGE_UPLOAD_PATH, os.R_OK | os.W_OK, msg), (settings.THUMBNAIL_UPLOAD_PATH, os.R_OK | os.W_OK, msg), (settings.GALLERY_IMAGE_PATH, os.R_OK | os.W_OK, msg), (settings.GALLERY_IMAGE_THUMBNAIL_PATH, os.R_OK | os.W_OK, msg), (settings.GALLERY_VIDEO_PATH, os.R_OK | os.W_OK, msg), (settings.GALLERY_VIDEO_THUMBNAIL_PATH, os.R_OK | os.W_OK, msg), (settings.GROUP_AVATAR_PATH, os.R_OK | os.W_OK, msg), ) filepath_results = [] for path, perms, notes in filepaths: path = os.path.join(settings.MEDIA_ROOT, path) path_exists = os.path.isdir(path) path_perms = os.access(path, perms) if path_exists and path_perms: filepath_results.append( (INFO, '%s: %s %s %s' % (path, path_exists, path_perms, notes))) status['filepaths'] = filepath_results # Check RabbitMQ. rabbitmq_results = [] try: rabbit_conn = establish_connection(connect_timeout=2) rabbit_conn.connect() rabbitmq_results.append( (INFO, 'Successfully connected to RabbitMQ.')) rabbitmq_results.append( (INFO, 'Queue size: %s' % rabbitmq_queue_size())) except (socket.error, IOError) as exc: rabbitmq_results.append( (ERROR, 'Error connecting to RabbitMQ: %s' % str(exc))) except Exception as exc: rabbitmq_results.append( (ERROR, 'Exception while looking at RabbitMQ: %s' % str(exc))) status['RabbitMQ'] = rabbitmq_results # Check ES. es_results = [] try: es_utils.get_doctype_stats(es_utils.all_read_indexes()[0]) es_results.append( (INFO, ('Successfully connected to ElasticSearch and index ' 'exists.'))) except es_utils.ES_EXCEPTIONS as exc: es_results.append( (ERROR, 'ElasticSearch problem: %s' % str(exc))) except Exception as exc: es_results.append( (ERROR, 'Exception while looking at ElasticSearch: %s' % str(exc))) status['ElasticSearch'] = es_results # Check Celery. # start = time.time() # pong = celery.task.ping() # rabbit_results = r = {'duration': time.time() - start} # status_summary['rabbit'] = pong == 'pong' and r['duration'] < 1 # Check Redis. redis_results = [] if hasattr(settings, 'REDIS_BACKENDS'): for backend in settings.REDIS_BACKENDS: try: redis_client(backend) redis_results.append((INFO, '%s: Pass!' % backend)) except RedisError: redis_results.append((ERROR, '%s: Fail!' % backend)) status['Redis'] = redis_results status_code = 200 status_summary = {} for component, output in status.items(): if ERROR in [item[0] for item in output]: status_code = 500 status_summary[component] = False else: status_summary[component] = True return render(request, 'services/monitor.html', { 'component_status': status, 'status_summary': status_summary}, status=status_code)
def monitor(request): """View for services monitor.""" status = {} # Note: To add a new component to the services monitor, do your # testing and then add a name -> list of output tuples map to # status. # Check memcached. memcache_results = [] try: for cache_name, cache_props in settings.CACHES.items(): result = True backend = cache_props['BACKEND'] location = cache_props['LOCATION'] # LOCATION can be a string or a list of strings if isinstance(location, basestring): location = location.split(';') if 'memcache' in backend: for loc in location: # TODO: this doesn't handle unix: variant ip, port = loc.split(':') result = test_memcached(ip, int(port)) memcache_results.append( (INFO, '%s:%s %s' % (ip, port, result))) if not memcache_results: memcache_results.append((ERROR, 'memcache is not configured.')) elif len(memcache_results) < 2: memcache_results.append( (ERROR, ('You should have at least 2 memcache servers. ' 'You have %s.' % len(memcache_results)))) else: memcache_results.append((INFO, 'memcached servers look good.')) except Exception as exc: memcache_results.append( (ERROR, 'Exception while looking at memcached: %s' % str(exc))) status['memcached'] = memcache_results # Check Libraries and versions libraries_results = [] try: Image.new('RGB', (16, 16)).save(StringIO.StringIO(), 'JPEG') libraries_results.append((INFO, 'PIL+JPEG: Got it!')) except Exception as exc: libraries_results.append((ERROR, 'PIL+JPEG: Probably missing: ' 'Failed to create a jpeg image: %s' % exc)) status['libraries'] = libraries_results # Check file paths. msg = 'We want read + write.' filepaths = ( (settings.USER_AVATAR_PATH, os.R_OK | os.W_OK, msg), (settings.IMAGE_UPLOAD_PATH, os.R_OK | os.W_OK, msg), (settings.THUMBNAIL_UPLOAD_PATH, os.R_OK | os.W_OK, msg), (settings.GALLERY_IMAGE_PATH, os.R_OK | os.W_OK, msg), (settings.GALLERY_IMAGE_THUMBNAIL_PATH, os.R_OK | os.W_OK, msg), (settings.GALLERY_VIDEO_PATH, os.R_OK | os.W_OK, msg), (settings.GALLERY_VIDEO_THUMBNAIL_PATH, os.R_OK | os.W_OK, msg), (settings.GROUP_AVATAR_PATH, os.R_OK | os.W_OK, msg), ) filepath_results = [] for path, perms, notes in filepaths: path = os.path.join(settings.MEDIA_ROOT, path) path_exists = os.path.isdir(path) path_perms = os.access(path, perms) if path_exists and path_perms: filepath_results.append( (INFO, '%s: %s %s %s' % (path, path_exists, path_perms, notes))) status['filepaths'] = filepath_results # Check RabbitMQ. rabbitmq_results = [] try: rabbit_conn = establish_connection(connect_timeout=2) rabbit_conn.connect() rabbitmq_results.append((INFO, 'Successfully connected to RabbitMQ.')) rabbitmq_results.append( (INFO, 'Queue size: %s' % rabbitmq_queue_size())) except (socket.error, IOError) as exc: rabbitmq_results.append( (ERROR, 'Error connecting to RabbitMQ: %s' % str(exc))) except Exception as exc: rabbitmq_results.append( (ERROR, 'Exception while looking at RabbitMQ: %s' % str(exc))) status['RabbitMQ'] = rabbitmq_results # Check ES. es_results = [] try: es_utils.get_doctype_stats(es_utils.all_read_indexes()[0]) es_results.append( (INFO, ('Successfully connected to ElasticSearch and index ' 'exists.'))) except es_utils.ES_EXCEPTIONS as exc: es_results.append((ERROR, 'ElasticSearch problem: %s' % str(exc))) except Exception as exc: es_results.append( (ERROR, 'Exception while looking at ElasticSearch: %s' % str(exc))) status['ElasticSearch'] = es_results # Check Celery. # start = time.time() # pong = celery.task.ping() # rabbit_results = r = {'duration': time.time() - start} # status_summary['rabbit'] = pong == 'pong' and r['duration'] < 1 # Check Redis. redis_results = [] if hasattr(settings, 'REDIS_BACKENDS'): for backend in settings.REDIS_BACKENDS: try: redis_client(backend) redis_results.append((INFO, '%s: Pass!' % backend)) except RedisError: redis_results.append((ERROR, '%s: Fail!' % backend)) status['Redis'] = redis_results status_code = 200 status_summary = {} for component, output in status.items(): if ERROR in [item[0] for item in output]: status_code = 500 status_summary[component] = False else: status_summary[component] = True return render(request, 'services/monitor.html', { 'component_status': status, 'status_summary': status_summary }, status=status_code)