def db(request):
"""
Require a Mongo test database.
Provides a Mongo test database named after the requesting test function. Mongo databases are
created/destroyed based on the URI provided with the --mongo-uri option and tear-down
behavior is modified by the --keep-db option.
"""
from girderformindlogger.models import _dbClients, getDbConnection, pymongo
from girderformindlogger.models import model_base
from girderformindlogger.external import mongodb_proxy
mockDb = request.config.getoption('--mock-db')
dbUri = request.config.getoption('--mongo-uri')
dbName = 'girder_test_%s' % hashlib.md5(_uid(
request.node).encode('utf8')).hexdigest()
keepDb = request.config.getoption('--keep-db')
executable_methods = mongodb_proxy.EXECUTABLE_MONGO_METHODS
realMongoClient = pymongo.MongoClient
if mockDb:
mongodb_proxy.EXECUTABLE_MONGO_METHODS = set()
pymongo.MongoClient = mongomock.MongoClient
connection = getDbConnection(uri='%s/%s' % (dbUri, dbName), quiet=False)
# Force getDbConnection from models to return our connection
_dbClients[(None, None)] = connection
connection.drop_database(dbName)
# Since models store a local reference to the current database, we need to force them all to
# reconnect
for model in model_base._modelSingletons:
model.reconnect()
yield connection
if not keepDb:
connection.drop_database(dbName)
connection.close()
if mockDb:
mongodb_proxy.EXECUTABLE_MONGO_METHODS = executable_methods
pymongo.MongoClient = realMongoClient
def getStatus(mode='basic', user=None):
"""
Get a dictionary of status information regarding the Girder server.
:param mode: 'basic' returns values available to any anonymous user.
'quick' returns only values that are cheap to acquire.
'slow' provides all of that information and adds additional
:param user: a user record. Must have admin access to get anything other
than basic mode.
:returns: a status dictionary.
"""
isAdmin = (user is not None and user['admin'])
status = {}
status['bootTime'] = psutil.boot_time()
status['currentTime'] = time.time()
process = psutil.Process(os.getpid())
status['processStartTime'] = process.create_time()
if mode in ('quick', 'slow') and isAdmin:
status['virtualMemory'] = _objectToDict(psutil.virtual_memory())
status['swap'] = _objectToDict(psutil.swap_memory())
status['cpuCount'] = psutil.cpu_count()
status['processMemory'] = _objectToDict(process.memory_info())
status['processName'] = process.name()
status['cmdline'] = process.cmdline()
status['exe'] = process.exe()
status['cwd'] = process.cwd()
status['userName'] = process.username()
status['processCpuTimes'] = _objectToDict(process.cpu_times())
db = getDbConnection().get_database()
status['mongoBuildInfo'] = db.command('buildInfo')
status['cherrypyThreadsMaxUsed'] = len(
cherrypy.tools.status.seenThreads)
status['cherrypyThreadsInUse'] = len([
True for threadId in cherrypy.tools.status.seenThreads
if 'end' not in cherrypy.tools.status.seenThreads[threadId]
])
status['cherrypyThreadPoolSize'] = cherrypy.server.thread_pool
if mode == 'slow' and isAdmin:
_computeSlowStatus(process, status, db)
return status
def __init__(self, assetstore):
"""
:param assetstore: The assetstore to act on.
"""
super(GridFsAssetstoreAdapter, self).__init__(assetstore)
recent = False
try:
# Guard in case the connectionArgs is unhashable
key = (self.assetstore.get('mongohost'),
self.assetstore.get('replicaset'))
if key in _recentConnections:
recent = (time.time() - _recentConnections[key]['created']
< RECENT_CONNECTION_CACHE_TIME)
except TypeError:
key = None
try:
# MongoClient automatically reuses connections from a pool, but we
# want to avoid redoing ensureChunkIndices each time we get such a
# connection.
client = getDbConnection(self.assetstore.get('mongohost'),
self.assetstore.get('replicaset'),
quiet=recent)
self.chunkColl = MongoProxy(client[self.assetstore['db']].chunk)
if not recent:
_ensureChunkIndices(self.chunkColl)
if key is not None:
if len(_recentConnections) >= RECENT_CONNECTION_CACHE_MAX_SIZE:
_recentConnections.clear()
_recentConnections[key] = {
'created': time.time()
}
except pymongo.errors.ConnectionFailure:
logger.error('Failed to connect to GridFS assetstore %s',
self.assetstore['db'])
self.chunkColl = 'Failed to connect'
self.unavailable = True
except pymongo.errors.ConfigurationError:
logger.exception('Failed to configure GridFS assetstore %s',
self.assetstore['db'])
self.chunkColl = 'Failed to configure'
self.unavailable = True
def validateInfo(doc):
"""
Validate the assetstore -- make sure we can connect to it and that the
necessary indexes are set up.
"""
if not doc.get('db', ''):
raise ValidationException('Database name must not be empty.', 'db')
if '.' in doc['db'] or ' ' in doc['db']:
raise ValidationException('Database name cannot contain spaces'
' or periods.', 'db')
try:
chunkColl = getDbConnection(
doc.get('mongohost'), doc.get('replicaset'), autoRetry=False,
serverSelectionTimeoutMS=10000)[doc['db']].chunk
_ensureChunkIndices(chunkColl)
except pymongo.errors.ServerSelectionTimeoutError as e:
raise ValidationException(
'Could not connect to the database: %s' % str(e))
return doc
def _computeSlowStatus(process, status, db):
status['diskPartitions'] = [
_objectToDict(part) for part in psutil.disk_partitions()
]
try:
# This fails in certain environments, so guard it
status['diskIO'] = _objectToDict(psutil.disk_io_counters())
except Exception:
pass
# Report on the disk usage where the script is located
if hasattr(girderformindlogger, '__file__'):
status['girderPath'] = os.path.abspath(girderformindlogger.__file__)
status['girderDiskUsage'] = _objectToDict(
psutil.disk_usage(status['girderPath']))
# Report where our logs are and how much space is available for them
status['logs'] = []
for handler in logger.handlers:
try:
logInfo = {'path': handler.baseFilename}
logInfo['diskUsage'] = _objectToDict(
psutil.disk_usage(logInfo['path']))
status['logs'].append(logInfo)
except Exception:
# If we can't read information about the log, don't throw an
# exception
pass
status['mongoDbStats'] = db.command('dbStats')
status['mongoDbPath'] = getDbConnection().admin.command(
'getCmdLineOpts')['parsed']['storage']['dbPath']
status['mongoDbDiskUsage'] = _objectToDict(
psutil.disk_usage(status['mongoDbPath']))
status['processDirectChildrenCount'] = len(process.children())
status['processAllChildrenCount'] = len(process.children(True))
status['openFiles'] = [
_objectToDict(file) for file in process.open_files()
]
# I'd rather see textual names for the family and type of connections,
# so make a lookup table for them
connFamily = {
getattr(socket, key): key
for key in dir(socket) if key.startswith('AF_')
}
connType = {
getattr(socket, key): key
for key in dir(socket) if key.startswith('SOCK_')
}
connections = []
for conn in process.connections():
connDict = _objectToDict(conn)
connDict.pop('raddr', None)
connDict.pop('laddr', None)
connDict['family'] = connFamily.get(connDict['family'],
connDict['family'])
connDict['type'] = connType.get(connDict['type'], connDict['type'])
connections.append(connDict)
status['connections'] = connections
if hasattr(process, 'io_counters'):
status['ioCounters'] = _objectToDict(process.io_counters())
status['cherrypyThreads'] = {}
for threadId in cherrypy.tools.status.seenThreads:
info = cherrypy.tools.status.seenThreads[threadId].copy()
if 'end' in info:
info['duration'] = info['end'] - info['start']
info['idle'] = time.time() - info['end']
status['cherrypyThreads'][threadId] = info