def process_annotations(event):
"""Add annotations to an image on a ``data.process`` event"""
info = event.info
if 'anot' in info.get('file', {}).get('exts', []):
reference = info.get('reference', None)
try:
reference = json.loads(reference)
except (ValueError, TypeError):
print(TerminalColor.error(
'Warning: Could not get reference from the annotation param. '
'Make sure you have at ctk-cli>=1.3.1 installed.'
))
return
if 'userId' not in reference or 'itemId' not in reference:
print(TerminalColor.error(
'Annotation reference does not contain required information.'
))
return
userId = reference['userId']
imageId = reference['itemId']
# load model classes
Item = ModelImporter.model('item')
File = ModelImporter.model('file')
User = ModelImporter.model('user')
Annotation = ModelImporter.model('annotation', plugin='large_image')
# load models from the database
user = User.load(userId, force=True)
image = File.load(imageId, level=AccessType.READ, user=user)
item = Item.load(image['itemId'], level=AccessType.WRITE, user=user)
file = File.load(
info.get('file', {}).get('_id'),
level=AccessType.READ, user=user
)
if not (item and user and file):
print(TerminalColor.error(
'Could not load models from the database'
))
return
try:
data = json.loads(
''.join(File.download(file)())
)
except Exception:
print(TerminalColor.error(
'Could not parse annotation file'
))
return
Annotation.createAnnotation(
item,
user,
data
)
def getDbConnection(uri=None, replicaSet=None):
"""
Get a MongoClient object that is connected to the configured database.
We lazy-instantiate a module-level singleton, the MongoClient objects
manage their own connection pools internally.
:param uri: if specified, connect to this mongo db rather than the one in
the config.
:param replicaSet: if uri is specified, use this replica set.
"""
global _dbClients
origKey = (uri, replicaSet)
if origKey in _dbClients:
return _dbClients[origKey]
if uri is None or uri == '':
dbConf = getDbConfig()
uri = dbConf.get('uri')
replicaSet = dbConf.get('replica_set')
clientOptions = {
'connectTimeoutMS': 15000,
# This is the maximum time between when we fetch data from a cursor.
# If it times out, the cursor is lost and we can't reconnect. If it
# isn't set, we have issues with replica sets when the primary goes
# down. This value can be overridden in the mongodb uri connection
# string with the socketTimeoutMS.
'socketTimeoutMS': 60000,
}
if uri is None:
dbUriRedacted = 'mongodb://*****:*****@')
if len(parts) == 2:
dbUriRedacted = 'mongodb://' + parts[1]
else:
dbUriRedacted = uri
if replicaSet:
client = pymongo.MongoReplicaSetClient(
uri,
replicaSet=replicaSet,
read_preference=ReadPreference.SECONDARY_PREFERRED,
**clientOptions)
else:
client = pymongo.MongoClient(uri, **clientOptions)
client = MongoProxy(client, logger=logger)
_dbClients[origKey] = _dbClients[(uri, replicaSet)] = client
desc = ''
if replicaSet:
desc += ', replica set: %s' % replicaSet
print(
TerminalColor.info('Connected to MongoDB: %s%s' %
(dbUriRedacted, desc)))
return client
def route(self, method, route, handler, nodoc=False, resource=None):
"""
Define a route for your REST resource.
:param method: The HTTP method, e.g. 'GET', 'POST', 'PUT', 'PATCH'
:type method: str
:param route: The route, as a list of path params relative to the
resource root. Elements of this list starting with ':' are assumed
to be wildcards.
:type route: tuple
:param handler: The method to be called if the route and method are
matched by a request. Wildcards in the route will be expanded and
passed as kwargs with the same name as the wildcard identifier.
:type handler: function
:param nodoc: If your route intentionally provides no documentation,
set this to True to disable the warning on startup.
:type nodoc: bool
:param resource: The name of the resource at the root of this route.
"""
if not hasattr(self, '_routes'):
self._routes = collections.defaultdict(
lambda: collections.defaultdict(list))
# Insertion sort to maintain routes in required order.
nLengthRoutes = self._routes[method.lower()][len(route)]
for i in range(len(nLengthRoutes)):
if self._shouldInsertRoute(route, nLengthRoutes[i][0]):
nLengthRoutes.insert(i, (route, handler))
break
else:
nLengthRoutes.append((route, handler))
# Now handle the api doc if the handler has any attached
if resource is None and hasattr(self, 'resourceName'):
resource = self.resourceName
elif resource is None:
resource = handler.__module__.rsplit('.', 1)[-1]
if hasattr(handler, 'description'):
if handler.description is not None:
docs.addRouteDocs(resource=resource,
route=route,
method=method,
info=handler.description.asDict(),
handler=handler)
elif not nodoc:
routePath = '/'.join([resource] + list(route))
print(
TerminalColor.warning(
'WARNING: No description docs present for route {} {}'.
format(method, routePath)))
# Warn if there is no access decorator on the handler function
if not hasattr(handler, 'accessLevel'):
routePath = '/'.join([resource] + list(route))
print(
TerminalColor.warning(
'WARNING: No access level specified for route {} {}'.
format(method, routePath)))
def route(self, method, route, handler, nodoc=False, resource=None):
"""
Define a route for your REST resource.
:param method: The HTTP method, e.g. 'GET', 'POST', 'PUT'
:type method: str
:param route: The route, as a list of path params relative to the
resource root. Elements of this list starting with ':' are assumed to
be wildcards.
:type route: list
:param handler: The method to be called if the route and method are
matched by a request. Wildcards in the route will be expanded and
passed as kwargs with the same name as the wildcard identifier.
:type handler: function
:param nodoc: If your route intentionally provides no documentation,
set this to True to disable the warning on startup.
:type nodoc: bool
"""
if not hasattr(self, '_routes'):
self._routes = collections.defaultdict(
lambda: collections.defaultdict(list))
# Insertion sort to maintain routes in required order.
def shouldInsert(a, b):
"""
Return bool representing whether route a should go before b. Checks
by comparing each token in order and making sure routes with
literals in forward positions come before routes with wildcards
in those positions.
"""
for i in xrange(0, len(a)):
if a[i][0] != ':' and b[i][0] == ':':
return True
return False
nLengthRoutes = self._routes[method.lower()][len(route)]
for i in xrange(0, len(nLengthRoutes)):
if shouldInsert(route, nLengthRoutes[i][0]):
nLengthRoutes.insert(i, (route, handler))
break
else:
nLengthRoutes.append((route, handler))
# Now handle the api doc if the handler has any attached
if resource is None and hasattr(self, 'resourceName'):
resource = self.resourceName
elif resource is None:
resource = handler.__module__.rsplit('.', 1)[-1]
if hasattr(handler, 'description'):
if handler.description is not None:
docs.addRouteDocs(
resource=resource, route=route, method=method,
info=handler.description.asDict(), handler=handler)
elif not nodoc:
routePath = '/'.join([resource] + list(route))
print TerminalColor.warning(
'WARNING: No description docs present for route {} {}'
.format(method, routePath))
def loadPlugins(plugins, root, appconf, apiRoot=None, curConfig=None,
buildDag=True):
"""
Loads a set of plugins into the application.
:param plugins: The set of plugins to load, by directory name.
:type plugins: list
:param root: The root node of the server tree.
:type root: object
:param appconf: The server's cherrypy configuration object.
:type appconf: dict
:param apiRoot: The cherrypy api root object.
:type apiRoot: object or None
:param curConfig: A girder config object to use.
:type curConfig: dict or None
:param buildDag: If the ``plugins`` parameter is already a topo-sorted list
with all dependencies resolved, set this to False and it will skip
rebuilding the DAG. Otherwise the dependency resolution and sorting
will occur within this method.
:type buildDag: bool
:returns: A 3-tuple containing the modified root, config, and apiRoot
objects.
:rtype tuple:
"""
# Register a pseudo-package for the root of all plugins. This must be
# present in the system module list in order to avoid import warnings.
if curConfig is None:
curConfig = _config.getConfig()
if 'plugins' in curConfig and 'plugin_directory' in curConfig['plugins']:
print(TerminalColor.warning(
'Warning: the plugin_directory setting is deprecated. Please use '
'the `girder-install plugin` command and remove this setting from '
'your config file.'))
if ROOT_PLUGINS_PACKAGE not in sys.modules:
module = imp.new_module(ROOT_PLUGINS_PACKAGE)
girder.plugins = module
sys.modules[ROOT_PLUGINS_PACKAGE] = module
print(TerminalColor.info('Resolving plugin dependencies...'))
if buildDag:
plugins = getToposortedPlugins(plugins, curConfig, ignoreMissing=True)
for plugin in plugins:
try:
root, appconf, apiRoot = loadPlugin(
plugin, root, appconf, apiRoot, curConfig=curConfig)
print(TerminalColor.success('Loaded plugin "%s"' % plugin))
except Exception:
print(TerminalColor.error(
'ERROR: Failed to load plugin "%s":' % plugin))
girder.logger.exception('Plugin load failure: %s' % plugin)
traceback.print_exc()
return root, appconf, apiRoot
def getDbConnection(uri=None, replicaSet=None):
"""
Get a MongoClient object that is connected to the configured database.
We lazy-instantiate a module-level singleton, the MongoClient objects
manage their own connection pools internally.
:param uri: if specified, connect to this mongo db rather than the one in
the config.
:param replicaSet: if uri is specified, use this replica set.
"""
global _dbClients
origKey = (uri, replicaSet)
if origKey in _dbClients:
return _dbClients[origKey]
if uri is None or uri == '':
dbConf = getDbConfig()
uri = dbConf.get('uri')
replicaSet = dbConf.get('replica_set')
clientOptions = {
'connectTimeoutMS': 15000,
# This is the maximum time between when we fetch data from a cursor.
# If it times out, the cursor is lost and we can't reconnect. If it
# isn't set, we have issues with replica sets when the primary goes
# down. This value can be overridden in the mongodb uri connection
# string with the socketTimeoutMS.
'socketTimeoutMS': 60000,
}
if uri is None:
dbUriRedacted = 'mongodb://*****:*****@')
if len(parts) == 2:
dbUriRedacted = 'mongodb://' + parts[1]
else:
dbUriRedacted = uri
if replicaSet:
client = pymongo.MongoReplicaSetClient(
uri, replicaSet=replicaSet,
read_preference=ReadPreference.SECONDARY_PREFERRED,
**clientOptions)
else:
client = pymongo.MongoClient(uri, **clientOptions)
client = MongoProxy(client, logger=logger)
_dbClients[origKey] = _dbClients[(uri, replicaSet)] = client
desc = ''
if replicaSet:
desc += ', replica set: %s' % replicaSet
print(TerminalColor.info('Connected to MongoDB: %s%s' % (dbUriRedacted,
desc)))
return client
def route(self, method, route, handler, nodoc=False, resource=None):
"""
Define a route for your REST resource.
:param method: The HTTP method, e.g. 'GET', 'POST', 'PUT', 'PATCH'
:type method: str
:param route: The route, as a list of path params relative to the
resource root. Elements of this list starting with ':' are assumed
to be wildcards.
:type route: tuple
:param handler: The method to be called if the route and method are
matched by a request. Wildcards in the route will be expanded and
passed as kwargs with the same name as the wildcard identifier.
:type handler: function
:param nodoc: If your route intentionally provides no documentation,
set this to True to disable the warning on startup.
:type nodoc: bool
:param resource: The name of the resource at the root of this route.
"""
if not hasattr(self, '_routes'):
self._routes = collections.defaultdict(
lambda: collections.defaultdict(list))
# Insertion sort to maintain routes in required order.
nLengthRoutes = self._routes[method.lower()][len(route)]
for i in xrange(0, len(nLengthRoutes)):
if self._shouldInsertRoute(route, nLengthRoutes[i][0]):
nLengthRoutes.insert(i, (route, handler))
break
else:
nLengthRoutes.append((route, handler))
# Now handle the api doc if the handler has any attached
if resource is None and hasattr(self, 'resourceName'):
resource = self.resourceName
elif resource is None:
resource = handler.__module__.rsplit('.', 1)[-1]
if hasattr(handler, 'description'):
if handler.description is not None:
docs.addRouteDocs(
resource=resource, route=route, method=method,
info=handler.description.asDict(), handler=handler)
elif not nodoc:
routePath = '/'.join([resource] + list(route))
print TerminalColor.warning(
'WARNING: No description docs present for route {} {}'
.format(method, routePath))
# Warn if there is no access decorator on the handler function
if not hasattr(handler, 'accessLevel'):
routePath = '/'.join([resource] + list(route))
print TerminalColor.warning(
'WARNING: No access level specified for route {} {}'
.format(method, routePath))
def findAllPlugins(curConfig=None):
"""
Walks the plugins directory to find all of the plugins. If the plugin has
a plugin.json file, this reads that file to determine dependencies.
"""
allPlugins = {}
pluginsDir = getPluginDir(curConfig)
if not pluginsDir:
print(
TerminalColor.warning('Plugin directory not found. No plugins '
'loaded.'))
return allPlugins
dirs = [
dir for dir in os.listdir(pluginsDir)
if os.path.isdir(os.path.join(pluginsDir, dir))
]
for plugin in dirs:
data = {}
configJson = os.path.join(pluginsDir, plugin, 'plugin.json')
configYml = os.path.join(pluginsDir, plugin, 'plugin.yml')
if os.path.isfile(configJson):
with open(configJson) as conf:
try:
data = json.load(conf)
except ValueError as e:
print(
TerminalColor.error(
'ERROR: Plugin "%s": plugin.json is not valid JSON.'
% plugin))
print e
continue
elif os.path.isfile(configYml):
with open(configYml) as conf:
try:
data = yaml.safe_load(conf)
except yaml.YAMLError as e:
print(
TerminalColor.error(
'ERROR: Plugin "%s": plugin.yml is not valid YAML.'
% plugin))
print e
continue
allPlugins[plugin] = {
'name': data.get('name', plugin),
'description': data.get('description', ''),
'version': data.get('version', ''),
'dependencies': set(data.get('dependencies', []))
}
return allPlugins
def findAllPlugins(curConfig=None):
"""
Walks the plugins directories to find all of the plugins. If the plugin has
a plugin.json file, this reads that file to determine dependencies.
"""
allPlugins = {}
pluginDirs = getPluginDirs(curConfig)
if not pluginDirs:
print(TerminalColor.warning('Plugin directory not found. No plugins '
'loaded.'))
return allPlugins
for pluginDir in pluginDirs:
dirs = [dir for dir in os.listdir(pluginDir) if os.path.isdir(
os.path.join(pluginDir, dir))]
for plugin in dirs:
data = {}
configJson = os.path.join(pluginDir, plugin, 'plugin.json')
configYml = os.path.join(pluginDir, plugin, 'plugin.yml')
if os.path.isfile(configJson):
with open(configJson) as conf:
try:
data = json.load(conf)
except ValueError as e:
print(
TerminalColor.error(
('ERROR: Plugin "%s": '
'plugin.json is not valid JSON.') % plugin))
print(e)
continue
elif os.path.isfile(configYml):
with open(configYml) as conf:
try:
data = yaml.safe_load(conf)
except yaml.YAMLError as e:
print(
TerminalColor.error(
('ERROR: Plugin "%s": '
'plugin.yml is not valid YAML.') % plugin))
print(e)
continue
allPlugins[plugin] = {
'name': data.get('name', plugin),
'description': data.get('description', ''),
'version': data.get('version', ''),
'dependencies': set(data.get('dependencies', []))
}
return allPlugins
def loadPlugins(plugins, root, appconf, apiRoot=None, curConfig=None):
"""
Loads a set of plugins into the application. The list passed in should not
already contain dependency information; dependent plugins will be loaded
automatically.
:param plugins: The set of plugins to load, by directory name.
:type plugins: list
:param root: The root node of the server tree.
:param appconf: The server's cherrypy configuration object.
:type appconf: dict
:returns: A list of plugins that were actually loaded, once dependencies
were resolved and topological sort was performed.
"""
# Register a pseudo-package for the root of all plugins. This must be
# present in the system module list in order to avoid import warnings.
if curConfig is None:
curConfig = config.getConfig()
if 'plugins' in curConfig and 'plugin_directory' in curConfig['plugins']:
print(TerminalColor.warning(
'Warning: the plugin_directory setting is deprecated. Please use '
'the `girder-install plugin` command and remove this setting from '
'your config file.'))
if ROOT_PLUGINS_PACKAGE not in sys.modules:
module = imp.new_module(ROOT_PLUGINS_PACKAGE)
girder.plugins = module
sys.modules[ROOT_PLUGINS_PACKAGE] = module
print(TerminalColor.info('Resolving plugin dependencies...'))
filteredDepGraph = {
pluginName: info['dependencies']
for pluginName, info in six.viewitems(findAllPlugins(curConfig))
if pluginName in plugins
}
for pset in toposort(filteredDepGraph):
for plugin in pset:
try:
root, appconf, apiRoot = loadPlugin(
plugin, root, appconf, apiRoot, curConfig=curConfig)
print(TerminalColor.success('Loaded plugin "{}"'
.format(plugin)))
except Exception:
print(TerminalColor.error(
'ERROR: Failed to load plugin "{}":'.format(plugin)))
traceback.print_exc()
return root, appconf, apiRoot
def process_annotations(event):
"""Add annotations to an image on a ``data.process`` event"""
info = event.info
if 'anot' in info.get('file', {}).get('exts', []):
reference = info.get('reference', None)
try:
reference = json.loads(reference)
except (ValueError, TypeError):
print(
TerminalColor.error(
'Warning: Could not get reference from the annotation param. '
'Make sure you have at ctk-cli>=1.3.1 installed.'))
return
if 'userId' not in reference or 'itemId' not in reference:
print(
TerminalColor.error(
'Annotation reference does not contain required information.'
))
return
userId = reference['userId']
imageId = reference['itemId']
# load model classes
Item = ModelImporter.model('item')
File = ModelImporter.model('file')
User = ModelImporter.model('user')
Annotation = ModelImporter.model('annotation', plugin='large_image')
# load models from the database
user = User.load(userId, force=True)
image = File.load(imageId, level=AccessType.READ, user=user)
item = Item.load(image['itemId'], level=AccessType.WRITE, user=user)
file = File.load(info.get('file', {}).get('_id'),
level=AccessType.READ,
user=user)
if not (item and user and file):
print(
TerminalColor.error('Could not load models from the database'))
return
try:
data = json.loads(''.join(File.download(file)()))
except Exception:
print(TerminalColor.error('Could not parse annotation file'))
return
Annotation.createAnnotation(item, user, data)
def loadPlugins(plugins, root, appconf, apiRoot=None, curConfig=None):
"""
Loads a set of plugins into the application. The list passed in should not
already contain dependency information; dependent plugins will be loaded
automatically.
:param plugins: The set of plugins to load, by directory name.
:type plugins: list
:param root: The root node of the server tree.
:param appconf: The server's cherrypy configuration object.
:type appconf: dict
:returns: A list of plugins that were actually loaded, once dependencies
were resolved and topological sort was performed.
"""
# Register a pseudo-package for the root of all plugins. This must be
# present in the system module list in order to avoid import warnings.
if curConfig is None:
curConfig = config.getConfig()
if ROOT_PLUGINS_PACKAGE not in sys.modules:
module = imp.new_module(ROOT_PLUGINS_PACKAGE)
girder.plugins = module
sys.modules[ROOT_PLUGINS_PACKAGE] = module
print(TerminalColor.info('Resolving plugin dependencies...'))
filteredDepGraph = {
pluginName: info['dependencies']
for pluginName, info in six.iteritems(findAllPlugins(curConfig))
if pluginName in plugins
}
for pset in toposort(filteredDepGraph):
for plugin in pset:
try:
root, appconf, apiRoot = loadPlugin(plugin,
root,
appconf,
apiRoot,
curConfig=curConfig)
print(
TerminalColor.success('Loaded plugin "{}"'.format(plugin)))
except Exception:
print(
TerminalColor.error(
'ERROR: Failed to load plugin "{}":'.format(plugin)))
traceback.print_exc()
return root, appconf, apiRoot
def loadPlugins(plugins, root, appconf, apiRoot=None, curConfig=None):
"""
Loads a set of plugins into the application. The list passed in should not
already contain dependency information; dependent plugins will be loaded
automatically.
:param plugins: The set of plugins to load, by directory name.
:type plugins: list
:param root: The root node of the server tree.
:param appconf: The server's cherrypy configuration object.
:type appconf: dict
:returns: A list of plugins that were actually loaded, once dependencies
were resolved and topological sort was performed.
"""
# Register a pseudo-package for the root of all plugins. This must be
# present in the system module list in order to avoid import warnings.
if curConfig is None:
curConfig = config.getConfig()
if "plugins" in curConfig and "plugin_directory" in curConfig["plugins"]:
pluginDir = curConfig["plugins"]["plugin_directory"]
elif os.path.exists(os.path.join(PACKAGE_DIR, "plugins")):
pluginDir = os.path.join(PACKAGE_DIR, "plugins")
else:
pluginDir = os.path.join(ROOT_DIR, "plugins")
if ROOT_PLUGINS_PACKAGE not in sys.modules:
sys.modules[ROOT_PLUGINS_PACKAGE] = type(
"", (), {"__path__": pluginDir, "__package__": ROOT_PLUGINS_PACKAGE, "__name__": ROOT_PLUGINS_PACKAGE}
)()
print(TerminalColor.info("Resolving plugin dependencies..."))
filteredDepGraph = {
pluginName: info["dependencies"]
for pluginName, info in six.iteritems(findAllPlugins(curConfig))
if pluginName in plugins
}
for pset in toposort(filteredDepGraph):
for plugin in pset:
try:
root, appconf, apiRoot = loadPlugin(plugin, root, appconf, apiRoot, curConfig=curConfig)
print(TerminalColor.success('Loaded plugin "{}"'.format(plugin)))
except Exception:
print(TerminalColor.error('ERROR: Failed to load plugin "{}":'.format(plugin)))
traceback.print_exc()
return root, appconf, apiRoot
def getPluginDir(curConfig=None):
"""
Returns the /path/to the currently configured plugin directory.
"""
if curConfig is None:
curConfig = config.getConfig()
# This uses the plugin directory specified in the config first.
if 'plugins' in curConfig and 'plugin_directory' in curConfig['plugins']:
pluginsDir = curConfig['plugins']['plugin_directory']
# If none is specified, it looks if there is a plugin directory next
# to the girder python package. This is the case when running from the
# git repository.
elif os.path.isdir(os.path.join(ROOT_DIR, 'plugins')):
pluginsDir = os.path.join(ROOT_DIR, 'plugins')
# As a last resort, use plugins inside the girder python package.
# This is intended to occur when girder is pip installed.
else:
pluginsDir = os.path.join(PACKAGE_DIR, 'plugins')
if not os.path.exists(pluginsDir):
try:
os.makedirs(pluginsDir)
except OSError:
if not os.path.exists(pluginsDir):
print(
TerminalColor.warning('Could not create plugin directory.')
)
pluginsDir = None
return pluginsDir
def __init__(self):
self.name = None
self._indices = []
self._textIndex = None
self._textLanguage = None
self._filterKeys = {
AccessType.READ: set(),
AccessType.WRITE: set(),
AccessType.ADMIN: set(),
AccessType.SITE_ADMIN: set()
}
self.initialize()
db_connection = getDbConnection()
self.database = db_connection.get_default_database()
self.collection = MongoProxy(self.database[self.name])
for index in self._indices:
if isinstance(index, (list, tuple)):
self.collection.ensure_index(index[0], **index[1])
else:
self.collection.ensure_index(index)
if type(self._textIndex) is dict:
textIdx = [(k, 'text') for k in self._textIndex.keys()]
try:
self.collection.ensure_index(
textIdx,
weights=self._textIndex,
default_language=self._textLanguage)
except pymongo.errors.OperationFailure:
print(
TerminalColor.warning('WARNING: Text search not enabled.'))
def _finish_3d_coords_gen(inchikey, user, future):
resp = future.result()
query = {'inchikey': inchikey}
updates = {}
updates.setdefault('$unset', {})['generating_3d_coords'] = ''
if resp.status_code == 200:
sdf_data = resp.text
cjson = json.loads(avogadro.convert_str(sdf_data, 'sdf', 'cjson'))
cjson = whitelist_cjson(cjson)
updates.setdefault('$set', {})['cjson'] = cjson
else:
print('Generating SDF failed!')
print('Status code was:', resp.status_code)
print('Reason was:', resp.reason)
update_result = super(MoleculeModel,
MoleculeModel()).update(query, updates)
if update_result.matched_count == 0:
raise ValidationException('Invalid inchikey (%s)' % inchikey)
# Upload the molecule to virtuoso
try:
semantic.upload_molecule(MoleculeModel().findOne(query))
except requests.ConnectionError:
print(TerminalColor.warning('WARNING: Couldn\'t connect to Jena.'))
def reconnect(self):
"""
Reconnect to the database and rebuild indices if necessary. Users should
typically not have to call this method.
"""
db_connection = getDbConnection()
self.database = db_connection.get_default_database()
self.collection = MongoProxy(self.database[self.name])
for index in self._indices:
if isinstance(index, (list, tuple)):
self.collection.ensure_index(index[0], **index[1])
else:
self.collection.ensure_index(index)
if type(self._textIndex) is dict:
textIdx = [(k, 'text') for k in self._textIndex.keys()]
try:
self.collection.ensure_index(
textIdx,
weights=self._textIndex,
default_language=self._textLanguage)
except pymongo.errors.OperationFailure:
print(
TerminalColor.warning('WARNING: Text search not enabled.'))
def getPluginDir(curConfig=None):
"""
Returns the /path/to the currently configured plugin directory.
"""
if curConfig is None:
curConfig = config.getConfig()
# This uses the plugin directory specified in the config first.
if "plugins" in curConfig and "plugin_directory" in curConfig["plugins"]:
pluginsDir = curConfig["plugins"]["plugin_directory"]
# If none is specified, it looks if there is a plugin directory next
# to the girder python package. This is the case when running from the
# git repository.
elif os.path.isdir(os.path.join(ROOT_DIR, "plugins")):
pluginsDir = os.path.join(ROOT_DIR, "plugins")
# As a last resort, use plugins inside the girder python package.
# This is intended to occur when girder is pip installed.
else:
pluginsDir = os.path.join(PACKAGE_DIR, "plugins")
if not os.path.exists(pluginsDir):
try:
os.makedirs(pluginsDir)
except OSError:
if not os.path.exists(pluginsDir):
print(TerminalColor.warning("Could not create plugin directory."))
pluginsDir = None
return pluginsDir
def __init__(self):
self.name = None
self._indices = []
self._textIndex = None
self._textLanguage = None
self.initialize()
db_cfg = getDbConfig()
db_connection = getDbConnection()
dbName = db_cfg['database']
self.database = db_connection[dbName]
self.collection = self.database[self.name]
for index in self._indices:
if isinstance(index, (list, tuple)):
self.collection.ensure_index(index[0], **index[1])
else:
self.collection.ensure_index(index)
if type(self._textIndex) is dict:
textIdx = [(k, 'text') for k in self._textIndex.keys()]
try:
self.collection.ensure_index(
textIdx,
weights=self._textIndex,
default_language=self._textLanguage)
except pymongo.errors.OperationFailure:
print(
TerminalColor.warning('WARNING: Text search not enabled.'))
def getPluginDirs(curConfig=None):
"""Return an ordered list of directories that plugins can live in."""
failedPluginDirs = set()
if curConfig is None:
curConfig = config.getConfig()
if 'plugins' in curConfig and 'plugin_directory' in curConfig['plugins']:
pluginDirs = curConfig['plugins']['plugin_directory'].split(':')
else:
pluginDirs = [defaultPluginDir()]
for pluginDir in pluginDirs:
if not os.path.exists(pluginDir):
try:
os.makedirs(pluginDir)
except OSError:
if not os.path.exists(pluginDir):
print(
TerminalColor.warning(
'Could not create plugin directory %s.' %
pluginDir))
failedPluginDirs.add(pluginDir)
return [dir for dir in pluginDirs if dir not in failedPluginDirs]
def errorResponse(self, reason='A parameter was invalid.', code=400):
"""
This helper will build an errorResponse declaration for you. Many
endpoints will be able to use the default parameter values for one of
their responses.
:param reason: The reason or list of reasons why the error occurred.
:type reason: str, list, or tuple
:param code: HTTP status code.
:type code: int
"""
code = str(code)
# Combine list of reasons into a single string.
# swagger-ui renders the description using Markdown.
if not isinstance(reason, six.string_types):
reason = '\n\n'.join(reason)
if code in self._responses:
print(TerminalColor.warning(
"WARNING: Error response for code '%s' is already defined "
"(old: '%s', new: '%s')"
% (code, self._responses[code]['description'], reason)))
self._responses[code] = {
'description': reason
}
return self
def __init__(self):
self.name = None
self._indices = []
self._textIndex = None
self._textLanguage = None
self._filterKeys = {
AccessType.READ: set(),
AccessType.WRITE: set(),
AccessType.ADMIN: set(),
AccessType.SITE_ADMIN: set()
}
self.initialize()
db_connection = getDbConnection()
self.database = db_connection.get_default_database()
self.collection = MongoProxy(self.database[self.name])
for index in self._indices:
if isinstance(index, (list, tuple)):
self.collection.ensure_index(index[0], **index[1])
else:
self.collection.ensure_index(index)
if type(self._textIndex) is dict:
textIdx = [(k, 'text') for k in self._textIndex.keys()]
try:
self.collection.ensure_index(
textIdx, weights=self._textIndex,
default_language=self._textLanguage)
except pymongo.errors.OperationFailure:
print(
TerminalColor.warning('WARNING: Text search not enabled.'))
def __init__(self):
self.name = None
self._indices = []
self._textIndex = None
self._textLanguage = None
self.initialize()
db_cfg = getDbConfig()
db_connection = getDbConnection()
dbName = db_cfg['database']
self.database = db_connection[dbName]
self.collection = self.database[self.name]
for index in self._indices:
if isinstance(index, (list, tuple)):
self.collection.ensure_index(index[0], **index[1])
else:
self.collection.ensure_index(index)
if type(self._textIndex) is dict:
textIdx = [(k, 'text') for k in self._textIndex.keys()]
try:
self.collection.ensure_index(
textIdx, weights=self._textIndex,
default_language=self._textLanguage)
except pymongo.errors.OperationFailure:
print(
TerminalColor.warning('WARNING: Text search not enabled.'))
def _on_complete(mol):
# Upload the molecule to Jen
try:
semantic.upload_molecule(mol)
except requests.ConnectionError:
print(
TerminalColor.warning(
'WARNING: Couldn\'t connect to Jena.'))
def _validateParamInfo(self, dataType, paramType, name):
"""
Helper to convert and validate the dataType and paramType.
Prints warnings if invalid values were passed.
"""
# Legacy data type conversions
if dataType == 'int':
dataType = 'integer'
# Parameter Object spec:
# If type is "file", then the swagger "consumes" field MUST be either
# "multipart/form-data", "application/x-www-form-urlencoded" or both
# and the parameter MUST be in "formData".
if dataType == 'file':
paramType = 'formData'
# Get type and format from common name
dataTypeFormat = None
if dataType in self._dataTypeMap:
dataType, dataTypeFormat = self._dataTypeMap[dataType]
# If we are dealing with the body then the dataType might be defined
# by a schema added using addModel(...), we don't know for sure as we
# don't know the resource name here to look it up.
elif paramType != 'body':
print(
TerminalColor.warning(
'WARNING: Invalid dataType "%s" specified for parameter names "%s"'
% (dataType, name)))
# Parameter Object spec:
# Since the parameter is not located at the request body, it is limited
# to simple types (that is, not an object).
if paramType != 'body' and dataType not in (
'string', 'number', 'integer', 'long', 'boolean', 'array',
'file', 'float', 'double', 'date', 'dateTime'):
print(
TerminalColor.warning(
'WARNING: Invalid dataType "%s" specified for parameter "%s"'
% (dataType, name)))
if paramType == 'form':
paramType = 'formData'
return dataType, dataTypeFormat, paramType
def findEntryPointPlugins(allPlugins):
# look for plugins enabled via setuptools `entry_points`
for entry_point in iter_entry_points(group='girder.plugin'):
# set defaults
allPlugins[entry_point.name] = {
'name': entry_point.name,
'description': '',
'version': '',
'dependencies': set()
}
configJson = os.path.join('girder', 'plugin.json')
configYml = os.path.join('girder', 'plugin.yml')
data = {}
try:
if pkg_resources.resource_exists(entry_point.name, configJson):
with pkg_resources.resource_stream(entry_point.name,
configJson) as conf:
try:
data = json.load(codecs.getreader('utf8')(conf))
except ValueError as e:
print(
TerminalColor.error(
'ERROR: Plugin "%s": plugin.json is not valid '
'JSON.' % entry_point.name))
print(e)
elif pkg_resources.resource_exists(entry_point.name, configYml):
with pkg_resources.resource_stream(entry_point.name,
configYml) as conf:
try:
data = yaml.safe_load(conf)
except yaml.YAMLError as e:
print(
TerminalColor.error(
'ERROR: Plugin "%s": plugin.yml is not valid '
'YAML.' % entry_point.name))
print(e)
except ImportError:
pass
if data == {}:
data = getattr(entry_point.load(), 'config', {})
allPlugins[entry_point.name].update(data)
allPlugins[entry_point.name]['dependencies'] = set(
allPlugins[entry_point.name]['dependencies'])
def findEntryPointPlugins(allPlugins):
# look for plugins enabled via setuptools `entry_points`
for entry_point in iter_entry_points(group='girder.plugin'):
# set defaults
allPlugins[entry_point.name] = {
'name': entry_point.name,
'description': '',
'version': '',
'dependencies': set()
}
configJson = os.path.join('girder', 'plugin.json')
configYml = os.path.join('girder', 'plugin.yml')
data = {}
try:
if pkg_resources.resource_exists(entry_point.name, configJson):
with pkg_resources.resource_stream(
entry_point.name, configJson) as conf:
try:
data = json.load(codecs.getreader('utf8')(conf))
except ValueError as e:
print(
TerminalColor.error(
'ERROR: Plugin "%s": plugin.json is not valid '
'JSON.' % entry_point.name))
print(e)
elif pkg_resources.resource_exists(entry_point.name, configYml):
with pkg_resources.resource_stream(
entry_point.name, configYml) as conf:
try:
data = yaml.safe_load(conf)
except yaml.YAMLError as e:
print(
TerminalColor.error(
'ERROR: Plugin "%s": plugin.yml is not valid '
'YAML.' % entry_point.name))
print(e)
except ImportError:
pass
if data == {}:
data = getattr(entry_point.load(), 'config', {})
allPlugins[entry_point.name].update(data)
allPlugins[entry_point.name]['dependencies'] = set(
allPlugins[entry_point.name]['dependencies'])
def findAllPlugins(curConfig=None):
"""
Walks the plugins directory to find all of the plugins. If the plugin has
a plugin.json file, this reads that file to determine dependencies.
"""
allPlugins = {}
pluginsDir = getPluginDir(curConfig)
if not pluginsDir:
print(TerminalColor.warning("Plugin directory not found. No plugins " "loaded."))
return allPlugins
dirs = [dir for dir in os.listdir(pluginsDir) if os.path.isdir(os.path.join(pluginsDir, dir))]
for plugin in dirs:
data = {}
configJson = os.path.join(pluginsDir, plugin, "plugin.json")
configYml = os.path.join(pluginsDir, plugin, "plugin.yml")
if os.path.isfile(configJson):
with open(configJson) as conf:
try:
data = json.load(conf)
except ValueError as e:
print(TerminalColor.error('ERROR: Plugin "%s": plugin.json is not valid JSON.' % plugin))
print(e)
continue
elif os.path.isfile(configYml):
with open(configYml) as conf:
try:
data = yaml.safe_load(conf)
except yaml.YAMLError as e:
print(TerminalColor.error('ERROR: Plugin "%s": plugin.yml is not valid YAML.' % plugin))
print(e)
continue
allPlugins[plugin] = {
"name": data.get("name", plugin),
"description": data.get("description", ""),
"version": data.get("version", ""),
"dependencies": set(data.get("dependencies", [])),
}
return allPlugins
def route(self, method, route, handler, nodoc=False):
"""
Define a route for your REST resource.
:param method: The HTTP method, e.g. 'GET', 'POST', 'PUT'
:type method: str
:param route: The route, as a list of path params relative to the
resource root. Elements of this list starting with ':' are assumed to
be wildcards.
:type route: list
:param handler: The method to be called if the route and method are
matched by a request. Wildcards in the route will be expanded and
passed as kwargs with the same name as the wildcard identifier.
:type handler: function
:param nodoc: If your route intentionally provides no documentation,
set this to True to disable the warning on startup.
:type nodoc: bool
"""
if not hasattr(self, '_routes'):
self._routes = {
'get': [],
'post': [],
'put': [],
'delete': []
}
self._routes[method.lower()].append((route, handler))
# Now handle the api doc if the handler has any attached
resourceName = handler.im_class.__module__.rsplit('.', 1)[-1]
if hasattr(handler, 'description'):
docs.addRouteDocs(
resource=resourceName, route=route, method=method,
info=handler.description.asDict(), handler=handler)
elif not nodoc:
routePath = '/'.join([resourceName] + list(route))
print TerminalColor.warning(
'WARNING: No description docs present for route {} {}'
.format(method, routePath))
def _ensureInit(self):
"""
Calls ``Resource.__init__`` if the subclass constructor did not already
do so.
In the past, Resource subclasses were not expected to call their
superclass constructor.
"""
if not hasattr(self, '_routes'):
Resource.__init__(self)
print(TerminalColor.warning(
'WARNING: Resource subclass "%s" did not call '
'"Resource__init__()" from its constructor.' %
self.__class__.__name__))
def _ensureInit(self):
"""
Calls ``Resource.__init__`` if the subclass constructor did not already
do so.
In the past, Resource subclasses were not expected to call their
superclass constructor.
"""
if not hasattr(self, '_routes'):
Resource.__init__(self)
print(
TerminalColor.warning(
'WARNING: Resource subclass "%s" did not call '
'"Resource__init__()" from its constructor.' %
self.__class__.__name__))
def createNotebooks(event):
# If there is no current asset store, just return
try:
Assetstore().getCurrent()
except GirderException:
print(
TerminalColor.warning('WARNING: no current asset store. '
'Notebook will not be created.'))
return
user = event.info
folder_model = Folder()
result = lookUpPath('user/%s/Private' % user['login'], force=True)
private_folder = result['document']
oc_folder = folder_model.createFolder(private_folder,
'oc',
parentType='folder',
creator=user,
public=True,
reuseExisting=True)
notebook_folder = folder_model.createFolder(oc_folder,
'notebooks',
parentType='folder',
creator=user,
public=True,
reuseExisting=True)
notebooks_dir = os.path.join(os.path.dirname(__file__), 'notebooks')
upload_model = Upload()
for file in glob.glob('%s/*.ipynb' % notebooks_dir):
size = os.path.getsize(file)
name = os.path.basename(file)
with open(file, 'rb') as fp:
upload_model.uploadFromFile(
fp,
size=size,
name=name,
parentType='folder',
parent={'_id': ObjectId(notebook_folder['_id'])},
user=user,
mimeType='application/x-ipynb+json')
def addDeps(plugin):
if plugin not in allPlugins:
message = 'Required plugin %s does not exist.' % plugin
if ignoreMissing:
print(TerminalColor.error(message))
girder.logger.error(message)
return
else:
raise ValidationException(message)
deps = allPlugins[plugin]['dependencies']
dag[plugin] = deps
for dep in deps:
if dep in visited:
return
visited.add(dep)
addDeps(dep)
def addDeps(plugin):
if plugin not in allPlugins:
message = 'Required plugin %s does not exist.' % plugin
if ignoreMissing:
print(TerminalColor.error(message))
girder.logger.error(message)
return
else:
raise ValidationException(message)
deps = allPlugins[plugin]['dependencies']
dag[plugin] = deps
for dep in deps:
if dep in visited:
return
visited.add(dep)
addDeps(dep)
def addModel(name, model, resources=None, silent=False):
"""
Add a model to the Swagger documentation.
:param resources: The type(s) of resource(s) to add the model to. New
resource types may be implicitly defined, with the expectation that
routes will be added for them at some point. If no resources are
passed, the model will be exposed for every resource type
:param resources: str or tuple/list[str]
:param name: The name of the model.
:type name: str
:param model: The model to add.
:type model: dict
:param silent: Set this to True to suppress warnings.
:type silent: bool
.. warning:: This is a low-level API which does not validate the format of
``model``. See the `Swagger Model documentation`_ for a complete
specification of the correct format for ``model``.
.. versionchanged:: The syntax and behavior of this function was modified
after v1.3.2. The previous implementation did not include a resources
parameter.
.. _Swagger Model documentation: https://github.com/swagger-api/
swagger-spec/blob/d79c205485d702302003d4de2f2c980d1caf10f9/
versions/1.2.md#527-model-object
"""
if resources:
if isinstance(resources, six.string_types):
resources = (resources, )
for resource in resources:
models[resource][name] = model
else:
if not silent:
print(
TerminalColor.warning(
'WARNING: adding swagger models without specifying resources '
'to bind to is discouraged (%s).' % name))
models[None][name] = model
def addModel(name, model, resources=None, silent=False):
"""
Add a model to the Swagger documentation.
:param resources: The type(s) of resource(s) to add the model to. New
resource types may be implicitly defined, with the expectation that
routes will be added for them at some point. If no resources are
passed, the model will be exposed for every resource type
:param resources: str or tuple/list[str]
:param name: The name of the model.
:type name: str
:param model: The model to add.
:type model: dict
:param silent: Set this to True to suppress warnings.
:type silent: bool
.. warning:: This is a low-level API which does not validate the format of
``model``. See the `Swagger Model documentation`_ for a complete
specification of the correct format for ``model``.
.. versionchanged:: The syntax and behavior of this function was modified
after v1.3.2. The previous implementation did not include a resources
parameter.
.. _Swagger Model documentation: https://github.com/swagger-api/
swagger-spec/blob/d79c205485d702302003d4de2f2c980d1caf10f9/
versions/1.2.md#527-model-object
"""
if resources:
if isinstance(resources, six.string_types):
resources = (resources,)
for resource in resources:
models[resource][name] = model
else:
if not silent:
print(TerminalColor.warning(
'WARNING: adding swagger models without specifying resources '
'to bind to is discouraged (%s).' % name))
models[None][name] = model
def getPluginDirs(curConfig=None):
"""Return an ordered list of directories that plugins can live in."""
failedPluginDirs = set()
if curConfig is None:
curConfig = _config.getConfig()
if 'plugins' in curConfig and 'plugin_directory' in curConfig['plugins']:
pluginDirs = curConfig['plugins']['plugin_directory'].split(':')
else:
pluginDirs = [defaultPluginDir()]
for pluginDir in pluginDirs:
try:
mkdir(pluginDir)
except OSError:
print(TerminalColor.warning(
'Could not create plugin directory %s.' % pluginDir))
failedPluginDirs.add(pluginDir)
return [dir for dir in pluginDirs if dir not in failedPluginDirs]
def reconnect(self):
"""
Reconnect to the database and rebuild indices if necessary. Users should
typically not have to call this method.
"""
db_connection = getDbConnection()
self.database = db_connection.get_default_database()
self.collection = MongoProxy(self.database[self.name])
for index in self._indices:
if isinstance(index, (list, tuple)):
self.collection.create_index(index[0], **index[1])
else:
self.collection.create_index(index)
if type(self._textIndex) is dict:
textIdx = [(k, 'text') for k in six.viewkeys(self._textIndex)]
try:
self.collection.create_index(
textIdx, weights=self._textIndex,
default_language=self._textLanguage)
except pymongo.errors.OperationFailure:
print(
TerminalColor.warning('WARNING: Text search not enabled.'))
def loadPlugins(plugins, root, appconf):
"""
Loads a set of plugins into the application. The list passed in should not
already contain dependency information; dependent plugins will be loaded
automatically.
:param plugins: The set of plugins to load, by directory name.
:type plugins: list
:param root: The root node of the server tree.
:param appconf: The server's cherrypy configuration object.
:type appconf: dict
:returns: A list of plugins that were actually loaded, once dependencies
were resolved and topological sort was performed.
"""
# Register a pseudo-package for the root of all plugins. This must be
# present in the system module list in order to avoid import warnings.
if ROOT_PLUGINS_PACKAGE not in sys.modules:
sys.modules[ROOT_PLUGINS_PACKAGE] = type(
'', (), {
'__path__': os.path.join(ROOT_DIR, 'plugins'),
'__package__': ROOT_PLUGINS_PACKAGE,
'__name__': ROOT_PLUGINS_PACKAGE
})()
print TerminalColor.info('Resolving plugin dependencies...')
filteredDepGraph = {
pluginName: info['dependencies']
for pluginName, info in findAllPlugins().iteritems()
if pluginName in plugins
}
for pset in toposort(filteredDepGraph):
for plugin in pset:
try:
loadPlugin(plugin, root, appconf)
print TerminalColor.success(
'Loaded plugin "{}"'.format(plugin))
except:
print TerminalColor.error(
'ERROR: Failed to load plugin "{}":'.format(plugin))
traceback.print_exc()
def loadPlugins(plugins, root):
"""
Loads a set of plugins into the application. The list passed in should not
already contain dependency information; dependent plugins will be loaded
automatically.
:param plugins: The set of plugins to load, by directory name.
:type plugins: list
:param root: The root node of the server tree.
:returns: A list of plugins that were actually loaded, once dependencies
were resolved and topological sort was performed.
"""
# Register a pseudo-package for the root of all plugins. This must be
# present in the system module list in order to avoid import warnings.
if not ROOT_PLUGINS_PACKAGE in sys.modules:
sys.modules[ROOT_PLUGINS_PACKAGE] = type(
"",
(),
{
"__path__": os.path.join(ROOT_DIR, "plugins"),
"__package__": ROOT_PLUGINS_PACKAGE,
"__name__": ROOT_PLUGINS_PACKAGE,
},
)()
print TerminalColor.info("Resolving plugin dependencies...")
filteredDepGraph = {
pluginName: info["dependencies"] for pluginName, info in findAllPlugins().iteritems() if pluginName in plugins
}
for pset in toposort(filteredDepGraph):
for plugin in pset:
try:
loadPlugin(plugin, root)
print TerminalColor.success('Loaded plugin "{}"'.format(plugin))
except:
print TerminalColor.error('ERROR: Failed to load plugin "{}":'.format(plugin))
traceback.print_exc()
def create(self, params):
body = self.getBodyJson()
user = self.getCurrentUser()
public = body.get('public', False)
if 'fileId' in body:
file_id = body['fileId']
calc_id = body.get('calculationId')
file = self.model('file').load(file_id, user=user)
parts = file['name'].split('.')
input_format = parts[-1]
name = '.'.join(parts[:-1])
if input_format not in Molecule.input_formats:
raise RestException('Input format not supported.', code=400)
contents = functools.reduce(
lambda x, y: x + y,
self.model('file').download(file, headers=False)())
data_str = contents.decode()
# For now piggy backing experimental results upload here!
# This should be refactored ...
json_data = json.loads(data_str)
if 'experiment' in json_data:
return self._process_experimental(json_data)
# Use the SDF format as it is the one with bonding that 3Dmol uses.
output_format = 'sdf'
if input_format == 'pdb':
(output, _) = openbabel.convert_str(data_str, input_format,
output_format)
else:
output = avogadro.convert_str(data_str, input_format,
output_format)
# Get some basic molecular properties we want to add to the database.
props = avogadro.molecule_properties(data_str, input_format)
pieces = props['spacedFormula'].strip().split(' ')
atomCounts = {}
for i in range(0, int(len(pieces) / 2)):
atomCounts[pieces[2 * i]] = int(pieces[2 * i + 1])
cjson = []
if input_format == 'cjson':
cjson = json.loads(data_str)
elif input_format == 'pdb':
cjson = json.loads(avogadro.convert_str(
output, 'sdf', 'cjson'))
else:
cjson = json.loads(
avogadro.convert_str(data_str, input_format, 'cjson'))
atom_count = openbabel.atom_count(data_str, input_format)
if atom_count > 1024:
raise RestException(
'Unable to generate inchi, molecule has more than 1024 atoms .',
code=400)
(inchi, inchikey) = openbabel.to_inchi(output, 'sdf')
if not inchi:
raise RestException('Unable to extract inchi', code=400)
# Check if the molecule exists, only create it if it does.
molExists = self._model.find_inchikey(inchikey)
mol = {}
if molExists:
mol = molExists
else:
# Whitelist parts of the CJSON that we store at the top level.
cjsonmol = {}
cjsonmol['atoms'] = cjson['atoms']
cjsonmol['bonds'] = cjson['bonds']
cjsonmol['chemical json'] = cjson['chemical json']
mol = self._model.create_xyz(
user, {
'name':
chemspider.find_common_name(inchikey,
props['formula']),
'inchi':
inchi,
'inchikey':
inchikey,
output_format:
output,
'cjson':
cjsonmol,
'properties':
props,
'atomCounts':
atomCounts
}, public)
# Upload the molecule to virtuoso
try:
semantic.upload_molecule(mol)
except requests.ConnectionError:
print(
TerminalColor.warning(
'WARNING: Couldn\'t connect to virtuoso.'))
if 'vibrations' in cjson or 'basisSet' in cjson:
# We have some calculation data, let's add it to the calcs.
sdf = output
moleculeId = mol['_id']
calc_props = {}
if calc_id is not None:
calc = self._calc_model.load(calc_id,
user=user,
level=AccessType.ADMIN)
calc_props = calc['properties']
# The calculation is no longer pending
if 'pending' in calc_props:
del calc_props['pending']
if input_format == 'json':
jsonInput = json.loads(data_str)
# Don't override existing properties
new_calc_props = avogadro.calculation_properties(jsonInput)
new_calc_props.update(calc_props)
calc_props = new_calc_props
# Use basisSet from cjson if we don't already have one.
if 'basisSet' in cjson and 'basisSet' not in calc_props:
calc_props['basisSet'] = cjson['basisSet']
# Use functional from cjson properties if we don't already have
# one.
functional = parse('properties.functional').find(cjson)
if functional and 'functional' not in calc_props:
calc_props['functional'] = functional[0].value
# Add theory priority to 'sort' calculations
theory = calc_props.get('theory')
functional = calc_props.get('functional')
if theory in constants.theory_priority:
priority = constants.theory_priority[theory]
calc_props['theoryPriority'] = priority
if calc_id is not None:
calc['properties'] = calc_props
calc['cjson'] = cjson
calc['fileId'] = file_id
self._calc_model.save(calc)
else:
self._calc_model.create_cjson(user, cjson, calc_props,
moleculeId, file_id, public)
elif 'xyz' in body or 'sdf' in body:
if 'xyz' in body:
input_format = 'xyz'
data = body['xyz']
else:
input_format = 'sdf'
data = body['sdf']
(inchi, inchikey) = openbabel.to_inchi(data, input_format)
mol = {'inchi': inchi, 'inchikey': inchikey, input_format: data}
if 'name' in body:
mol['name'] = body['name']
mol = self._model.create_xyz(user, mol, public)
elif 'inchi' in body:
inchi = body['inchi']
mol = self._model.create(user, inchi, public)
else:
raise RestException('Invalid request', code=400)
return self._clean(mol)
def param(self, name, description, paramType='query', dataType='string',
required=True, enum=None, default=None):
"""
This helper will build a parameter declaration for you. It has the most
common options as defaults, so you won't have to repeat yourself as much
when declaring the APIs.
Note that we could expose more parameters from the Parameter Object
spec, for example: format, allowEmptyValue, minimum, maximum, pattern,
uniqueItems.
:param name: name of the parameter used in the REST query.
:param description: explanation of the parameter.
:param paramType: how is the parameter sent. One of 'query', 'path',
'body', 'header', or 'formData'.
:param dataType: the data type expected in the parameter. This is one
of 'integer', 'long', 'float', 'double', 'string',
'byte', 'binary', 'boolean', 'date', 'dateTime',
'password', or 'file'.
:param required: True if the request will fail if this parameter is not
present, False if the parameter is optional.
:param enum: a fixed list of possible values for the field.
"""
# Legacy data type conversions
if dataType == 'int':
dataType = 'integer'
elif dataType == 'File':
print(TerminalColor.warning(
"WARNING: dataType 'File' should be updated to 'file'"))
dataType = 'file'
# Get type and format from common name
dataTypeFormat = None
if dataType in self._dataTypeMap:
(dataType, dataTypeFormat) = self._dataTypeMap[dataType]
else:
print(TerminalColor.warning(
"WARNING: Invalid dataType '%s' specified for parameter "
"named '%s'" % (dataType, name)))
# Parameter Object spec:
# Since the parameter is not located at the request body, it is limited
# to simple types (that is, not an object).
if paramType != 'body':
if dataType not in ('string', 'number', 'integer', 'boolean',
'array', 'file'):
print(TerminalColor.warning(
"WARNING: Invalid dataType '%s' specified for parameter "
"named '%s'" % (dataType, name)))
if paramType == 'form':
print(TerminalColor.warning(
"WARNING: paramType 'form' should be updated to 'formData'"))
paramType = 'formData'
# Parameter Object spec:
# If type is "file", then consumes MUST be either
# "multipart/form-data", "application/x-www-form-urlencoded" or both
# and the parameter MUST be in "formData".
if dataType == 'file':
if paramType != 'formData':
print(TerminalColor.warning(
"WARNING: Invalid paramType '%s' specified for dataType "
"'file' in parameter named '%s'"
% (paramType, name)))
paramType = 'formData'
param = {
'name': name,
'description': description,
'in': paramType,
'required': required
}
if paramType == 'body':
param['schema'] = {
'type': dataType
}
else:
param['type'] = dataType
if dataTypeFormat is not None:
param['format'] = dataTypeFormat
if enum:
param['enum'] = enum
if default is not None:
param['default'] = default
self._params.append(param)
return self
def create_molecule(data_str,
input_format,
user,
public,
gen3d=True,
provenance='uploaded by user'):
using_2d_format = (input_format in openbabel_2d_formats)
inchi_format = 'inchi'
if using_2d_format or input_format in openbabel_3d_formats:
inchi, inchikey = openbabel.to_inchi(data_str, input_format)
else:
sdf_data = avogadro.convert_str(data_str, input_format, 'sdf')
inchi, inchikey = openbabel.to_inchi(sdf_data, 'sdf')
if not inchi:
raise RestException('Unable to extract InChI', code=400)
# Check if the molecule exists, only create it if it does.
molExists = MoleculeModel().find_inchikey(inchikey)
mol = {}
if molExists:
mol = molExists
else:
# Get some basic molecular properties we want to add to the
# database.
# Use sdf without 3d generation for avogadro's molecule properties
sdf_no_3d = openbabel.gen_sdf_no_3d(inchi, inchi_format)[0]
props = avogadro.molecule_properties(sdf_no_3d, 'sdf')
smiles = openbabel.to_smiles(inchi, inchi_format)
pieces = props['spacedFormula'].strip().split(' ')
atomCounts = {}
for i in range(0, int(len(pieces) / 2)):
atomCounts[pieces[2 * i]] = int(pieces[2 * i + 1])
mol_dict = {
'inchi': inchi,
'inchikey': inchikey,
'smiles': smiles,
'properties': props,
'atomCounts': atomCounts,
'provenance': provenance
}
# Generate an svg file for an image
schedule_svg_gen(mol_dict, user)
# Set a name if we find one
name = chemspider.find_common_name(inchikey)
if name is not None:
mol_dict['name'] = name
cjson = {}
if input_format == 'cjson':
cjson = json.loads(data_str)
if not cjson and using_2d_format:
# Generate 3d coordinates in a background thread
if gen3d:
schedule_3d_coords_gen(mol_dict, user)
# This will be complete other than the cjson
return MoleculeModel().create(user, mol_dict, public)
else:
if input_format in openbabel_3d_formats:
sdf_data, mime = openbabel.convert_str(data_str, input_format,
'sdf')
cjson = json.loads(
avogadro.convert_str(sdf_data, 'sdf', 'cjson'))
else:
cjson = json.loads(
avogadro.convert_str(data_str, input_format, 'cjson'))
mol_dict['cjson'] = whitelist_cjson(cjson)
mol = MoleculeModel().create(user, mol_dict, public)
# Upload the molecule to virtuoso
try:
semantic.upload_molecule(mol)
except requests.ConnectionError:
print(TerminalColor.warning('WARNING: Couldn\'t connect to Jena.'))
return mol
import os
from chemspipy import ChemSpider
from girder.constants import TerminalColor
try:
chemspikey = os.environ['chemspikey']
except KeyError:
chemspikey = None
print(
TerminalColor.warning(
'WARNING: chemspikey not set, common names will not be resolved.'))
def find_common_name(inchikey):
# Try to find the common name for the compound, if not, return None.
name = None
if chemspikey:
cs = ChemSpider(chemspikey)
if (len(inchikey) > 0):
result = cs.search(inchikey)
if (len(result) == 1):
name = result[0].common_name
return name
def findAllPlugins(curConfig=None):
"""
Walks the plugins directories to find all of the plugins. If the plugin has
a plugin.json file, this reads that file to determine dependencies.
"""
allPlugins = {}
# look for plugins enabled via setuptools `entry_points`
for entry_point in iter_entry_points(group='girder.plugin'):
# set defaults
allPlugins[entry_point.name] = {
'name': entry_point.name,
'description': '',
'version': '',
'dependencies': set()
}
allPlugins[entry_point.name].update(
getattr(entry_point.load(), 'config', {})
)
pluginDirs = getPluginDirs(curConfig)
if not pluginDirs:
print(TerminalColor.warning('Plugin directory not found.'))
return allPlugins
for pluginDir in pluginDirs:
dirs = [dir for dir in os.listdir(pluginDir) if os.path.isdir(
os.path.join(pluginDir, dir))]
for plugin in dirs:
data = {}
configJson = os.path.join(pluginDir, plugin, 'plugin.json')
configYml = os.path.join(pluginDir, plugin, 'plugin.yml')
if os.path.isfile(configJson):
with open(configJson) as conf:
try:
data = json.load(conf)
except ValueError as e:
print(
TerminalColor.error(
('ERROR: Plugin "%s": '
'plugin.json is not valid JSON.') % plugin))
print(e)
elif os.path.isfile(configYml):
with open(configYml) as conf:
try:
data = yaml.safe_load(conf)
except yaml.YAMLError as e:
print(
TerminalColor.error(
('ERROR: Plugin "%s": '
'plugin.yml is not valid YAML.') % plugin))
print(e)
allPlugins[plugin] = {
'name': data.get('name', plugin),
'description': data.get('description', ''),
'version': data.get('version', ''),
'dependencies': set(data.get('dependencies', []))
}
return allPlugins
from .. import yaml_importer # noqa
# This is the list of jobs we want to load. It could be changed to search the
# local directory for .yml files
_jobList = ['kmeans', 'surv', 'iGPSe', 'iGPSePart2', 'silhouette']
# This is the list of jobs we succeeded in loading. Other modules can use this
# list to determine which jobs are available
jobList = []
# from each job, import doc as (job)
for job in _jobList:
try:
_temp = __import__(job, globals(), locals(), ['doc'], -1)
if not 'task' in _temp.doc:
print(TerminalColor.error(
'ERROR: Job not specified peroperly "%s":' % job))
girder.logger.info('Job not specified properly: %s' % job)
continue
globals()[job] = _temp.doc
jobList.append(job)
except yaml.parser.ParserError:
print(TerminalColor.error(
'ERROR: Failed to parse job "%s":' % job))
girder.logger.exception('Job yaml parse error: %s' % job)
except Exception:
print(TerminalColor.error(
'ERROR: Failed to load job "%s":' % job))
girder.logger.exception('Job load failure: %s' % job)
__all__ = jobList
def route(self, method, route, handler, nodoc=False, resource=None):
"""
Define a route for your REST resource.
:param method: The HTTP method, e.g. 'GET', 'POST', 'PUT'
:type method: str
:param route: The route, as a list of path params relative to the
resource root. Elements of this list starting with ':' are assumed to
be wildcards.
:type route: list
:param handler: The method to be called if the route and method are
matched by a request. Wildcards in the route will be expanded and
passed as kwargs with the same name as the wildcard identifier.
:type handler: function
:param nodoc: If your route intentionally provides no documentation,
set this to True to disable the warning on startup.
:type nodoc: bool
"""
if not hasattr(self, '_routes'):
self._routes = collections.defaultdict(
lambda: collections.defaultdict(list))
# Insertion sort to maintain routes in required order.
def shouldInsert(a, b):
"""
Return bool representing whether route a should go before b. Checks
by comparing each token in order and making sure routes with
literals in forward positions come before routes with wildcards
in those positions.
"""
for i in xrange(0, len(a)):
if a[i][0] != ':' and b[i][0] == ':':
return True
return False
nLengthRoutes = self._routes[method.lower()][len(route)]
for i in xrange(0, len(nLengthRoutes)):
if shouldInsert(route, nLengthRoutes[i][0]):
nLengthRoutes.insert(i, (route, handler))
break
else:
nLengthRoutes.append((route, handler))
# Now handle the api doc if the handler has any attached
if resource is None and hasattr(self, 'resourceName'):
resource = self.resourceName
elif resource is None:
resource = handler.__module__.rsplit('.', 1)[-1]
if hasattr(handler, 'description'):
if handler.description is not None:
docs.addRouteDocs(resource=resource,
route=route,
method=method,
info=handler.description.asDict(),
handler=handler)
elif not nodoc:
routePath = '/'.join([resource] + list(route))
print TerminalColor.warning(
'WARNING: No description docs present for route {} {}'.format(
method, routePath))
def create_molecule(data_str, input_format, user, public):
# Use the SDF format as it is the one with bonding that 3Dmol uses.
sdf_format = 'sdf'
if input_format == 'pdb':
(sdf_data, _) = openbabel.convert_str(data_str, input_format, sdf_format)
elif input_format == 'inchi':
(sdf_data, _) = openbabel.from_inchi(data_str, sdf_format)
elif input_format == 'smi' or input_format == 'smiles':
(sdf_data, _) = openbabel.from_smiles(data_str, sdf_format)
else:
sdf_data = avogadro.convert_str(data_str, input_format, sdf_format)
atom_count = openbabel.atom_count(sdf_data, sdf_format)
if atom_count > 1024:
raise RestException('Unable to generate inchi, molecule has more than 1024 atoms .', code=400)
(inchi, inchikey) = openbabel.to_inchi(sdf_data, sdf_format)
if not inchi:
raise RestException('Unable to extract inchi', code=400)
# Check if the molecule exists, only create it if it does.
molExists = MoleculeModel().find_inchikey(inchikey)
mol = {}
if molExists:
mol = molExists
else:
# Get some basic molecular properties we want to add to the
# database.
props = avogadro.molecule_properties(sdf_data, sdf_format)
pieces = props['spacedFormula'].strip().split(' ')
atomCounts = {}
for i in range(0, int(len(pieces) / 2)):
atomCounts[pieces[2 * i ]] = int(pieces[2 * i + 1])
cjson = {}
if input_format == 'cjson':
cjson = json.loads(data_str)
else:
cjson = json.loads(avogadro.convert_str(sdf_data, sdf_format,
'cjson'))
smiles = openbabel.to_smiles(sdf_data, sdf_format)
# Generate an svg file for an image
svg_data = openbabel.to_svg(smiles, 'smiles')
# Find the cjson version key
version_key = 'chemicalJson'
if version_key not in cjson:
if 'chemical json' in cjson:
version_key = 'chemical json'
else:
raise RestException('No "chemicalJson" key found', 400)
# Whitelist parts of the CJSON that we store at the top level.
cjsonmol = {}
cjsonmol['atoms'] = cjson['atoms']
cjsonmol['bonds'] = cjson['bonds']
cjsonmol['chemicalJson'] = cjson[version_key]
mol_dict = {
'inchi': inchi,
'inchikey': inchikey,
'smiles': smiles,
sdf_format: sdf_data,
'cjson': cjsonmol,
'properties': props,
'atomCounts': atomCounts,
'svg': svg_data
}
# Set a name if we find one
name = chemspider.find_common_name(inchikey)
if name is not None:
mol_dict['name'] = name
mol = MoleculeModel().create(user, mol_dict, public)
# Upload the molecule to virtuoso
try:
semantic.upload_molecule(mol)
except requests.ConnectionError:
print(TerminalColor.warning('WARNING: Couldn\'t connect to Jena.'))
return mol
def loadPlugins(plugins,
root,
appconf,
apiRoot=None,
curConfig=None,
buildDag=True):
"""
Loads a set of plugins into the application.
:param plugins: The set of plugins to load, by directory name.
:type plugins: list
:param root: The root node of the server tree.
:type root: object
:param appconf: The server's cherrypy configuration object.
:type appconf: dict
:param apiRoot: The cherrypy api root object.
:type apiRoot: object or None
:param curConfig: A girder config object to use.
:type curConfig: dict or None
:param buildDag: If the ``plugins`` parameter is already a topo-sorted list
with all dependencies resolved, set this to False and it will skip
rebuilding the DAG. Otherwise the dependency resolution and sorting
will occur within this method.
:type buildDag: bool
:returns: A 3-tuple containing the modified root, config, and apiRoot
objects.
:rtype tuple:
"""
# Register a pseudo-package for the root of all plugins. This must be
# present in the system module list in order to avoid import warnings.
if curConfig is None:
curConfig = config.getConfig()
if 'plugins' in curConfig and 'plugin_directory' in curConfig['plugins']:
print(
TerminalColor.warning(
'Warning: the plugin_directory setting is deprecated. Please use '
'the `girder-install plugin` command and remove this setting from '
'your config file.'))
if ROOT_PLUGINS_PACKAGE not in sys.modules:
module = imp.new_module(ROOT_PLUGINS_PACKAGE)
girder.plugins = module
sys.modules[ROOT_PLUGINS_PACKAGE] = module
print(TerminalColor.info('Resolving plugin dependencies...'))
if buildDag:
plugins = getToposortedPlugins(plugins, curConfig, ignoreMissing=True)
for plugin in plugins:
try:
root, appconf, apiRoot = loadPlugin(plugin,
root,
appconf,
apiRoot,
curConfig=curConfig)
print(TerminalColor.success('Loaded plugin "%s"' % plugin))
except Exception:
print(
TerminalColor.error('ERROR: Failed to load plugin "%s":' %
plugin))
girder.logger.exception('Plugin load failure: %s' % plugin)
traceback.print_exc()
return root, appconf, apiRoot
continue
# For each of our sources, try to import the named class from the
# source module
className = source['className']
sourceModule = __import__(
source['moduleName'].lstrip('.'), globals(), locals(), [className],
len(source['moduleName']) - len(source['moduleName'].lstrip('.')))
sourceClass = getattr(sourceModule, className)
# Add the source class to the locals name so that it can be reached by
# importing the tilesource module
locals().update({className: sourceClass})
# add it to our list of exports
all.append(sourceClass)
# add it to our dictionary of available sources if it has a name
if getattr(sourceClass, 'name', None):
AvailableTileSources[sourceClass.name] = sourceClass
except ImportError:
if girder:
print(TerminalColor.error('Error: Could not import %s' % className))
logger.exception('Error: Could not import %s' % className)
else:
logger.warning('Error: Could not import %s' % className)
# Create a partial function that will work through the known functions to get a
# tile source.
getTileSource = functools.partial(getTileSourceFromDict,
AvailableTileSources)
all.append(getTileSource)
__all__ = all
def getDbConnection(uri=None, replicaSet=None, autoRetry=True, **kwargs):
"""
Get a MongoClient object that is connected to the configured database.
We lazy-instantiate a module-level singleton, the MongoClient objects
manage their own connection pools internally. Any extra kwargs you pass to
this method will be passed through to the MongoClient.
:param uri: if specified, connect to this mongo db rather than the one in
the config.
:param replicaSet: if uri is specified, use this replica set.
:param autoRetry: if this connection should automatically retry operations
in the event of an AutoReconnect exception. If you're testing the
connection, set this to False. If disabled, this also will not cache
the mongo client, so make sure to only disable if you're testing a
connection.
:type autoRetry: bool
"""
global _dbClients
origKey = (uri, replicaSet)
if origKey in _dbClients:
return _dbClients[origKey]
if uri is None or uri == '':
dbConf = getDbConfig()
uri = dbConf.get('uri')
replicaSet = dbConf.get('replica_set')
clientOptions = {
# This is the maximum time between when we fetch data from a cursor.
# If it times out, the cursor is lost and we can't reconnect. If it
# isn't set, we have issues with replica sets when the primary goes
# down. This value can be overridden in the mongodb uri connection
# string with the socketTimeoutMS.
'socketTimeoutMS': 60000,
'connectTimeoutMS': 20000,
'read_preference': ReadPreference.SECONDARY_PREFERRED,
'replicaSet': replicaSet
}
clientOptions.update(kwargs)
if uri is None:
dbUriRedacted = 'mongodb://*****:*****@')
if len(parts) == 2:
dbUriRedacted = 'mongodb://' + parts[1]
else:
dbUriRedacted = uri
client = pymongo.MongoClient(uri, **clientOptions)
if autoRetry:
client = MongoProxy(client, logger=logger)
_dbClients[origKey] = _dbClients[(uri, replicaSet)] = client
desc = ''
if replicaSet:
desc += ', replica set: %s' % replicaSet
print(TerminalColor.info('Connected to MongoDB: %s%s' % (dbUriRedacted,
desc)))
return client
import os
from chemspipy import ChemSpider
from girder.constants import TerminalColor
try:
chemspikey = os.environ['chemspikey']
except KeyError:
chemspikey = None
print(TerminalColor.warning('WARNING: chemspikey not set, common names will not be resolved.'))
def find_common_name(inchikey):
# Try to find the common name for the compound, if not, return None.
name = None
if chemspikey:
cs = ChemSpider(chemspikey)
if (len(inchikey) > 0):
result = cs.search(inchikey)
if (len(result) == 1):
name = result[0].common_name
return name
# limitations under the License.
###############################################################################
import datetime
import six
import string
from girder.constants import AccessType, TerminalColor, TokenScope
from .model_base import AccessControlledModel
try:
from random import SystemRandom
random = SystemRandom()
random.random() # potentially raises NotImplementedError
except NotImplementedError: # pragma: no cover
print(TerminalColor.warning(
'WARNING: using non-cryptographically secure PRNG.'))
import random
def genToken(length=64):
"""
Use this utility function to generate a random string of
a desired length.
"""
return ''.join(random.choice(string.ascii_letters + string.digits)
for x in range(length))
class Token(AccessControlledModel):
"""
This model stores session tokens for user authentication.
def getDbConnection(uri=None, replicaSet=None, autoRetry=True, **kwargs):
"""
Get a MongoClient object that is connected to the configured database.
We lazy-instantiate a module-level singleton, the MongoClient objects
manage their own connection pools internally. Any extra kwargs you pass to
this method will be passed through to the MongoClient.
:param uri: if specified, connect to this mongo db rather than the one in
the config.
:param replicaSet: if uri is specified, use this replica set.
:param autoRetry: if this connection should automatically retry operations
in the event of an AutoReconnect exception. If you're testing the
connection, set this to False. If disabled, this also will not cache
the mongo client, so make sure to only disable if you're testing a
connection.
:type autoRetry: bool
"""
global _dbClients
origKey = (uri, replicaSet)
if origKey in _dbClients:
return _dbClients[origKey]
if uri is None or uri == '':
dbConf = getDbConfig()
uri = dbConf.get('uri')
replicaSet = dbConf.get('replica_set')
clientOptions = {
# This is the maximum time between when we fetch data from a cursor.
# If it times out, the cursor is lost and we can't reconnect. If it
# isn't set, we have issues with replica sets when the primary goes
# down. This value can be overridden in the mongodb uri connection
# string with the socketTimeoutMS.
'socketTimeoutMS': 60000,
'connectTimeoutMS': 20000,
'serverSelectionTimeoutMS': 20000,
'read_preference': ReadPreference.SECONDARY_PREFERRED,
'replicaSet': replicaSet
}
clientOptions.update(kwargs)
if uri is None:
dbUriRedacted = 'mongodb://*****:*****@')
if len(parts) == 2:
dbUriRedacted = 'mongodb://' + parts[1]
else:
dbUriRedacted = uri
client = pymongo.MongoClient(uri, **clientOptions)
# Make sure we can connect to the mongo server at startup
client.server_info()
if autoRetry:
client = MongoProxy(client, logger=logger)
_dbClients[origKey] = _dbClients[(uri, replicaSet)] = client
desc = ''
if replicaSet:
desc += ', replica set: %s' % replicaSet
print(
TerminalColor.info('Connected to MongoDB: %s%s' %
(dbUriRedacted, desc)))
return client
###############################################################################
import datetime
import six
import string
from girder.constants import AccessType, TerminalColor, TokenScope
from .model_base import AccessControlledModel
try:
from random import SystemRandom
random = SystemRandom()
random.random() # potentially raises NotImplementedError
except NotImplementedError: # pragma: no cover
print(
TerminalColor.warning(
'WARNING: using non-cryptographically secure PRNG.'))
import random
def genToken(length=64):
"""
Use this utility function to generate a random string of
a desired length.
"""
return ''.join(
random.choice(string.ascii_letters + string.digits)
for x in range(length))
class Token(AccessControlledModel):
"""
