def process_annotations(event): """Add annotations to an image on a ``data.process`` event""" info = event.info if 'anot' in info.get('file', {}).get('exts', []): reference = info.get('reference', None) try: reference = json.loads(reference) except (ValueError, TypeError): print(TerminalColor.error( 'Warning: Could not get reference from the annotation param. ' 'Make sure you have at ctk-cli>=1.3.1 installed.' )) return if 'userId' not in reference or 'itemId' not in reference: print(TerminalColor.error( 'Annotation reference does not contain required information.' )) return userId = reference['userId'] imageId = reference['itemId'] # load model classes Item = ModelImporter.model('item') File = ModelImporter.model('file') User = ModelImporter.model('user') Annotation = ModelImporter.model('annotation', plugin='large_image') # load models from the database user = User.load(userId, force=True) image = File.load(imageId, level=AccessType.READ, user=user) item = Item.load(image['itemId'], level=AccessType.WRITE, user=user) file = File.load( info.get('file', {}).get('_id'), level=AccessType.READ, user=user ) if not (item and user and file): print(TerminalColor.error( 'Could not load models from the database' )) return try: data = json.loads( ''.join(File.download(file)()) ) except Exception: print(TerminalColor.error( 'Could not parse annotation file' )) return Annotation.createAnnotation( item, user, data )
def loadPlugins(plugins, root, appconf, apiRoot=None, curConfig=None): """ Loads a set of plugins into the application. The list passed in should not already contain dependency information; dependent plugins will be loaded automatically. :param plugins: The set of plugins to load, by directory name. :type plugins: list :param root: The root node of the server tree. :param appconf: The server's cherrypy configuration object. :type appconf: dict :returns: A list of plugins that were actually loaded, once dependencies were resolved and topological sort was performed. """ # Register a pseudo-package for the root of all plugins. This must be # present in the system module list in order to avoid import warnings. if curConfig is None: curConfig = config.getConfig() if 'plugins' in curConfig and 'plugin_directory' in curConfig['plugins']: pluginDir = curConfig['plugins']['plugin_directory'] elif os.path.exists(os.path.join(PACKAGE_DIR, 'plugins')): pluginDir = os.path.join(PACKAGE_DIR, 'plugins') else: pluginDir = os.path.join(ROOT_DIR, 'plugins') if ROOT_PLUGINS_PACKAGE not in sys.modules: sys.modules[ROOT_PLUGINS_PACKAGE] = type( '', (), { '__path__': pluginDir, '__package__': ROOT_PLUGINS_PACKAGE, '__name__': ROOT_PLUGINS_PACKAGE })() print TerminalColor.info('Resolving plugin dependencies...') filteredDepGraph = { pluginName: info['dependencies'] for pluginName, info in findAllPlugins(curConfig).iteritems() if pluginName in plugins } for pset in toposort(filteredDepGraph): for plugin in pset: try: root, appconf, apiRoot = loadPlugin(plugin, root, appconf, apiRoot, curConfig=curConfig) print TerminalColor.success( 'Loaded plugin "{}"'.format(plugin)) except Exception: print TerminalColor.error( 'ERROR: Failed to load plugin "{}":'.format(plugin)) traceback.print_exc() return root, appconf, apiRoot
def loadPlugins(plugins, root, appconf, apiRoot=None, curConfig=None): """ Loads a set of plugins into the application. The list passed in should not already contain dependency information; dependent plugins will be loaded automatically. :param plugins: The set of plugins to load, by directory name. :type plugins: list :param root: The root node of the server tree. :param appconf: The server's cherrypy configuration object. :type appconf: dict :returns: A list of plugins that were actually loaded, once dependencies were resolved and topological sort was performed. """ # Register a pseudo-package for the root of all plugins. This must be # present in the system module list in order to avoid import warnings. if curConfig is None: curConfig = config.getConfig() if 'plugins' in curConfig and 'plugin_directory' in curConfig['plugins']: pluginDir = curConfig['plugins']['plugin_directory'] elif os.path.exists(os.path.join(PACKAGE_DIR, 'plugins')): pluginDir = os.path.join(PACKAGE_DIR, 'plugins') else: pluginDir = os.path.join(ROOT_DIR, 'plugins') if ROOT_PLUGINS_PACKAGE not in sys.modules: sys.modules[ROOT_PLUGINS_PACKAGE] = type('', (), { '__path__': pluginDir, '__package__': ROOT_PLUGINS_PACKAGE, '__name__': ROOT_PLUGINS_PACKAGE })() print TerminalColor.info('Resolving plugin dependencies...') filteredDepGraph = { pluginName: info['dependencies'] for pluginName, info in findAllPlugins(curConfig).iteritems() if pluginName in plugins } for pset in toposort(filteredDepGraph): for plugin in pset: try: root, appconf, apiRoot = loadPlugin( plugin, root, appconf, apiRoot, curConfig=curConfig) print TerminalColor.success('Loaded plugin "{}"' .format(plugin)) except Exception: print TerminalColor.error( 'ERROR: Failed to load plugin "{}":'.format(plugin)) traceback.print_exc() return root, appconf, apiRoot
def findAllPlugins(curConfig=None): """ Walks the plugins directory to find all of the plugins. If the plugin has a plugin.json file, this reads that file to determine dependencies. """ allPlugins = {} pluginsDir = getPluginDir(curConfig) if not pluginsDir: print( TerminalColor.warning('Plugin directory not found. No plugins ' 'loaded.')) return allPlugins dirs = [ dir for dir in os.listdir(pluginsDir) if os.path.isdir(os.path.join(pluginsDir, dir)) ] for plugin in dirs: data = {} configJson = os.path.join(pluginsDir, plugin, 'plugin.json') configYml = os.path.join(pluginsDir, plugin, 'plugin.yml') if os.path.isfile(configJson): with open(configJson) as conf: try: data = json.load(conf) except ValueError as e: print( TerminalColor.error( 'ERROR: Plugin "%s": plugin.json is not valid JSON.' % plugin)) print e continue elif os.path.isfile(configYml): with open(configYml) as conf: try: data = yaml.safe_load(conf) except yaml.YAMLError as e: print( TerminalColor.error( 'ERROR: Plugin "%s": plugin.yml is not valid YAML.' % plugin)) print e continue allPlugins[plugin] = { 'name': data.get('name', plugin), 'description': data.get('description', ''), 'version': data.get('version', ''), 'dependencies': set(data.get('dependencies', [])) } return allPlugins
def findAllPlugins(curConfig=None): """ Walks the plugins directories to find all of the plugins. If the plugin has a plugin.json file, this reads that file to determine dependencies. """ allPlugins = {} pluginDirs = getPluginDirs(curConfig) if not pluginDirs: print(TerminalColor.warning('Plugin directory not found. No plugins ' 'loaded.')) return allPlugins for pluginDir in pluginDirs: dirs = [dir for dir in os.listdir(pluginDir) if os.path.isdir( os.path.join(pluginDir, dir))] for plugin in dirs: data = {} configJson = os.path.join(pluginDir, plugin, 'plugin.json') configYml = os.path.join(pluginDir, plugin, 'plugin.yml') if os.path.isfile(configJson): with open(configJson) as conf: try: data = json.load(conf) except ValueError as e: print( TerminalColor.error( ('ERROR: Plugin "%s": ' 'plugin.json is not valid JSON.') % plugin)) print(e) continue elif os.path.isfile(configYml): with open(configYml) as conf: try: data = yaml.safe_load(conf) except yaml.YAMLError as e: print( TerminalColor.error( ('ERROR: Plugin "%s": ' 'plugin.yml is not valid YAML.') % plugin)) print(e) continue allPlugins[plugin] = { 'name': data.get('name', plugin), 'description': data.get('description', ''), 'version': data.get('version', ''), 'dependencies': set(data.get('dependencies', [])) } return allPlugins
def process_annotations(event): """Add annotations to an image on a ``data.process`` event""" info = event.info if 'anot' in info.get('file', {}).get('exts', []): reference = info.get('reference', None) try: reference = json.loads(reference) except (ValueError, TypeError): print( TerminalColor.error( 'Warning: Could not get reference from the annotation param. ' 'Make sure you have at ctk-cli>=1.3.1 installed.')) return if 'userId' not in reference or 'itemId' not in reference: print( TerminalColor.error( 'Annotation reference does not contain required information.' )) return userId = reference['userId'] imageId = reference['itemId'] # load model classes Item = ModelImporter.model('item') File = ModelImporter.model('file') User = ModelImporter.model('user') Annotation = ModelImporter.model('annotation', plugin='large_image') # load models from the database user = User.load(userId, force=True) image = File.load(imageId, level=AccessType.READ, user=user) item = Item.load(image['itemId'], level=AccessType.WRITE, user=user) file = File.load(info.get('file', {}).get('_id'), level=AccessType.READ, user=user) if not (item and user and file): print( TerminalColor.error('Could not load models from the database')) return try: data = json.loads(''.join(File.download(file)())) except Exception: print(TerminalColor.error('Could not parse annotation file')) return Annotation.createAnnotation(item, user, data)
def loadPlugins(plugins, root, appconf, apiRoot=None, curConfig=None, buildDag=True): """ Loads a set of plugins into the application. :param plugins: The set of plugins to load, by directory name. :type plugins: list :param root: The root node of the server tree. :type root: object :param appconf: The server's cherrypy configuration object. :type appconf: dict :param apiRoot: The cherrypy api root object. :type apiRoot: object or None :param curConfig: A girder config object to use. :type curConfig: dict or None :param buildDag: If the ``plugins`` parameter is already a topo-sorted list with all dependencies resolved, set this to False and it will skip rebuilding the DAG. Otherwise the dependency resolution and sorting will occur within this method. :type buildDag: bool :returns: A 3-tuple containing the modified root, config, and apiRoot objects. :rtype tuple: """ # Register a pseudo-package for the root of all plugins. This must be # present in the system module list in order to avoid import warnings. if curConfig is None: curConfig = _config.getConfig() if 'plugins' in curConfig and 'plugin_directory' in curConfig['plugins']: print(TerminalColor.warning( 'Warning: the plugin_directory setting is deprecated. Please use ' 'the `girder-install plugin` command and remove this setting from ' 'your config file.')) if ROOT_PLUGINS_PACKAGE not in sys.modules: module = imp.new_module(ROOT_PLUGINS_PACKAGE) girder.plugins = module sys.modules[ROOT_PLUGINS_PACKAGE] = module print(TerminalColor.info('Resolving plugin dependencies...')) if buildDag: plugins = getToposortedPlugins(plugins, curConfig, ignoreMissing=True) for plugin in plugins: try: root, appconf, apiRoot = loadPlugin( plugin, root, appconf, apiRoot, curConfig=curConfig) print(TerminalColor.success('Loaded plugin "%s"' % plugin)) except Exception: print(TerminalColor.error( 'ERROR: Failed to load plugin "%s":' % plugin)) girder.logger.exception('Plugin load failure: %s' % plugin) traceback.print_exc() return root, appconf, apiRoot
def findEntryPointPlugins(allPlugins): # look for plugins enabled via setuptools `entry_points` for entry_point in iter_entry_points(group='girder.plugin'): # set defaults allPlugins[entry_point.name] = { 'name': entry_point.name, 'description': '', 'version': '', 'dependencies': set() } configJson = os.path.join('girder', 'plugin.json') configYml = os.path.join('girder', 'plugin.yml') data = {} try: if pkg_resources.resource_exists(entry_point.name, configJson): with pkg_resources.resource_stream(entry_point.name, configJson) as conf: try: data = json.load(codecs.getreader('utf8')(conf)) except ValueError as e: print( TerminalColor.error( 'ERROR: Plugin "%s": plugin.json is not valid ' 'JSON.' % entry_point.name)) print(e) elif pkg_resources.resource_exists(entry_point.name, configYml): with pkg_resources.resource_stream(entry_point.name, configYml) as conf: try: data = yaml.safe_load(conf) except yaml.YAMLError as e: print( TerminalColor.error( 'ERROR: Plugin "%s": plugin.yml is not valid ' 'YAML.' % entry_point.name)) print(e) except ImportError: pass if data == {}: data = getattr(entry_point.load(), 'config', {}) allPlugins[entry_point.name].update(data) allPlugins[entry_point.name]['dependencies'] = set( allPlugins[entry_point.name]['dependencies'])
def findEntryPointPlugins(allPlugins): # look for plugins enabled via setuptools `entry_points` for entry_point in iter_entry_points(group='girder.plugin'): # set defaults allPlugins[entry_point.name] = { 'name': entry_point.name, 'description': '', 'version': '', 'dependencies': set() } configJson = os.path.join('girder', 'plugin.json') configYml = os.path.join('girder', 'plugin.yml') data = {} try: if pkg_resources.resource_exists(entry_point.name, configJson): with pkg_resources.resource_stream( entry_point.name, configJson) as conf: try: data = json.load(codecs.getreader('utf8')(conf)) except ValueError as e: print( TerminalColor.error( 'ERROR: Plugin "%s": plugin.json is not valid ' 'JSON.' % entry_point.name)) print(e) elif pkg_resources.resource_exists(entry_point.name, configYml): with pkg_resources.resource_stream( entry_point.name, configYml) as conf: try: data = yaml.safe_load(conf) except yaml.YAMLError as e: print( TerminalColor.error( 'ERROR: Plugin "%s": plugin.yml is not valid ' 'YAML.' % entry_point.name)) print(e) except ImportError: pass if data == {}: data = getattr(entry_point.load(), 'config', {}) allPlugins[entry_point.name].update(data) allPlugins[entry_point.name]['dependencies'] = set( allPlugins[entry_point.name]['dependencies'])
def findAllPlugins(curConfig=None): """ Walks the plugins directory to find all of the plugins. If the plugin has a plugin.json file, this reads that file to determine dependencies. """ allPlugins = {} pluginsDir = getPluginDir(curConfig) if not pluginsDir: print(TerminalColor.warning("Plugin directory not found. No plugins " "loaded.")) return allPlugins dirs = [dir for dir in os.listdir(pluginsDir) if os.path.isdir(os.path.join(pluginsDir, dir))] for plugin in dirs: data = {} configJson = os.path.join(pluginsDir, plugin, "plugin.json") configYml = os.path.join(pluginsDir, plugin, "plugin.yml") if os.path.isfile(configJson): with open(configJson) as conf: try: data = json.load(conf) except ValueError as e: print(TerminalColor.error('ERROR: Plugin "%s": plugin.json is not valid JSON.' % plugin)) print(e) continue elif os.path.isfile(configYml): with open(configYml) as conf: try: data = yaml.safe_load(conf) except yaml.YAMLError as e: print(TerminalColor.error('ERROR: Plugin "%s": plugin.yml is not valid YAML.' % plugin)) print(e) continue allPlugins[plugin] = { "name": data.get("name", plugin), "description": data.get("description", ""), "version": data.get("version", ""), "dependencies": set(data.get("dependencies", [])), } return allPlugins
def loadPlugins(plugins, root): """ Loads a set of plugins into the application. The list passed in should not already contain dependency information; dependent plugins will be loaded automatically. :param plugins: The set of plugins to load, by directory name. :type plugins: list :param root: The root node of the server tree. :returns: A list of plugins that were actually loaded, once dependencies were resolved and topological sort was performed. """ # Register a pseudo-package for the root of all plugins. This must be # present in the system module list in order to avoid import warnings. if not ROOT_PLUGINS_PACKAGE in sys.modules: sys.modules[ROOT_PLUGINS_PACKAGE] = type( "", (), { "__path__": os.path.join(ROOT_DIR, "plugins"), "__package__": ROOT_PLUGINS_PACKAGE, "__name__": ROOT_PLUGINS_PACKAGE, }, )() print TerminalColor.info("Resolving plugin dependencies...") filteredDepGraph = { pluginName: info["dependencies"] for pluginName, info in findAllPlugins().iteritems() if pluginName in plugins } for pset in toposort(filteredDepGraph): for plugin in pset: try: loadPlugin(plugin, root) print TerminalColor.success('Loaded plugin "{}"'.format(plugin)) except: print TerminalColor.error('ERROR: Failed to load plugin "{}":'.format(plugin)) traceback.print_exc()
def loadPlugins(plugins, root, appconf, apiRoot=None, curConfig=None): """ Loads a set of plugins into the application. The list passed in should not already contain dependency information; dependent plugins will be loaded automatically. :param plugins: The set of plugins to load, by directory name. :type plugins: list :param root: The root node of the server tree. :param appconf: The server's cherrypy configuration object. :type appconf: dict :returns: A list of plugins that were actually loaded, once dependencies were resolved and topological sort was performed. """ # Register a pseudo-package for the root of all plugins. This must be # present in the system module list in order to avoid import warnings. if curConfig is None: curConfig = config.getConfig() if 'plugins' in curConfig and 'plugin_directory' in curConfig['plugins']: print(TerminalColor.warning( 'Warning: the plugin_directory setting is deprecated. Please use ' 'the `girder-install plugin` command and remove this setting from ' 'your config file.')) if ROOT_PLUGINS_PACKAGE not in sys.modules: module = imp.new_module(ROOT_PLUGINS_PACKAGE) girder.plugins = module sys.modules[ROOT_PLUGINS_PACKAGE] = module print(TerminalColor.info('Resolving plugin dependencies...')) filteredDepGraph = { pluginName: info['dependencies'] for pluginName, info in six.viewitems(findAllPlugins(curConfig)) if pluginName in plugins } for pset in toposort(filteredDepGraph): for plugin in pset: try: root, appconf, apiRoot = loadPlugin( plugin, root, appconf, apiRoot, curConfig=curConfig) print(TerminalColor.success('Loaded plugin "{}"' .format(plugin))) except Exception: print(TerminalColor.error( 'ERROR: Failed to load plugin "{}":'.format(plugin))) traceback.print_exc() return root, appconf, apiRoot
def addDeps(plugin): if plugin not in allPlugins: message = 'Required plugin %s does not exist.' % plugin if ignoreMissing: print(TerminalColor.error(message)) girder.logger.error(message) return else: raise ValidationException(message) deps = allPlugins[plugin]['dependencies'] dag[plugin] = deps for dep in deps: if dep in visited: return visited.add(dep) addDeps(dep)
continue # For each of our sources, try to import the named class from the # source module className = source['className'] sourceModule = __import__( source['moduleName'].lstrip('.'), globals(), locals(), [className], len(source['moduleName']) - len(source['moduleName'].lstrip('.'))) sourceClass = getattr(sourceModule, className) # Add the source class to the locals name so that it can be reached by # importing the tilesource module locals().update({className: sourceClass}) # add it to our list of exports all.append(sourceClass) # add it to our dictionary of available sources if it has a name if getattr(sourceClass, 'name', None): AvailableTileSources[sourceClass.name] = sourceClass except ImportError: if girder: print(TerminalColor.error('Error: Could not import %s' % className)) logger.exception('Error: Could not import %s' % className) else: logger.warning('Error: Could not import %s' % className) # Create a partial function that will work through the known functions to get a # tile source. getTileSource = functools.partial(getTileSourceFromDict, AvailableTileSources) all.append(getTileSource) __all__ = all
def findAllPlugins(curConfig=None): """ Walks the plugins directories to find all of the plugins. If the plugin has a plugin.json file, this reads that file to determine dependencies. """ allPlugins = {} # look for plugins enabled via setuptools `entry_points` for entry_point in iter_entry_points(group='girder.plugin'): # set defaults allPlugins[entry_point.name] = { 'name': entry_point.name, 'description': '', 'version': '', 'dependencies': set() } allPlugins[entry_point.name].update( getattr(entry_point.load(), 'config', {}) ) pluginDirs = getPluginDirs(curConfig) if not pluginDirs: print(TerminalColor.warning('Plugin directory not found.')) return allPlugins for pluginDir in pluginDirs: dirs = [dir for dir in os.listdir(pluginDir) if os.path.isdir( os.path.join(pluginDir, dir))] for plugin in dirs: data = {} configJson = os.path.join(pluginDir, plugin, 'plugin.json') configYml = os.path.join(pluginDir, plugin, 'plugin.yml') if os.path.isfile(configJson): with open(configJson) as conf: try: data = json.load(conf) except ValueError as e: print( TerminalColor.error( ('ERROR: Plugin "%s": ' 'plugin.json is not valid JSON.') % plugin)) print(e) elif os.path.isfile(configYml): with open(configYml) as conf: try: data = yaml.safe_load(conf) except yaml.YAMLError as e: print( TerminalColor.error( ('ERROR: Plugin "%s": ' 'plugin.yml is not valid YAML.') % plugin)) print(e) allPlugins[plugin] = { 'name': data.get('name', plugin), 'description': data.get('description', ''), 'version': data.get('version', ''), 'dependencies': set(data.get('dependencies', [])) } return allPlugins
from .. import yaml_importer # noqa # This is the list of jobs we want to load. It could be changed to search the # local directory for .yml files _jobList = ['kmeans', 'surv', 'iGPSe', 'iGPSePart2', 'silhouette'] # This is the list of jobs we succeeded in loading. Other modules can use this # list to determine which jobs are available jobList = [] # from each job, import doc as (job) for job in _jobList: try: _temp = __import__(job, globals(), locals(), ['doc'], -1) if not 'task' in _temp.doc: print(TerminalColor.error( 'ERROR: Job not specified peroperly "%s":' % job)) girder.logger.info('Job not specified properly: %s' % job) continue globals()[job] = _temp.doc jobList.append(job) except yaml.parser.ParserError: print(TerminalColor.error( 'ERROR: Failed to parse job "%s":' % job)) girder.logger.exception('Job yaml parse error: %s' % job) except Exception: print(TerminalColor.error( 'ERROR: Failed to load job "%s":' % job)) girder.logger.exception('Job load failure: %s' % job) __all__ = jobList
def loadPlugins(plugins, root, appconf, apiRoot=None, curConfig=None, buildDag=True): """ Loads a set of plugins into the application. :param plugins: The set of plugins to load, by directory name. :type plugins: list :param root: The root node of the server tree. :type root: object :param appconf: The server's cherrypy configuration object. :type appconf: dict :param apiRoot: The cherrypy api root object. :type apiRoot: object or None :param curConfig: A girder config object to use. :type curConfig: dict or None :param buildDag: If the ``plugins`` parameter is already a topo-sorted list with all dependencies resolved, set this to False and it will skip rebuilding the DAG. Otherwise the dependency resolution and sorting will occur within this method. :type buildDag: bool :returns: A 3-tuple containing the modified root, config, and apiRoot objects. :rtype tuple: """ # Register a pseudo-package for the root of all plugins. This must be # present in the system module list in order to avoid import warnings. if curConfig is None: curConfig = config.getConfig() if 'plugins' in curConfig and 'plugin_directory' in curConfig['plugins']: print( TerminalColor.warning( 'Warning: the plugin_directory setting is deprecated. Please use ' 'the `girder-install plugin` command and remove this setting from ' 'your config file.')) if ROOT_PLUGINS_PACKAGE not in sys.modules: module = imp.new_module(ROOT_PLUGINS_PACKAGE) girder.plugins = module sys.modules[ROOT_PLUGINS_PACKAGE] = module print(TerminalColor.info('Resolving plugin dependencies...')) if buildDag: plugins = getToposortedPlugins(plugins, curConfig, ignoreMissing=True) for plugin in plugins: try: root, appconf, apiRoot = loadPlugin(plugin, root, appconf, apiRoot, curConfig=curConfig) print(TerminalColor.success('Loaded plugin "%s"' % plugin)) except Exception: print( TerminalColor.error('ERROR: Failed to load plugin "%s":' % plugin)) girder.logger.exception('Plugin load failure: %s' % plugin) traceback.print_exc() return root, appconf, apiRoot