def _api_plugins(plugins):
"""Return instance plugins."""
if not plugins:
return []
plugins_ns = 'treadmill.api.instance.plugins'
_LOGGER.info('Instance api plugins: %r', plugins)
return [plugin_manager.load(plugins_ns, name) for name in plugins]
def _resolver(self):
if self._dns is not None:
return self._dns
dns = plugin_manager.load('treadmill.context', 'dns')
dns.init(self._context)
self._dns = dns
return self._dns
def _api_plugins(plugins):
"""Return api plugins.
"""
if not plugins:
return []
plugins_ns = 'treadmill.api.allocation.plugins'
return [plugin_manager.load(plugins_ns, name) for name in plugins]
def conn(self):
"""Lazily establishes connection to admin LDAP."""
if self._conn:
return self._conn
plugin = plugin_manager.load('treadmill.context', 'admin')
self._conn = plugin.connect(self.url, self.ldap_suffix, self.user,
self.password)
return self._conn
def setUp(self):
"""Setup common test variables"""
self.runner = click.testing.CliRunner()
self.blackout_mod = plugin_manager.load('treadmill.cli.admin',
'blackout')
self.blackout_cli = self.blackout_mod.init()
context.GLOBAL.cell = 'test'
context.GLOBAL.zk.url = 'zookeeper://xxx@yyy:123'
context.GLOBAL.zk = mock.Mock()
def _api_plugins(initialized):
"""Return instance plugins."""
if initialized is not None:
return initialized
plugins_ns = 'treadmill.api.instance.plugins'
plugins = context.GLOBAL.get('api.instance.plugins', [])
_LOGGER.info('Instance api plugins: %r', plugins)
return [plugin_manager.load(plugins_ns, name) for name in plugins]
def get_command(self, ctx, cmd_name):
try:
return plugin_manager.load(section, cmd_name).init()
except ImportError as import_err:
print('dependency error: {}:{} - {}'.format(
section, cmd_name, str(import_err)),
file=sys.stderr)
except KeyError:
raise click.UsageError('Invalid command: %s' % cmd_name)
def _api_plugins(initialized):
"""Return instance plugins."""
if initialized is not None:
return initialized
plugins_ns = 'treadmill.api.instance.plugins'
return [
plugin_manager.load(plugins_ns, name)
for name in context.GLOBAL.get('api.instance.plugins', [])
]
def update(scheduler, job_id, event, resource, expression, count):
"""Update a job/model"""
model, action = event.split(':')
_LOGGER.debug('model: %s, action: %s', model, action)
model_module = plugin_manager.load('treadmill.cron', model)
_LOGGER.debug('model_module: %r', model_module)
return model_module.update(scheduler, job_id, model, action, resource,
expression, count)
def _cleanup_exception_rules(tm_env, container_dir, app):
"""Clean up firewall exception rules"""
try:
firewall_plugin = plugin_manager.load('treadmill.firewall.plugins',
'firewall')
firewall_plugin.cleanup_exception_rules(tm_env, container_dir, app)
except Exception: # pylint: disable=W0703
_LOGGER.exception(
'Error in firewall plugin, skip cleaning firewall exception rules.'
)
def get_runtime_cls(runtime_name):
"""Get runtime classs
Raise Key exception if runtime class does not exist
"""
try:
runtime_cls = plugin_manager.load(_RUNTIME_NAMESPACE, runtime_name)
return runtime_cls
except KeyError:
_LOGGER.error('Runtime not supported: %s', runtime_name)
raise
def _init_plugins(self):
"""Initialize plugins."""
if self._plugins:
return
_LOGGER.debug('Loading plugins.')
# TODO: Thsi is a hack, need a better way to determine if plugin
# should be loaded.
if self.get('dns_domain', resolve=False):
_LOGGER.debug('Loading dns plugin.')
dns = plugin_manager.load('treadmill.context', 'dns')
dns.init(self)
self._plugins.append(dns)
if self.get('ldap_url', resolve=False):
_LOGGER.debug('Loading admin plugin.')
ldap = plugin_manager.load('treadmill.context', 'admin')
ldap.init(self)
self._plugins.append(ldap)
def _configure(self):
"""Configures the dispatcher with the monitor actions defined in
the config directory.
"""
config = {}
for name in os.listdir(self._config_dir):
path = os.path.join(self._config_dir, name)
if not os.path.isfile(path):
continue
_LOGGER.debug('Configuring for file: %s', path)
with io.open(path) as f:
for line in f.readlines():
parts = line.rstrip().split(';', 2)
if len(parts) < 2:
_LOGGER.warning('skiping config line %s', line)
continue
try:
handler = plugin_manager.load(_TOMESTONES_PLUGINS,
parts[1])
except KeyError:
_LOGGER.warning('Tomestone handler does not exist: %r',
parts[1])
continue
params = {}
if len(parts) > 2:
params = json.loads(parts[2])
impl = handler(self._tm_env, params)
config[parts[0]] = impl
self._dirwatcher = dirwatch.DirWatcher()
self._dispatcher = dirwatch.DirWatcherDispatcher(self._dirwatcher)
self._tombstones = collections.deque()
for path, handler in six.iteritems(config):
fs.mkdir_safe(path)
self._dirwatcher.add_dir(path)
self._dispatcher.register(
path, {
dirwatch.DirWatcherEvent.CREATED:
lambda p, h=handler: self._on_created(p, h)
})
_LOGGER.info('Watching %s with handler %r', path, handler)
for name in os.listdir(path):
self._on_created(os.path.join(path, name), handler)
_LOGGER.info('Monitor configured')
def _format(item, how=None):
"""Formats the object given global format setting."""
if OUTPUT_FORMAT is None:
how = pretty_formatter
else:
how = OUTPUT_FORMAT
try:
fmt = plugin_manager.load('treadmill.formatters', how)
return fmt.format(item)
except KeyError:
return str(item)
def _load_alert_backend(plugin_name):
backend = _NoOpBackend()
if plugin_name is None:
return backend
try:
backend = plugin_manager.load('treadmill.alert.plugins', plugin_name)
except KeyError:
_LOGGER.info('Alert backend %r could not been loaded.', plugin_name)
return backend
def init(apis):
"""Module initialization."""
handlers = []
for apiname in apis:
try:
_LOGGER.info('Loading api: %s', apiname)
wsapi_mod = plugin_manager.load('treadmill.websocket.api', apiname)
handlers.extend(wsapi_mod.init())
except ImportError as err:
_LOGGER.warning('Unable to load %s api: %s', apiname, err)
return handlers
def _load_impl(self):
"""Load the implementation class of the service.
"""
if isinstance(self._service_impl, six.string_types):
impl_class = plugin_manager.load('treadmill.services',
self._service_impl)
else:
impl_class = self._service_impl
assert issubclass(impl_class, BaseResourceServiceImpl), \
'Invalid implementation %r' % impl_class
return impl_class
def _setup_rate_limit(self):
"""Setup the http request rate limit control."""
if self.rate_limit is not None:
_LOGGER.info('Starting REST (rate limit: %s) server on %s:%i',
self.rate_limit, self.host, self.port)
try:
limit = plugin_manager.load('treadmill.rest', 'limit')
limit.wrap(FLASK_APP, self.rate_limit)
except Exception:
_LOGGER.exception('Unable to setup rate limit.')
raise
else:
_LOGGER.info('Starting REST (no rate limit) server on %s:%i',
self.host, self.port)
def detect(traits):
"""Detect traits usign plugins.
"""
result = []
for trait in traits:
try:
plugin = plugin_manager.load('treadmill.server.traits', trait)
if plugin():
result.append(trait)
except Exception: # pylint: disable=W0703
_LOGGER.exception('Error processing trait plugin: %s', trait)
return result
def conn(self):
"""Lazily creates Zookeeper client.
"""
if self._conn:
return self._conn
_LOGGER.debug('Connecting to Zookeeper %s', self.url)
plugin = plugin_manager.load('treadmill.context', 'zookeeper')
self._conn = plugin.connect(self.url, idpath=self.idpath)
if self._listeners:
for listener in self._listeners:
self._conn.add_listener(listener)
return self._conn
def conn(self):
"""Lazily creates Zookeeper client."""
if self._conn:
return self._conn
_LOGGER.debug('Connecting to Zookeeper %s', self.url)
self.proid, _ = self.url[len('zookeeper://'):].split('@')
plugin = plugin_manager.load('treadmill.context', 'zookeeper')
self._conn = plugin.connect(self.url)
if self._listeners:
for listener in self._listeners:
self._conn.add_listener(listener)
return self._conn
def get_version():
"""Get server version data.
"""
distributions = {
dist.project_name: dist.version
for dist in iter(pkg_resources.working_set)
if dist.project_name.startswith(_TREADMILL_DIST_PREFIX)
}
version = {'distributions': distributions, 'since': int(time.time())}
for name in plugin_manager.names('treadmill.version'):
plugin = plugin_manager.load('treadmill.version', name)
version.update(plugin())
return version
def _load_profile(self):
"""Loads the profile."""
if not self._profile_name:
return
# Load once.
if self._defaults is not None:
return
self._defaults = {}
try:
profile_mod = plugin_manager.load('treadmill.profiles',
self._profile_name)
self._defaults = profile_mod.PROFILE
except KeyError:
_LOGGER.warning('Profile not found: %s', self._profile_name)
def _run_gc(gc_plugins, interval):
"""Garbage collector plugins executor."""
plugins = {name: plugin_manager.load(_MODULE, name) for name in gc_plugins}
servers = {name: set() for name in gc_plugins}
while True:
for name, plugin in plugins.items():
servers[name] = plugin.list()
_LOGGER.debug('snoozing for %d minutes', interval / 60)
time.sleep(interval)
for name, plugin in plugins.items():
_LOGGER.info('%s cleanup started', name.upper())
for server in servers[name] - _ec2_instances():
plugin.delete(server)
_LOGGER.info('%s cleanup complete', name.upper())
def _setup_auth(self):
"""Setup the http authentication."""
if self.auth_type is not None:
_LOGGER.info('Starting REST server: %s, auth: %s', self.socket,
self.auth_type)
try:
auth = plugin_manager.load('treadmill.rest.authentication',
self.auth_type)
FLASK_APP.wsgi_app = auth.wrap(FLASK_APP.wsgi_app)
except KeyError:
_LOGGER.error('Unsupported auth type: %s', self.auth_type)
raise
except Exception:
_LOGGER.exception('Unable to load auth plugin.')
raise
else:
_LOGGER.info('Starting REST (noauth) server on %s', self.socket)
def _run_sync(cellsync_plugins, once):
"""Sync Zookeeper with LDAP, runs with lock held.
"""
while True:
# Sync app groups
if not cellsync_plugins:
cellsync_plugins = plugin_manager.names('treadmill.cellsync')
for name in cellsync_plugins:
try:
plugin = plugin_manager.load('treadmill.cellsync', name)
plugin()
except Exception: # pylint: disable=W0703
_LOGGER.exception('Error processing sync plugin: %s', name)
if once:
return
time.sleep(60)
def setUp(self):
"""Setup common test variables"""
self.runner = click.testing.CliRunner()
self.configure = plugin_manager.load('treadmill.cli',
'configure').init()
with tempfile.NamedTemporaryFile(delete=False) as f:
yaml.dump(
{
'memory': '128M',
'cpu': '5%',
'disk': '100M',
'identity_group': 'ident-group',
},
f,
encoding='utf-8')
self.manifest = f.name
def _load_profile(self):
"""Loads the profile.
"""
# Load once.
if self._defaults is not None:
return
self._defaults = {
'dns_domain': '.'.join(socket.getfqdn().split('.')[1:])
}
if self._profile_name:
try:
profile_mod = plugin_manager.load('treadmill.profiles',
self._profile_name)
self._defaults.update(profile_mod.PROFILE)
except KeyError:
_LOGGER.warning('Profile not found: %s', self._profile_name)
def load_packages(packages, lazy=True):
"""Laze load aliases from packages.
Aliases will not be resolved.
"""
exes = {}
for name in packages:
alias_mod = plugin_manager.load('treadmill.bootstrap', name)
exes.update(getattr(alias_mod, 'ALIASES'))
tm = os.environ.get('TREADMILL')
if tm is not None:
exes['treadmill'] = tm
global _EXECUTABLES # pylint: disable=W0603
_EXECUTABLES = exes
if not lazy:
resolved = {path: resolve(path) for path in _EXECUTABLES}
_EXECUTABLES = resolved
def _run_gc(gc_plugins, interval):
"""Garbage collector plugins executor."""
plugins = {name: plugin_manager.load(_MODULE, name) for name in gc_plugins}
servers = {name: set() for name in gc_plugins}
while True:
for name, plugin in plugins.items():
servers[name] = plugin.list()
_LOGGER.info('Snoozing for %d seconds', interval)
time.sleep(interval)
for name, plugin in plugins.items():
_LOGGER.info('%s cleanup started', name.upper())
if isinstance(servers[name], dict):
for server in set(servers[name]) - _ec2_instances():
plugin.delete(server, servers[name][server])
else:
for server in servers[name] - _ec2_instances():
plugin.delete(server)
_LOGGER.info('%s cleanup completed', name.upper())
