def discover():
"""Discover plugins for kubernetes based on the kubernetes service
discovery configuration(s).
Returns:
list[str]: A list of host:port addresses for plugins discovered
via kubernetes.
"""
addresses = []
cfg = config.options.get('plugin.discover.kubernetes')
if not cfg:
return addresses
# Currently, everything we want to be able to discover (namely, endpoints)
# should all be in the same namespace, so we define it globally. If other
# methods of lookup are added later, we could also have a namespace per
# resource, e.g. so we can look for endpoints in namespace X and pods in
# namespace Y, etc.
#
# If no namespace is provided via user configuration, if will default to
# the 'default' namespace.
ns = config.options.get('plugin.discover.kubernetes.namespace', 'default')
logger.debug(_('Using namespace "{}" for k8s discovery').format(ns))
# Currently, we only support plugin discovery via kubernetes service
# endpoints, under the `plugin.discover.kubernetes.endpoints` config
# field.
#
# We can support other means later.
endpoints_cfg = cfg.get('endpoints')
if endpoints_cfg:
addresses.extend(_register_from_endpoints(ns=ns, cfg=endpoints_cfg))
return addresses
def write(self, rack, board, device, data):
"""Write data to the specified device.
Args:
rack (str): The rack which the device resides on.
board (str): The board which the device resides on.
device (str): The identifier for the device to write to.
data (list[WriteData]): The data to write to the device.
Returns:
synse_grpc.api.Transactions: The transactions that can be used
to track the given write request(s).
"""
logger.debug(_('Issuing gRPC write request'))
req = synse_grpc.api.WriteInfo(
deviceFilter=synse_grpc.api.DeviceFilter(
device=device,
board=board,
rack=rack,
),
data=[d.to_grpc() for d in data])
timeout = config.options.get('grpc.timeout', None)
resp = self.grpc.Write(req, timeout=timeout)
return resp
def metainfo(self, rack=None, board=None):
"""Get all meta-information from a plugin.
Args:
rack (str): The rack to filter by.
board (str): The board to filter by.
Returns:
list[synse_plugin.api.MetainfoResponse]: All device meta-information
provided by the plugin.
"""
logger.debug(_('Issuing gRPC metainfo request'))
# If the rack or board is not specified, pass it through as an
# empty string.
rack = rack if rack is not None else ''
board = board if board is not None else ''
req = synse_api.MetainfoRequest(
rack=rack,
board=board
)
timeout = config.options.get('grpc.timeout', None)
resp = [r for r in self.stub.Metainfo(req, timeout=timeout)]
return resp
async def write_route(request, rack, board, device):
"""Write data to a known device.
The data POSTed here should be JSON with an 'action' field and 'raw'
field, if applicable. If no data is posted, the write will fail.
Args:
request (sanic.request.Request): The incoming request.
rack (str): The identifier of the rack which the device resides on.
board (str): The identifier of the board which the device resides on.
device (str): The identifier of the device to write to.
Returns:
sanic.response.HTTPResponse: The endpoint response.
"""
try:
data = request.json
except Exception as e:
raise errors.InvalidJsonError(
_('Invalid JSON specified: {}').format(request.body)) from e
logger.debug(_('Write route: POSTed JSON: {}').format(data))
if not any([x in data for x in ['action', 'raw']]):
raise errors.InvalidArgumentsError(
_('Invalid data POSTed for write. Must contain "action" and/or "raw"'
))
response = await commands.write(rack, board, device, data)
return response.to_json()
async def add_transaction(transaction_id, context, plugin_name):
"""Add a new transaction to the transaction cache.
This cache tracks transactions and maps them to the plugin from which they
originated, as well as the context of the transaction.
Args:
transaction_id (str): The ID of the transaction.
context (dict): The action/raw data of the write transaction that
can be used to help identify the transaction.
plugin_name (str): The name of the plugin to associate with the
transaction.
Returns:
bool: True if successful; False otherwise.
"""
ttl = config.options.get('cache.transaction.ttl', None)
logger.debug(
_('Caching transaction {} from plugin {} ({})').format(
transaction_id, plugin_name, context))
return await transaction_cache.set(transaction_id, {
'plugin': plugin_name,
'context': context
},
ttl=ttl)
def register_plugins():
"""Register all of the configured plugins.
Plugins can either use a unix socket or TCP for communication. Unix
socket based plugins will be detected from the presence of the socket
file in a well-known directory, or via configuration. TCP based plugins
will need to be made known to Synse Server via configuration.
Upon initialization, the Plugin instances are automatically registered
with the PluginManager.
"""
# Register plugins from local config (file, env)
unix = register_unix()
tcp = register_tcp()
# Get addresses of plugins to register via service discovery
discovered = []
addresses = kubernetes.discover()
for address in addresses:
plugin_id = register_plugin(address, 'tcp')
if plugin_id is None:
logger.error(_('Failed to register plugin with address: {}').format(address))
continue
discovered.append(plugin_id)
diff = set(Plugin.manager.plugins) - set(unix + tcp + discovered)
# Now that we have found all current plugins, we will want to clear out
# any old plugins which may no longer be present.
logger.debug(_('Plugins to purge from manager: {}').format(diff))
Plugin.manager.purge(diff)
logger.debug(_('Plugin registration complete'))
def read_cached(self, start=None, end=None):
"""Get the cached readings from a plugin. If caching readings
is disabled for the plugin, this will get a dump of the current
reading state.
Args:
start (str): An RFC3339 or RFC3339Nano formatted timestamp
which defines a starting bound on the cache data to
return. If no timestamp is specified, there will not
be a starting bound. (default: None)
end (str): An RFC3339 or RFC3339Nano formatted timestamp
which defines an ending bound on the cache data to
return. If no timestamp is specified, there will not
be an ending bound. (default: None)
Yields:
synse_grpc.api.DeviceReading: A cached reading value
with its associated device routing info.
"""
logger.debug(_('Issuing gRPC read cached request'))
bounds = synse_grpc.api.Bounds(
start=start or '',
end=end or '',
)
timeout = config.options.get('grpc.timeout', None)
for reading in self.grpc.ReadCached(bounds, timeout=timeout):
yield reading
async def scan(rack=None, board=None, force=False):
"""The handler for the Synse Server "scan" API command.
Args:
rack (str): The rack to filter the scan results by.
board (str): The board to filter the scan results by.
force (bool): Force a re-scan of the meta-information.
Returns:
ScanResponse: The "scan" response scheme model.
"""
logger.debug(_('Scan Command (args: {}, {}, force: {})').format(rack, board, force))
if force:
await cache.clear_all_meta_caches()
# Plugins are registered on scan. If no plugins exist and a scan is
# performed (e.g. on startup), we will find and register plugins.
# Additionally, if we are forcing re-scan, we will re-register plugins.
# This allows us to pick up any dynamically added plugins and clear out
# any plugins that were removed.
if len(plugin.Plugin.manager.plugins) == 0 or force:
logger.debug(_('Re-registering plugins'))
plugin.register_plugins()
cache_data = await cache.get_scan_cache()
# Filter the scan results by rack.
if rack is not None:
if not cache_data:
raise errors.FailedScanCommandError(
_('Unable to filter by resource - no scan results returned')
)
for r in cache_data['racks']:
if r['id'] == rack:
cache_data = r
break
else:
raise errors.RackNotFoundError(
_('Rack "{}" not found in scan results').format(rack)
)
# Filter the rack results by board.
if board is not None:
for b in cache_data['boards']:
if b['id'] == board:
cache_data = b
break
else:
raise errors.BoardNotFoundError(
_('Board "{}" not found in scan results').format(board)
)
return ScanResponse(
data=cache_data
)
async def check_transaction(transaction_id):
"""The handler for the Synse Server "transaction" API command.
Args:
transaction_id (str|None): The id of the transaction to check. If
the ID is None, a list of all transactions currently in the
cache is returned.
Returns:
TransactionResponse: The "transaction" response scheme model.
TransactionListResponse: The list of all transactions.
"""
logger.debug(_('Transaction Command (args: {})').format(transaction_id))
# If we are not given a transaction ID, then we want to return
# the list of all actively tracked transactions.
if transaction_id is None:
all_keys = cache.transaction_cache._cache.keys()
# Keys in the cache are prepended with 'transaction', so here we get all
# keys with that prefix and strip the prefix.
transaction_ids = [
k[11:] for k in all_keys if k.startswith('transaction')
]
return scheme.TransactionListResponse(transaction_ids)
# Otherwise, get the specified transaction.
transaction = await cache.get_transaction(transaction_id)
if not transaction:
raise errors.TransactionNotFoundError(
_('Transaction with id "{}" not found').format(transaction_id))
plugin_name = transaction.get('plugin')
context = transaction.get('context')
if not plugin_name:
# TODO - in the future, what we could do is attempt sending the transaction
# request to *all* of the known plugins. this could be useful in the event
# that synse goes down. since everything is just stored in memory, a new
# synse instance will have lost the transaction cache.
#
# alternatively, we could think about having an internal api command to
# essentially dump the active transactions so that we can rebuild the cache.
raise errors.TransactionNotFoundError(
_('Unable to determine managing plugin for transaction {}.').
format(transaction_id))
_plugin = plugin.get_plugin(plugin_name)
if not _plugin:
raise errors.PluginNotFoundError(
_('Unable to find plugin "{}"').format(plugin_name))
try:
resp = _plugin.client.transaction(transaction_id)
except grpc.RpcError as ex:
raise errors.FailedTransactionCommandError(str(ex)) from ex
return scheme.TransactionResponse(transaction_id, context, resp[0])
def purge(self, ids):
"""Remove all of the specified Plugins from the manager.
Args:
ids (list[str]): The ids of the Plugins to remove.
"""
for plugin_id in ids:
if plugin_id in self.plugins:
del self.plugins[plugin_id]
logger.debug(_('PluginManager purged plugins: {}').format(ids))
def purge(self, names):
"""Remove all of the specified Plugins from the manager.
Args:
names (list[str]): The names of the Plugins to remove.
"""
for name in names:
if name in self.plugins:
del self.plugins[name]
logger.debug(_('PluginManager purged plugins: {}').format(names))
async def capabilities():
"""The handler for the Synse Server "capabilities" API command.
Returns:
CapabilitiesResponse: The "capabilities" response scheme model.
"""
logger.debug(_('Capabilities Command'))
cache_data = await cache.get_capabilities_cache()
return CapabilitiesResponse(data=cache_data)
def metainfo(self):
"""Get the plugin metainfo.
Returns:
synse_grpc.api.Metadata: The plugin's metadata.
"""
logger.debug(_('Issuing gRPC metainfo request'))
req = synse_grpc.api.Empty()
timeout = config.options.get('grpc.timeout', None)
resp = self.grpc.Metainfo(req, timeout=timeout)
return resp
async def clear_cache(namespace):
"""Clear the cache with the given namespace.
Cache namespaces are defined in the cache module as variables with
a "NS_" prefix.
Args:
namespace (str): The namespace of the cache to clear.
"""
logger.debug(_('Invalidating cache: {}').format(namespace))
_cache = aiocache.caches.get('default')
return await _cache.clear(namespace=namespace)
def _channel(self):
"""Convenience method to create the client gRPC channel."""
if self.mode == 'unix':
target = 'unix:{}'.format(os.path.join(SOCKET_DIR, self.name + '.sock'))
elif self.mode == 'tcp':
target = self.addr
else:
raise errors.InvalidArgumentsError(
_('Invalid gRPC client mode: {}').format(self.mode)
)
logger.debug(_('Client gRPC channel: {}').format(target))
return grpc.insecure_channel(target)
def health(self):
"""Get the health of the plugin.
Returns:
synse_grpc.api.PluginHealth: The snapshot of the plugin's
health at the time the request was made.
"""
logger.debug(_('Issuing gRPC health request'))
req = synse_grpc.api.Empty()
timeout = config.options.get('grpc.timeout', None)
resp = self.grpc.Health(req, timeout=timeout)
return resp
def capabilities(self):
"""Get the plugin device capabilities.
Returns:
list[synse_grpc.api.DeviceCapability]: All device capability
information provided by the plugin.
"""
logger.debug(_('Issuing gRPC capabilities request'))
req = synse_grpc.api.Empty()
timeout = config.options.get('grpc.timeout', None)
resp = [c for c in self.grpc.Capabilities(req, timeout=timeout)]
return resp
def register_tcp_plugins():
"""Register the plugins that use TCP for communication.
Return:
list[str]: The names of all plugins that were registered.
"""
logger.debug(_('Registering plugins (tcp)'))
configured = config.options.get('plugin.tcp', {})
logger.debug(_('TCP plugins in configuration: {}').format(configured))
if not configured:
return []
manager = Plugin.manager
# Track the names of all plugins that are registered.
registered = []
for name, address in configured.items():
if manager.get(name) is None:
# A new plugin gets added to the manager on initialization.
plugin = Plugin(name=name, address=address, mode='tcp')
logger.debug(_('Created new plugin (tcp): {}').format(plugin))
else:
logger.debug(
_('TCP Plugin "{}" already exists - will not re-register').
format(name))
registered.append(name)
return list(set(registered))
async def _build_capabilities_cache():
"""Construct the list that will become the device capabilities cache.
Returns:
list: A list of dictionaries, where each dictionary corresponds to
a registered plugin. The plugin dict will identify the plugin
and enumerate the device kinds it supports and the output types
supported by those device kinds.
Raises:
errors.InternalApiError: All plugins failed the capabilities request.
"""
logger.debug(_('Building the device capabilities cache'))
capabilities = []
# First, we want to iterate through all of the known plugins and use
# their clients to get the capability info for each plugin.
plugin_count = len(Plugin.manager.plugins)
if plugin_count == 0:
logger.debug(_('Manager has no plugins - registering plugins'))
register_plugins()
plugin_count = len(Plugin.manager.plugins)
logger.debug(_('Plugins to get capabilities for: {}').format(plugin_count))
# Track which plugins failed to provide capability info for any reason.
failures = {}
# FIXME (etd): as of pylint 2.1.1, this gets marked with 'not-an-iterable'
# It still appears to work just fine, so need to figure out why it is getting
# marked as such and what should be done to fix it.
async for plugin_id, plugin in get_plugins(): # pylint: disable=not-an-iterable
logger.debug('{} - {}'.format(plugin_id, plugin))
devices = []
try:
for capability in plugin.client.capabilities():
devices.append({
'kind': capability.kind,
'outputs': capability.outputs
})
except grpc.RpcError as ex:
failures[plugin_id] = ex
logger.warning(
_('Failed to get capability for plugin: {}').format(plugin_id))
logger.warning(ex)
continue
capabilities.append({'plugin': plugin.tag, 'devices': devices})
# If we fail to read from all plugins (assuming there were any), then we
# can raise an error since it is likely something is mis-configured.
if plugin_count != 0 and plugin_count == len(failures):
raise errors.InternalApiError(
_('Failed to get capabilities for all plugins: {}').format(
failures))
return capabilities
def test(self):
"""Test that the plugin is reachable
Returns:
synse_grpc.api.Status: The status of the 'test' ping. This
should always resolve to 'ok' if the plugin is reachable,
or else result in a timeout/connection failure.
"""
logger.debug(_('Issuing gRPC test request'))
req = synse_grpc.api.Empty()
timeout = config.options.get('grpc.timeout', None)
resp = self.grpc.Test(req, timeout=timeout)
return resp
def _build_resource_info_cache(metainfo):
"""Build the resource info cache.
This builds the info cache, adhering to the Info response scheme,
using the contents of the meta-info cache.
Args:
metainfo (dict): The meta-info cache dictionary.
Returns:
dict: The constructed info cache.
"""
logger.debug(_('Building the info cache'))
info_cache = {}
for source in metainfo.values():
src = putil.device_info_to_dict(source)
rack = source.location.rack
board = source.location.board
device = source.uid
if rack in info_cache:
rdata = info_cache[rack]
if board in rdata['boards']:
bdata = rdata['boards'][board]
if device not in bdata['devices']:
bdata['devices'][device] = src
else:
rdata['boards'][board] = {
'board': board,
'devices': {
device: src
}
}
else:
info_cache[rack] = {
'rack': rack,
'boards': {
board: {
'board': board,
'devices': {
device: src
}
}
}
}
return info_cache
def remove(self, name):
"""Remove the plugin from the manager.
If a specified name does not exist in the managed plugins dictionary,
this will not fail, but it will log the event.
Args:
name (str): The name of the Plugin.
"""
if name not in self.plugins:
logger.debug(
_('"{}" is not known to PluginManager - nothing to remove').
format(name))
else:
del self.plugins[name]
async def boot_target_route(request, rack, board, device):
"""Endpoint to read/write boot target device data.
This route is an alias for the core `read` functionality when there
are no valid query parameters specified. It is an alias for the core
`write` functionality when there are valid query parameters specified.
Supported Query Parameters:
target: The boot target to set (hdd|pxe)
Args:
request (sanic.request.Request): The incoming request.
rack (str): The rack which the system resides on.
board (str): The board which the system resides on.
device (str): The system device.
Returns:
sanic.response.HTTPResponse: The endpoint response.
"""
await validate.validate_device_type(const.BOOT_TARGET_TYPES, rack, board,
device)
# Get the valid query parameters. If unsupported query parameters
# are specified, this will raise an error.
qparams = validate.validate_query_params(request.raw_args, 'target')
param_target = qparams.get('target')
# If any of the parameters are specified, this will be a write request
# using those parameters.
if param_target is not None:
logger.debug(
_('Boot target alias route: writing (query parameters: {})').
format(qparams))
if param_target not in const.boot_targets:
raise errors.InvalidArgumentsError(
_('Invalid boot target "{}". Must be one of: {}').format(
param_target, const.boot_targets))
data = {'action': 'target', 'raw': param_target}
transaction = await commands.write(rack, board, device, data)
return transaction.to_json()
# Otherwise, we just read from the device.
else:
logger.debug(_('Boot target alias route: reading'))
reading = await commands.read(rack, board, device)
return reading.to_json()
async def power_route(request, rack, board, device):
"""Endpoint to read/write power device data.
This route is an alias for the core `read` functionality when there
are no valid query parameters specified. It is an alias for the core
`write` functionality when there are valid query parameters specified.
Supported Query Parameters:
state: The power state to set (on|off|cycle)
Args:
request (sanic.request.Request): The incoming request.
rack (str): The rack which the power device resides on.
board (str): The board which the power device resides on.
device (str): The power device.
Returns:
sanic.response.HTTPResponse: The endpoint response.
"""
await validate.validate_device_type(const.TYPE_POWER, rack, board, device)
# Get the valid query parameters. If unsupported query parameters
# are specified, this will raise an error.
qparams = validate.validate_query_params(request.raw_args, 'state')
param_state = qparams.get('state')
# If any of the parameters are specified, this will be a write request
# using those parameters.
if param_state is not None:
logger.debug(
_('Power alias route: writing (query parameters: {})').format(
qparams))
if param_state not in const.power_actions:
raise errors.InvalidArgumentsError(
_('Invalid power state "{}". Must be one of: {}').format(
param_state, const.power_actions))
data = {'action': 'state', 'raw': param_state}
transaction = await commands.write(rack, board, device, data)
return transaction.to_json()
# Otherwise, we just read from the device.
else:
logger.debug(_('Power alias route: reading'))
reading = await commands.read(rack, board, device)
return reading.to_json()
def transaction(self, transaction_id):
"""Check the state of a write transaction.
Args:
transaction_id (str): The ID of the transaction to check.
Returns:
list[synse_grpc.api.WriteResponse]: The WriteResponse detailing the
status and state of the given write transaction.
"""
logger.debug(_('Issuing gRPC transaction request'))
req = synse_grpc.api.TransactionFilter(id=transaction_id)
timeout = config.options.get('grpc.timeout', None)
resp = [r for r in self.grpc.Transaction(req, timeout=timeout)]
return resp
async def lock_route(request, rack, board, device): # pylint: disable=unused-argument
"""Endpoint to read/write lock device data.
This route is an alias for the core `read` functionality when there
are no valid query parameters specified. It is an alias for the core
`write` functionality when there are valid query parameters specified.
Args:
request (sanic.request.Request): The incoming request.
rack (str): The rack which the lock resides on.
board (str): The board which the lock resides on.
device (str): The lock device.
Returns:
sanic.response.HTTPResponse: The endpoint response.
"""
await validate.validate_device_type(const.LOCK_TYPES, rack, board, device)
# Get the valid query parameters. If unsupported query parameters
# are specified, this will raise an error.
qparams = validate.validate_query_params(request.raw_args, 'action')
param_action = qparams.get('action')
# If any of the parameters are specified, this will be a write request
# using those parameters.
if param_action is not None:
logger.debug(
_('Lock alias route: writing (query parameters: {})').format(
qparams))
if param_action not in const.lock_actions:
raise errors.InvalidArgumentsError(
_('Invalid boot target "{}". Must be one of: {}').format(
param_action, const.lock_actions))
data = {
'action': param_action,
}
transaction = await commands.write(rack, board, device, data)
return transaction.to_json()
# Otherwise, we just read from the device.
else:
logger.debug(_('Lock alias route: reading'))
reading = await commands.read(rack, board, device)
return reading.to_json()
def check_transaction(self, transaction_id):
"""Check the state of a write transaction.
Args:
transaction_id (str): The ID of the transaction to check.
Returns:
synse_plugin.api.WriteResponse: The WriteResponse detailing the
status and state of the given write transaction.
"""
logger.debug(_('Issuing gRPC transaction check'))
req = synse_api.TransactionId(
id=transaction_id
)
timeout = config.options.get('grpc.timeout', None)
resp = self.stub.TransactionCheck(req, timeout=timeout)
return resp
def register_client(name, addr, mode):
"""Register a new internal client for a plugin.
Args:
name (str): The name of the plugin.
addr (str): The address which the plugin communicates over.
mode (str): The communication mode of the plugin (either
'unix' or 'tcp').
Returns:
SynseInternalClient: The client instance associated with the
name given. If a client does not exist for the given name,
a new one will be created.
"""
cli = SynseInternalClient.get_client(name)
if cli is None:
logger.debug(_('Registering new client for Plugin: {}').format(name))
cli = SynseInternalClient.register(name, addr, mode)
return cli
def register(cls, name, addr, mode):
"""Register a new client instance.
Args:
name (str): The name of the plugin for the client.
addr (str): The address the plugin will communicate over.
mode (str): The communication mode of the plugin (either
'tcp' or 'unix').
Returns:
SynseInternalClient: The newly registered client instance.
"""
SynseInternalClient(name, addr, mode)
cli = cls._client_stubs[name]
logger.debug(
_('Registered client "{}" for mode "{}", address "{}"')
.format(cli.name, cli.mode, cli.addr)
)
return cli
async def info(rack, board=None, device=None):
"""The handler for the Synse Server "info" API command.
Args:
rack (str): The rack to get information for.
board (str): The board to get information for.
device (str): The device to get information for.
Returns:
InfoResponse: The "info" response scheme model.
"""
logger.debug(
_('Info Command (args: {}, {}, {})').format(rack, board, device))
if rack is None:
raise errors.InvalidArgumentsError(
_('No rack specified when issuing info command'))
_cache = await cache.get_resource_info_cache()
r, b, d = get_resources(_cache, rack, board, device)
if board is not None:
# We have: rack, board, device
if device is not None:
response = d
# We have: rack, board
else:
response = {
'board': b['board'],
'location': {
'rack': r['rack']
},
'devices': list(b['devices'].keys())
}
else:
# We have: rack
response = {'rack': r['rack'], 'boards': list(r['boards'].keys())}
return InfoResponse(response)