def test_all_ready_true_has_plugins(self):
p1 = plugin.Plugin({
'id': '1',
'tag': 'foo'
}, {}, client.PluginClientV3('foo', 'tcp'))
p2 = plugin.Plugin({
'id': '2',
'tag': 'foo'
}, {}, client.PluginClientV3('foo', 'tcp'))
p3 = plugin.Plugin({
'id': '3',
'tag': 'foo'
}, {}, client.PluginClientV3('foo', 'tcp'))
p1._reconnect = asynctest.CoroutineMock()
p2._reconnect = asynctest.CoroutineMock()
p3._reconnect = asynctest.CoroutineMock()
p1.mark_active()
p2.mark_active()
p3.mark_active()
m = plugin.PluginManager()
m.plugins = {
'1': p1,
'2': p2,
'3': p3,
}
assert m.all_ready() is True
async def test_reconnect_with_retries(self, test_mock, delay_mock):
p = plugin.Plugin(
client=client.PluginClientV3('localhost:5001', 'tcp'),
info={
'tag': 'test/foo',
'id': '123',
'vcs': 'example.com'
},
version={},
)
p.disabled = True # the disabled value should not be altered via reconnect
p.active = False
await p._reconnect()
assert p.disabled is True
assert p.active is True
delay_mock.assert_has_calls([
mock.call(),
mock.call(),
])
test_mock.assert_has_calls([
mock.call(),
mock.call(),
mock.call(),
])
def test_register_duplicate_id_existing_disabled_new_address_changed(
self, mock_version, mock_metadata): # noqa
"""Plugins with the same Plugin ID are registered. The cached plugin is disabled and has
a different address than the new plugin, which is active. Synse should replace the cached
disabled instance with the new active one.
"""
m = plugin.PluginManager()
p = plugin.Plugin(
{
'id': '123',
'tag': 'foo'
},
{},
client.PluginClientV3('somewhere:6789', 'tcp'),
)
p.disabled = True
m.plugins = {'123': p}
plugin_id = m.register('localhost:5432', 'tcp')
assert plugin_id == '123'
assert len(m.plugins) == 1
assert m.plugins[plugin_id].active is True
assert m.plugins[plugin_id].disabled is False
# Since the seed plugin is disabled, the registration process should replace
# the old Plugin instance with the newly created one, which should no longer
# be disabled. This will be a new instance, so new object ID.
assert id(m.plugins[plugin_id]) != id(p)
mock_metadata.assert_called_once()
mock_version.assert_called_once()
def test_register_duplicate_plugin_id_both_active(self, mock_version,
mock_metadata):
"""Plugins with the same Plugin ID are registered. Both are considered active,
so Synse should keep the cached Plugin instance.
"""
m = plugin.PluginManager()
p = plugin.Plugin(
{
'id': 'foo',
'tag': 'foo'
},
{},
client.PluginClientV3('foo', 'tcp'),
)
m.plugins = {'123': p}
plugin_id = m.register('localhost:5432', 'tcp')
assert plugin_id == '123'
# Ensure nothing new was added to the manager.
assert len(m.plugins) == 1
assert m.plugins[plugin_id].active is True
assert m.plugins[plugin_id].disabled is False
# Since the plugins are both active, Synse keeps the existing one, so
# the object ID of the cached Plugin should not have changed from the
# seed value.
assert id(m.plugins[plugin_id]) == id(p)
mock_metadata.assert_called_once()
mock_version.assert_called_once()
async def test_plugin_client_error(mocker):
# Mock test data
mock_get = mocker.patch(
'synse_server.plugin.manager.get',
return_value=Plugin(
info={
'id': '123456',
'tag': 'test-plugin'
},
version={},
client=client.PluginClientV3('localhost:5001', 'tcp'),
),
)
mock_health = mocker.patch(
'synse_grpc.client.PluginClientV3.health',
side_effect=ValueError(),
)
# --- Test case -----------------------------
plugin_id = '123456'
with pytest.raises(errors.ServerError):
await cmd.plugin(plugin_id)
mock_get.assert_called_once()
mock_get.assert_called_with(plugin_id)
mock_health.assert_called_once()
def test_init_missing_id(self):
with pytest.raises(ValueError):
plugin.Plugin(
client=client.PluginClientV3('localhost:5432', 'tcp'),
info={
'tag': 'foo',
},
version={},
)
def test_bucket_plugins(self):
p1 = plugin.Plugin(
client=client.PluginClientV3('localhost:5001', 'tcp'),
version={},
info={
'tag': 'test/foo',
'id': '123',
'vcs': 'https://github.com/vapor-ware/synse-server',
},
)
p2 = plugin.Plugin(
client=client.PluginClientV3('localhost:5002', 'tcp'),
version={},
info={
'tag': 'test/bar',
'id': '456',
'vcs': 'https://github.com/vapor-ware/synse-server',
},
)
p1._reconnect = asynctest.CoroutineMock
p2._reconnect = asynctest.CoroutineMock
m = plugin.PluginManager()
m.plugins[p1.id] = p1
m.plugins[p2.id] = p2
existing, new, removed = m.bucket_plugins([
('localhost:5001', 'tcp'),
('localhost:5003', 'tcp'),
])
assert len(existing) == 1
assert len(new) == 1
assert len(removed) == 1
assert ('localhost:5003', 'tcp') in new
assert p1 in existing
assert p2 in removed
def test_refresh_state_fails_refresh(self, test_mock):
p = plugin.Plugin(
client=client.PluginClientV3('localhost:5001', 'tcp'),
info={'tag': 'test/foo', 'id': '123'},
version={},
)
p.disabled = False
p.active = True
p.refresh_state()
assert p.disabled is False
assert p.active is False
test_mock.assert_called_once()
def simple_plugin():
"""Fixture to return a new ``synse_server.plugin.Plugin`` instance
configured minimally.
"""
p = plugin.Plugin(
client=client.PluginClientV3('localhost:5432', 'tcp'),
info={
'tag': 'test/foo',
'id': '123',
'vcs': 'https://github.com/vapor-ware/synse-server',
},
version={},
)
p.active = True
p._reconnect = asynctest.CoroutineMock()
return p
def test_init_ok(self):
c = client.PluginClientV3('localhost:5432', 'tcp')
p = plugin.Plugin(
client=c,
info={
'tag': 'foo',
'id': '123',
},
version={},
)
assert p.active is False
assert p.client == c
assert p.address == 'localhost:5432'
assert p.protocol == 'tcp'
assert p.tag == 'foo'
assert p.id == '123'
def test_register_duplicate_plugin_id(self, mock_version, mock_metadata):
m = plugin.PluginManager()
m.plugins = {
'123': plugin.Plugin(
{'id': 'foo', 'tag': 'foo'},
{},
client.PluginClientV3('foo', 'tcp'),
),
}
plugin_id = m.register('localhost:5432', 'tcp')
assert plugin_id == '123'
# Ensure nothing new was added to the manager.
assert len(m.plugins) == 1
mock_metadata.assert_called_once()
mock_version.assert_called_once()
async def test_read_fails_read_multiple_one_fail(mocker, simple_plugin, temperature_reading):
# Mock test data
error_plugin = plugin.Plugin(
client=client.PluginClientV3('localhost:5433', 'tcp'),
info={
'tag': 'test/bar',
'id': '456',
'vcs': 'https://github.com/vapor-ware/synse-server',
},
version={},
)
mocker.patch.dict('synse_server.plugin.PluginManager.plugins', {
'123': simple_plugin,
'456': error_plugin,
})
mock_read_ok = mocker.MagicMock(
return_value=[
temperature_reading,
],
)
mock_read_error = mocker.MagicMock(
side_effect=ValueError(),
)
simple_plugin.client.read = mock_read_ok
error_plugin.client.read = mock_read_error
# --- Test case -----------------------------
# Set the plugins to active to start.
simple_plugin.active = True
error_plugin.active = True
with pytest.raises(errors.ServerError):
await cmd.read('default', [['default/foo']])
assert simple_plugin.active is True
assert error_plugin.active is False
mock_read_error.assert_called_once()
mock_read_error.assert_called_with(tags=['default/foo'])
async def test_update_device_cache_devices_rpc_error(
self, mocker, simple_plugin):
# Need to define a plugin different than simple_plugin so we have different instances.
p = plugin.Plugin(
client=client.PluginClientV3('localhost:5432', 'tcp'),
info={
'tag': 'test/bar',
'id': '456',
'vcs': 'https://github.com/vapor-ware/synse-server',
},
version={},
)
p.active = True
# Mock test data
mocker.patch.dict('synse_server.plugin.PluginManager.plugins', {
'123': simple_plugin,
'456': p,
})
mock_devices = mocker.patch(
'synse_grpc.client.PluginClientV3.devices',
side_effect=grpc.RpcError(),
)
# --- Test case -----------------------------
assert len(cache.device_cache._cache) == 0
await cache.update_device_cache()
assert len(cache.device_cache._cache) == 0
mock_devices.assert_has_calls([
mocker.call(),
mocker.call(),
])
def register(self, address: str, protocol: str) -> str:
"""Register a new Plugin with the manager.
With the provided address and communication protocol, the manager
will attempt to establish communication with the plugin to get its
metadata. If successful, it will generate a new Plugin instance for
the plugin and register it in the manager state.
Args:
address: The address of the plugin to register.
protocol: The protocol that the plugin uses. This must be
one of: 'unix', 'tcp'.
Returns:
The ID of the plugin that was registered.
"""
logger.info('registering new plugin', addr=address, protocol=protocol)
interceptors = []
if config.options.get('metrics.enabled'):
logger.debug('application metrics enabled: registering gRPC interceptor')
interceptors = [MetricsInterceptor()]
# Prior to registering the plugin, we need to get the plugin metadata
# and ensure that we can connect to the plugin. These calls may raise
# an exception - we want to let them propagate up to signal that registration
# for the particular address failed.
try:
c = client.PluginClientV3(
address=address,
protocol=protocol,
timeout=config.options.get('grpc.timeout'),
tls=config.options.get('grpc.tls.cert'),
interceptors=interceptors,
)
except Exception as e:
logger.error(
'failed to create plugin client',
address=address,
protocol=protocol,
timeout=config.options.get('grpc.timeout'),
tls=config.options.get('grpc.tls.cert'),
interceptors=interceptors,
)
raise errors.ClientCreateError('error creating plugin client') from e
# Let any exceptions here raise up. The caller should handle appropriately.
# Generally any exceptions raised here should not propagate past the caller,
# as a failure to communicate may be intermittent and should be retried later.
meta = c.metadata()
ver = c.version()
plugin = Plugin(
info=utils.to_dict(meta),
version=utils.to_dict(ver),
client=c,
loop=loop.synse_loop,
)
logger.debug(
'loaded plugin info',
id=plugin.id, version=plugin.version, addr=plugin.address, tag=plugin.tag,
)
if plugin.id in self.plugins:
# A plugin with the given ID has already been registered. This could happen
# in a number of cases:
#
# - During routine refresh of plugins, discovery or some other mechanism will
# find plugins and attempt to re-register them, relying on this block to
# determine whether or not the plugin needs to be re-registered.
# - Plugins were mis-configured and are having ID collisions. There is nothing
# that can be done here other than logging the potential collision.
# - Plugins were rescheduled and could potentially be given a new IP address.
# In this case, the 'cached' plugin could exist, but have the wrong address.
#
# In addition to checking whether or not the plugin exists in the manager cache,
# we also need to check some general state of the plugin, including whether it is
# enabled/disabled, what its address is, etc.
cached = self.plugins[plugin.id]
if cached.disabled:
if cached.address != plugin.address:
logger.info(
'address changed for existing plugin',
id=plugin.id, old_addr=cached.address, new_addr=plugin.address,
)
# Regardless of whether the plugin address changed, it was previously disabled
# due to some error. Since we were able to connect to it, we will cancel any pending
# reconnect tasks and use the newly connect client instance.
cached.cancel_tasks()
# Update the exported metrics disabled plugins gauge: remove the old disabled plugin
Monitor.plugin_disabled.labels(plugin.id).dec()
self.plugins[plugin.id] = plugin
logger.debug('re-registered existing plugin', new=plugin, previous=cached)
else:
if cached.address != plugin.address:
# If we have matching plugin IDs, but differing addresses, and both plugins
# are considered "active", there is a chance that we are communicating with
# different plugins which have the same ID. This is indicative of a plugin ID
# collision due to misconfiguration.
logger.warning(
'potential plugin ID collision: plugins with same ID, different addresses '
'detected. this may also indicate plugin cycling.',
id=plugin.id, old_addr=cached.address, new_addr=plugin.address,
)
else:
self.plugins[plugin.id] = plugin
logger.info('successfully registered new plugin', id=plugin.id, tag=plugin.tag)
# Since we were able to communicate with the plugin, ensure it is put in the active state.
self.plugins[plugin.id].mark_active()
return plugin.id
def register(self, address: str, protocol: str) -> str:
"""Register a new Plugin with the manager.
With the provided address and communication protocol, the manager
will attempt to establish communication with the plugin to get its
metadata. If successful, it will generate a new Plugin instance for
the plugin and register it in the manager state.
Args:
address: The address of the plugin to register.
protocol: The protocol that the plugin uses. This must be
one of: 'unix', 'tcp'.
Returns:
The ID of the plugin that was registered.
"""
logger.info('registering new plugin', addr=address, protocol=protocol)
interceptors = []
if config.options.get('metrics.enabled'):
logger.debug(
'application metrics enabled: registering gRPC interceptor')
interceptors = [MetricsInterceptor()]
# Prior to registering the plugin, we need to get the plugin metadata
# and ensure that we can connect to the plugin. These calls may raise
# an exception - we want to let them propagate up to signal that registration
# for the particular address failed.
try:
c = client.PluginClientV3(
address=address,
protocol=protocol,
timeout=config.options.get('grpc.timeout'),
tls=config.options.get('grpc.tls.cert'),
interceptors=interceptors,
)
except Exception as e:
logger.error(
'failed to create plugin client',
address=address,
protocol=protocol,
timeout=config.options.get('grpc.timeout'),
tls=config.options.get('grpc.tls.cert'),
interceptors=interceptors,
)
raise errors.ClientCreateError(
'error creating plugin client') from e
# Let any exceptions here raise up. The caller should handle appropriately.
# Generally any exceptions raised here should not propagate past the caller,
# as a failure to communicate may be intermittent and should be retried later.
meta = c.metadata()
ver = c.version()
plugin = Plugin(
info=utils.to_dict(meta),
version=utils.to_dict(ver),
client=c,
loop=loop.synse_loop,
)
logger.debug(
'loaded plugin info',
id=plugin.id,
version=plugin.version,
addr=plugin.address,
tag=plugin.tag,
)
if plugin.id in self.plugins:
# The plugin has already been registered. There is nothing left to
# do here, so just log and move on.
logger.debug('plugin with id already registered - skipping',
id=plugin.id)
else:
self.plugins[plugin.id] = plugin
logger.info('successfully registered new plugin',
id=plugin.id,
tag=plugin.tag)
self.plugins[plugin.id].mark_active()
return plugin.id
