def _from_kwargs(
system: System = None,
instance: Instance = None,
system_id: str = None,
instance_name: str = None,
instance_id: str = None,
**_,
) -> Tuple[System, Instance]:
if system and instance:
return system, instance
if not system:
if system_id:
system = db.query_unique(System, raise_missing=True, id=system_id)
elif instance:
system = db.query_unique(System,
raise_missing=True,
instances__contains=instance)
elif instance_id:
system = db.query_unique(System,
raise_missing=True,
instances__id=instance_id)
else:
raise NotFoundException("Unable to find System")
if not instance:
if instance_name:
instance = system.get_instance_by_name(instance_name)
elif instance_id:
instance = system.get_instance_by_id(instance_id)
else:
raise NotFoundException("Unable to find Instance")
return system, instance
def _determine_target_garden(operation: Operation) -> str:
"""Determine the system the operation is targeting"""
# Certain operations are ASSUMED to be targeted at the local garden
if ("READ" in operation.operation_type or "JOB" in operation.operation_type
or "FILE" in operation.operation_type or operation.operation_type
in ("PLUGIN_LOG_RELOAD", "SYSTEM_CREATE", "SYSTEM_RESCAN")
or "PUBLISH_EVENT" in operation.operation_type
or "RUNNER" in operation.operation_type or operation.operation_type
in ("PLUGIN_LOG_RELOAD", "SYSTEM_CREATE")):
return config.get("garden.name")
# Otherwise, each operation needs to be "parsed"
if operation.operation_type in ("SYSTEM_RELOAD", "SYSTEM_UPDATE"):
return _system_id_lookup(operation.args[0])
if operation.operation_type == "SYSTEM_DELETE":
# Force deletes get routed to local garden
if operation.kwargs.get("force"):
return config.get("garden.name")
return _system_id_lookup(operation.args[0])
if "INSTANCE" in operation.operation_type:
if "system_id" in operation.kwargs and "instance_name" in operation.kwargs:
return _system_id_lookup(operation.kwargs["system_id"])
else:
return _instance_id_lookup(operation.args[0])
if operation.operation_type == "REQUEST_CREATE":
target_system = System(
namespace=operation.model.namespace,
name=operation.model.system,
version=operation.model.system_version,
)
return _system_name_lookup(target_system)
if operation.operation_type.startswith("REQUEST"):
request = db.query_unique(Request, id=operation.args[0])
operation.kwargs["request"] = request
return config.get("garden.name")
if "GARDEN" in operation.operation_type:
if operation.operation_type == "GARDEN_SYNC":
sync_target = operation.kwargs.get("sync_target")
if sync_target:
return sync_target
return config.get("garden.name")
if operation.operation_type == "QUEUE_DELETE":
# Need to deconstruct the queue name
parts = operation.args[0].split(".")
version = parts[2].replace("-", ".")
return _system_name_lookup(
System(namespace=parts[0], name=parts[1], version=version))
raise Exception(f"Bad operation type {operation.operation_type}")
def default_system(self, command1):
return System(
name="foo",
version="1.0.0",
instances=[Instance(name="foo")],
commands=[command1],
)
def test_system_exists(self, plugin, bm_client, bg_system, bg_instance,
current_commands):
bg_system.commands = [Command("test")]
bm_client.update_system.return_value = bg_system
existing_system = System(
id="id",
name="test_system",
version="0.0.1",
instances=[bg_instance],
commands=current_commands,
metadata={"foo": "bar"},
)
bm_client.find_unique_system.return_value = existing_system
plugin._initialize()
bm_client.initialize_instance.assert_called_once_with(bg_instance.id)
bm_client.update_system.assert_called_once_with(
existing_system.id,
new_commands=bg_system.commands,
metadata=bg_system.metadata,
description=bg_system.description,
icon_name=bg_system.icon_name,
display_name=bg_system.display_name,
)
assert bm_client.create_system.called is False
assert bm_client.create_system.return_value == plugin.system
assert bm_client.initialize_instance.return_value == plugin.instance
def heartbeat(
instance_id: str = None,
instance: Instance = None,
system: System = None,
**_,
) -> Instance:
"""Instance heartbeat
Args:
instance_id: The Instance ID
instance: The Instance
system: The System
Returns:
The updated Instance
"""
system, instance = _from_kwargs(system=system,
instance=instance,
instance_id=instance_id)
system = db.modify(
system,
query={"instances__name": instance.name},
set__instances__S__status_info__heartbeat=datetime.utcnow(),
)
return system.get_instance_by_name(instance.name)
def _setup_system(self, client, inst_name, system, name, description, version, icon_name,
metadata, display_name, max_instances):
if system:
if name or description or version or icon_name or display_name or max_instances:
raise ValidationError("Sorry, you can't specify a system as well as system "
"creation helper keywords (name, description, version, "
"max_instances, display_name, and icon_name)")
if not system.instances:
raise ValidationError("Explicit system definition requires explicit instance "
"definition (use instances=[Instance(name='default')] "
"for default behavior)")
if not system.max_instances:
system.max_instances = len(system.instances)
else:
name = name or os.environ.get('BG_NAME', None)
version = version or os.environ.get('BG_VERSION', None)
if client.__doc__ and not description:
description = self.client.__doc__.split("\n")[0]
system = System(name=name, description=description, version=version,
icon_name=icon_name, commands=client._commands,
max_instances=max_instances or 1,
instances=[Instance(name=inst_name)],
metadata=metadata, display_name=display_name)
return system
def bg_system_2(system_dict, bg_instance, bg_command, bg_command_2):
"""A system with a different version."""
dict_copy = copy.deepcopy(system_dict)
dict_copy["version"] = "2.0.0"
dict_copy["instances"] = [bg_instance]
dict_copy["commands"] = [bg_command, bg_command_2]
return System(**dict_copy)
def test_new_instance(self, plugin, ez_client, bg_system, bg_instance):
existing_system = System(
id="id",
name="test_system",
version="0.0.1",
instances=[bg_instance],
max_instances=2,
metadata={"foo": "bar"},
template=None,
)
ez_client.find_unique_system.return_value = existing_system
new_name = "foo_instance"
plugin._config.instance_name = new_name
plugin._initialize_system()
assert ez_client.create_system.called is False
ez_client.update_system.assert_called_once_with(
existing_system.id,
new_commands=bg_system.commands,
metadata=bg_system.metadata,
description=bg_system.description,
icon_name=bg_system.icon_name,
display_name=bg_system.display_name,
template="<html>template</html>",
add_instance=ANY,
)
assert ez_client.update_system.call_args[1][
"add_instance"].name == new_name
def test_system_exists(self, plugin, ez_client, bg_system, bg_instance,
current_commands):
existing_system = System(
id="id",
name="test_system",
version="0.0.1",
instances=[bg_instance],
commands=current_commands,
metadata={"foo": "bar"},
template=None,
)
ez_client.find_unique_system.return_value = existing_system
bg_system.commands = [Command("test")]
ez_client.update_system.return_value = bg_system
plugin._initialize_system()
assert ez_client.create_system.called is False
ez_client.update_system.assert_called_once_with(
existing_system.id,
new_commands=bg_system.commands,
metadata=bg_system.metadata,
description=bg_system.description,
icon_name=bg_system.icon_name,
display_name=bg_system.display_name,
template="<html>template</html>",
)
def test_remove_system(self, find_mock, remove_mock):
find_mock.return_value = System(id='id')
remove_mock.return_value = 'delete_response'
self.assertEqual('delete_response',
self.client.remove_system(search='search params'))
find_mock.assert_called_once_with(search='search params')
remove_mock.assert_called_once_with('id')
def test_remove_system(self, find_mock, remove_mock):
find_mock.return_value = System(id="id")
remove_mock.return_value = "delete_response"
self.assertEqual("delete_response",
self.client.remove_system(search="search params"))
find_mock.assert_called_once_with(search="search params")
remove_mock.assert_called_once_with("id")
def test_max_instances(self, plugin):
system = System(
name="name",
version="1.0.0",
instances=[Instance(name="1"),
Instance(name="2")],
)
new_system = plugin._setup_system(system, {})
assert new_system.max_instances == 2
def test_max_instances(self, plugin, client):
system = System(
name="name",
version="1.0.0",
instances=[Instance(name="1"),
Instance(name="2")],
)
new_system = plugin._setup_system(client, "default", system, "", "",
"", "", {}, None, None)
assert new_system.max_instances == 2
def test_initialize_system_new_instance(self):
self.plugin.instance_name = 'new_instance'
existing_system = System(id='id', name='test_system', version='1.0.0',
instances=[self.instance], max_instances=2,
metadata={'foo': 'bar'})
self.bm_client_mock.find_unique_system.return_value = existing_system
self.plugin._initialize()
self.assertTrue(self.bm_client_mock.create_system.called)
self.assertTrue(self.bm_client_mock.update_system.called)
def update(
instance_id: str = None,
instance: Instance = None,
system: System = None,
new_status: str = None,
metadata: dict = None,
update_heartbeat: bool = True,
**_,
) -> Instance:
"""Update an Instance status.
Will also update the status_info heartbeat.
Args:
instance_id: The Instance ID
instance: The Instance
system: The System
new_status: The new status
metadata: New metadata
update_heartbeat: Set the heartbeat to the current time
Returns:
The updated Instance
"""
system, instance = _from_kwargs(system=system,
instance=instance,
instance_id=instance_id)
logger.debug(f"Updating instance {system}[{instance}]")
updates = {}
if new_status:
updates["set__instances__S__status"] = new_status
if new_status == "STOPPED":
lpm.update(instance_id=instance_id, restart=False, stopped=True)
if update_heartbeat:
updates["set__instances__S__status_info__heartbeat"] = datetime.utcnow(
)
if metadata:
metadata_update = dict(instance.metadata)
metadata_update.update(metadata)
updates["set__instances__S__metadata"] = metadata_update
system = db.modify(system,
query={"instances__name": instance.name},
**updates)
return system.get_instance_by_name(instance.name)
def _setup_system(self, system, plugin_kwargs):
helper_keywords = {
"name",
"version",
"description",
"icon_name",
"display_name",
"max_instances",
"metadata",
"namespace",
"template",
}
if system:
if helper_keywords.intersection(plugin_kwargs.keys()):
raise ValidationError(
"Sorry, you can't provide a complete system definition as well as "
"system creation helper kwargs %s" % helper_keywords
)
if not system.instances:
raise ValidationError(
"Explicit system definition requires explicit instance "
"definition (use instances=[Instance(name='default')] for "
"default behavior)"
)
if not system.max_instances:
system.max_instances = len(system.instances)
else:
# Commands are not defined here - they're set in the client property setter
system = System(
name=self._config.name,
version=self._config.version,
description=self._config.description,
namespace=self._config.namespace,
metadata=json.loads(self._config.metadata),
instances=[Instance(name=self._config.instance_name)],
max_instances=self._config.max_instances,
icon_name=self._config.icon_name,
display_name=self._config.display_name,
template=self._config.template,
)
return system
def setUp(self):
self.app = brew_view.app.test_client()
self.default_instance = Instance(name='default', status='RUNNING')
self.default_command = Command(id='54ac18f778c4b57e963f3c18',
name='command',
description='foo')
self.default_system = System(id='54ac18f778c4b57e963f3c18',
name='default_system',
version='1.0.0',
instances=[self.default_instance],
commands=[self.default_command])
self.client_mock = Mock(name='client_mock')
self.fake_context = MagicMock(
__enter__=Mock(return_value=self.client_mock),
__exit__=Mock(return_value=False))
def test_new_instance(self, plugin, bm_client, bg_system, bg_instance):
plugin.instance_name = "new_instance"
existing_system = System(
id="id",
name="test_system",
version="0.0.1",
instances=[bg_instance],
max_instances=2,
metadata={"foo": "bar"},
)
bm_client.find_unique_system.return_value = existing_system
plugin._initialize()
assert 2 == len(existing_system.instances)
assert bm_client.create_system.called is True
assert bm_client.update_system.called is True
def test_existing_multiple(self, tmp_path, loader, registry, plugin_1, bg_instance):
"""This is mainly to test that Instance IDs are correct
We save a system with 2 instances:
- instance1, 58542eb571afd47ead90beef
- instance2, 58542eb571afd47ead90beee
Then we load a plugin that defines instances [instance2, instance3].
Correct behavior is:
- instance1 removed from the database
- instance3 created in the database
- instance2 remains in the database, and the ID remains the same
"""
instance1 = Instance(name="instance1", id="58542eb571afd47ead90beef")
instance2 = Instance(name="instance2", id="58542eb571afd47ead90beee")
create_system(
System(name="foo", version="1.0", instances=[instance1, instance2])
)
plugin = tmp_path / "plugin"
plugin.mkdir()
write_file(
plugin,
textwrap.dedent(
"""
NAME='foo'
VERSION='1.0'
PLUGIN_ENTRY='entry.py'
INSTANCES=["instance2", "instance3"]
"""
),
)
plugin_runners = loader.load_plugin(plugin)
assert len(plugin_runners) == 2
assert db.query_unique(Instance, name="instance1") is None
assert db.query_unique(Instance, name="instance3") is not None
instance2_db = db.query_unique(Instance, name="instance2")
assert instance2_db is not None
assert instance2_db.id == instance2.id
def test_existing(self, loader, registry, plugin_1):
system_id = "58542eb571afd47ead90face"
instance_id = "58542eb571afd47ead90beef"
create_system(
System(
id=system_id,
name="foo",
version="1.0",
instances=[Instance(id=instance_id)],
)
)
plugin_runners = loader.load_plugin(plugin_1)
assert len(plugin_runners) == 1
assert str(plugin_runners[0].system.id) == system_id
assert str(plugin_runners[0].instance.id) == instance_id
assert plugin_runners[0].name == "foo[default]-1.0"
assert plugin_runners[0].entry_point == "entry.py"
def test_initialize_system_update_metadata(self):
self.system.commands = [Command('test')]
self.bm_client_mock.update_system.return_value = self.system
existing_system = System(id='id', name='test_system', version='1.0.0',
instances=[self.instance],
metadata={})
self.bm_client_mock.find_unique_system.return_value = existing_system
self.plugin._initialize()
self.assertFalse(self.bm_client_mock.create_system.called)
self.bm_client_mock.update_system.assert_called_once_with(self.instance.id,
new_commands=self.system.commands,
description=None,
display_name=None,
icon_name=None,
metadata={"foo": "bar"})
self.bm_client_mock.initialize_instance.assert_called_once_with(self.instance.id)
self.assertEqual(self.plugin.system, self.bm_client_mock.create_system.return_value)
self.assertEqual(self.plugin.instance, self.bm_client_mock.initialize_instance.return_value)
def setUp(self):
self.app = brew_view.app.test_client()
self.default_instance = Instance(name="default", status="RUNNING")
self.default_command = Command(
id="54ac18f778c4b57e963f3c18", name="command", description="foo"
)
self.default_system = System(
id="54ac18f778c4b57e963f3c18",
name="default_system",
version="1.0.0",
instances=[self.default_instance],
commands=[self.default_command],
)
self.client_mock = Mock(name="client_mock")
self.fake_context = MagicMock(
__enter__=Mock(return_value=self.client_mock),
__exit__=Mock(return_value=False),
)
def initialize(
instance_id: str = None,
instance: Instance = None,
system: System = None,
runner_id: str = None,
**_,
) -> Instance:
"""Initializes an instance.
Args:
instance_id: The Instance ID
instance: The Instance
system: The System
runner_id: The runner id to associate with this plugin, if any
Returns:
The updated Instance
"""
system, instance = _from_kwargs(system=system,
instance=instance,
instance_id=instance_id)
logger.debug(f"Initializing instance {system}[{instance}]")
queue_spec = queue.create(instance, system)
system = db.modify(
system,
query={"instances__name": instance.name},
**{
"set__instances__S__status": "INITIALIZING",
"set__instances__S__status_info__heartbeat": datetime.utcnow(),
"set__instances__S__metadata__runner_id": runner_id,
"set__instances__S__queue_type": queue_spec["queue_type"],
"set__instances__S__queue_info": queue_spec["queue_info"],
},
)
start(instance=instance, system=system)
return system.get_instance_by_name(instance.name)
def create_system(system: System) -> System:
"""Create a new System
Args:
system: The System to create
Returns:
The created System
"""
if system.namespace is None:
system.namespace = config.get("garden.name")
# Create in the database
system = db.create(system)
# Also need to let the routing module know
from beer_garden.router import add_routing_system
add_routing_system(system=system)
return system
def setUp(self):
self.safe_copy = os.environ.copy()
consumer_patcher = patch('brewtils.plugin.RequestConsumer')
self.addCleanup(consumer_patcher.stop)
self.consumer_patch = consumer_patcher.start()
self.instance = Instance(id='id', name='default', queue_type='rabbitmq',
queue_info={'url': 'url', 'admin': {'name': 'admin_queue'},
'request': {'name': 'request_queue'}})
self.system = System(name='test_system', version='1.0.0', instances=[self.instance],
metadata={'foo': 'bar'})
self.client = MagicMock(name='client', spec='command', _commands=['command'])
self.plugin = PluginBase(self.client, bg_host='localhost', system=self.system,
metadata={'foo': 'bar'})
self.plugin.instance = self.instance
self.bm_client_mock = Mock(create_system=Mock(return_value=self.system),
initialize_instance=Mock(return_value=self.instance))
self.plugin.bm_client = self.bm_client_mock
self.parser_mock = Mock()
self.plugin.parser = self.parser_mock
def test_no_instances(self, plugin, client):
system = System(name="name", version="1.0.0")
with pytest.raises(ValidationError,
match="explicit instance definition"):
plugin._setup_system(client, "default", system, "", "", "", "", {},
None, None)
def test_setup_system_no_max_instances(self):
system = System(name='test_system', version='1.0.0', instances=[Instance(name='1'),
Instance(name='2')])
new_system = self.plugin._setup_system(self.client, 'default', system, '', '', '', '', {},
None, None)
self.assertEqual(2, new_system.max_instances)
def test_setup_system_no_instances(self):
system = System(name='test_system', version='1.0.0')
self.assertRaises(ValidationError, self.plugin._setup_system, self.client,
'default', system,
'', '', '', '', {}, None, None)
def bg_system(system_dict, bg_instance, bg_command):
"""A system as a model."""
dict_copy = copy.deepcopy(system_dict)
dict_copy['instances'] = [bg_instance]
dict_copy['commands'] = [bg_command]
return System(**dict_copy)
def bg_command(command_dict, bg_parameter):
"""Use the bg_command fixture instead."""
dict_copy = copy.deepcopy(command_dict)
dict_copy['parameters'] = [bg_parameter]
dict_copy['system'] = System(id=system_id)
return Command(**dict_copy)
