def fake_pod_discovery(self):
discovered_pod = DiscoveredPod(
architectures=['amd64/generic'],
cores=random.randint(2, 4), memory=random.randint(1024, 4096),
local_storage=random.randint(1024, 1024 * 1024),
cpu_speed=random.randint(2048, 4048),
hints=DiscoveredPodHints(
cores=random.randint(2, 4), memory=random.randint(1024, 4096),
local_storage=random.randint(1024, 1024 * 1024),
cpu_speed=random.randint(2048, 4048)),
storage_pools=[
DiscoveredPodStoragePool(
id=factory.make_name('pool_id'),
name=factory.make_name('name'),
type=factory.make_name('type'),
path='/var/lib/path/%s' % factory.make_name('path'),
storage=random.randint(1024, 1024 * 1024),
)
for _ in range(3)
])
discovered_rack_1 = factory.make_RackController()
discovered_rack_2 = factory.make_RackController()
failed_rack = factory.make_RackController()
self.patch(pods_module, "discover_pod").return_value = ({
discovered_rack_1.system_id: discovered_pod,
discovered_rack_2.system_id: discovered_pod,
}, {
failed_rack.system_id: factory.make_exception(),
})
return (
discovered_pod,
[discovered_rack_1, discovered_rack_2],
[failed_rack])
def test__calls_DiscoverPod_on_all_clients(self):
rack_ids = [factory.make_name("system_id") for _ in range(3)]
pod = DiscoveredPod(
architectures=["amd64/generic"],
cores=random.randint(1, 8),
cpu_speed=random.randint(1000, 3000),
memory=random.randint(1024, 4096),
local_storage=random.randint(500, 1000),
hints=DiscoveredPodHints(
cores=random.randint(1, 8),
cpu_speed=random.randint(1000, 3000),
memory=random.randint(1024, 4096),
local_storage=random.randint(500, 1000),
),
)
clients = []
for rack_id in rack_ids:
client = Mock()
client.ident = rack_id
client.return_value = succeed({"pod": pod})
clients.append(client)
self.patch(pods_module, "getAllClients").return_value = clients
discovered = yield discover_pod(factory.make_name("pod"), {})
self.assertEquals(({rack_id: pod
for rack_id in rack_ids}, {}), discovered)
def test__calls_and_returns_correctly(self):
hints = DiscoveredPodHints(
cores=random.randint(1, 8),
cpu_speed=random.randint(1000, 2000),
memory=random.randint(1024, 8192),
local_storage=0,
)
pod = factory.make_Pod()
client = Mock()
client.return_value = succeed({"hints": hints})
result = wait_for_reactor(decompose_machine)(client, pod.power_type,
pod.power_parameters,
pod.id, pod.name)
self.assertThat(
client,
MockCalledOnceWith(
DecomposeMachine,
type=pod.power_type,
context=pod.power_parameters,
pod_id=pod.id,
name=pod.name,
),
)
self.assertEqual(hints, result)
def fake_pod_discovery(self):
discovered_pod = DiscoveredPod(
architectures=["amd64/generic"],
cores=random.randint(2, 4),
memory=random.randint(1024, 4096),
local_storage=random.randint(1024, 1024 * 1024),
cpu_speed=random.randint(2048, 4048),
hints=DiscoveredPodHints(
cores=random.randint(2, 4),
memory=random.randint(1024, 4096),
local_storage=random.randint(1024, 1024 * 1024),
cpu_speed=random.randint(2048, 4048),
),
)
discovered_rack_1 = factory.make_RackController()
discovered_rack_2 = factory.make_RackController()
failed_rack = factory.make_RackController()
self.patch(pods, "discover_pod").return_value = succeed((
{
discovered_rack_1.system_id: discovered_pod,
discovered_rack_2.system_id: discovered_pod,
},
{
failed_rack.system_id: factory.make_exception()
},
))
def test_returns_discovered_pod_and_errors(self):
pod_type = factory.make_name("pod")
pod = DiscoveredPod(
architectures=["amd64/generic"],
cores=random.randint(1, 8),
cpu_speed=random.randint(1000, 3000),
memory=random.randint(1024, 4096),
local_storage=random.randint(500, 1000),
hints=DiscoveredPodHints(
cores=random.randint(1, 8),
cpu_speed=random.randint(1000, 3000),
memory=random.randint(1024, 4096),
local_storage=random.randint(500, 1000),
),
)
clients = []
client = Mock()
error_rack_id = factory.make_name("system_id")
client.ident = error_rack_id
exception = UnknownPodType(pod_type)
client.return_value = fail(exception)
clients.append(client)
valid_rack_id = factory.make_name("system_id")
client = Mock()
client.ident = valid_rack_id
client.return_value = succeed({"pod": pod})
clients.append(client)
self.patch(pods_module, "getAllClients").return_value = clients
discovered = yield discover_pod(pod_type, {})
self.assertEqual(
({valid_rack_id: pod}, {error_rack_id: exception}), discovered
)
def compose(self, pod_id: int, context: dict, request: RequestedMachine):
"""Compose a virtual machine."""
with self._get_client(pod_id, context) as client:
storage_pools = client.storage_pools.all()
default_storage_pool = context.get(
"default_storage_pool_id", context.get("default_storage_pool"))
include_profile = client.profiles.exists(LXD_MAAS_PROFILE)
definition = get_lxd_machine_definition(
request, include_profile=include_profile)
definition["devices"] = {
**self._get_machine_disks(request.block_devices, storage_pools, default_storage_pool),
**self._get_machine_nics(request),
}
# Create the machine.
machine = client.virtual_machines.create(definition, wait=True)
# Pod hints are updated on the region after the machine is composed.
discovered_machine = self._get_discovered_machine(client,
machine,
storage_pools,
request=request)
# Update the machine cpu speed.
discovered_machine.cpu_speed = lxd_cpu_speed(client.resources)
return discovered_machine, DiscoveredPodHints()
def test_handles_driver_resolving_to_tuple_of_discovered(self):
fake_driver = MagicMock()
fake_driver.name = factory.make_name("pod")
fake_request = self.make_requested_machine()
pod_id = random.randint(1, 10)
pod_name = factory.make_name("pod")
machine = DiscoveredMachine(
hostname=factory.make_name("hostname"),
architecture="amd64/generic",
cores=random.randint(1, 8),
cpu_speed=random.randint(1000, 3000),
memory=random.randint(1024, 8192),
block_devices=[],
interfaces=[],
)
hints = DiscoveredPodHints(
cores=random.randint(1, 8),
cpu_speed=random.randint(1000, 2000),
memory=random.randint(1024, 8192),
local_storage=0,
)
fake_driver.compose.return_value = succeed((machine, hints))
self.patch(PodDriverRegistry, "get_item").return_value = fake_driver
result = yield pods.compose_machine(
fake_driver.name, {}, fake_request, pod_id=pod_id, name=pod_name
)
self.assertEqual({"machine": machine, "hints": hints}, result)
def get_pod_resources(self):
"""Get the pod resources."""
discovered_pod = DiscoveredPod(architectures=[],
cores=0,
cpu_speed=0,
memory=0,
local_storage=0,
hints=DiscoveredPodHints(
cores=0,
cpu_speed=0,
memory=0,
local_storage=0))
discovered_pod.architectures = [self.get_pod_arch()]
discovered_pod.capabilities = [
Capabilities.COMPOSABLE,
Capabilities.DYNAMIC_LOCAL_STORAGE,
Capabilities.OVER_COMMIT,
Capabilities.STORAGE_POOLS,
]
discovered_pod.cores = self.get_pod_cpu_count()
discovered_pod.cpu_speed = self.get_pod_cpu_speed()
discovered_pod.memory = self.get_pod_memory()
discovered_pod.storage_pools = self.get_pod_storage_pools()
discovered_pod.local_storage = sum(
pool.storage for pool in discovered_pod.storage_pools)
return discovered_pod
def decompose(self, pod_id: int, context: dict):
"""Decompose a virtual machine."""
machine = self._get_machine(pod_id, context)
if not machine:
maaslog.warning(
f"Pod {pod_id}: machine {context['instance_name']} not found")
return DiscoveredPodHints()
if machine.status_code != 102: # 102 - Stopped
machine.stop(force=True, wait=True)
# collect machine attributes before removing it
devices = machine.devices
client = machine.client
machine.delete(wait=True)
self._delete_machine_volumes(client, pod_id, devices)
# Hints are updated on the region for LXDPodDriver.
return DiscoveredPodHints()
def make_compose_machine_result(self, pod):
composed_machine = DiscoveredMachine(
hostname=factory.make_name('hostname'),
architecture=pod.architectures[0],
cores=1, memory=1024, cpu_speed=300,
block_devices=[], interfaces=[])
pod_hints = DiscoveredPodHints(
cores=random.randint(0, 10), memory=random.randint(1024, 4096),
cpu_speed=random.randint(1000, 3000), local_storage=0)
return composed_machine, pod_hints
def decompose(self, pod_id, context):
"""Decompose a virtual machine."""
client = yield self.get_client(pod_id, context)
machine = yield deferToThread(client.virtual_machines.get,
context["instance_name"])
# Stop the machine.
yield deferToThread(machine.stop)
yield deferToThread(machine.delete, wait=True)
# Hints are updated on the region for LXDPodDriver.
return DiscoveredPodHints()
def decompose(self, pod_id, context):
"""Decompose a virtual machine."""
client = yield self.get_client(pod_id, context)
def sync_decompose(instance_name):
machine = client.virtual_machines.get(instance_name)
machine.stop(force=True, wait=True)
machine.delete(wait=True)
yield deferToThread(sync_decompose, context["instance_name"])
# Hints are updated on the region for LXDPodDriver.
return DiscoveredPodHints()
class TestDiscoveredPodHints(MAASTestCase):
example = DiscoveredPodHints(
cores=random.randint(1, 8),
cpu_speed=random.randint(1000, 2000),
memory=random.randint(1024, 8192), local_storage=0)
def test_round_trip(self):
argument = arguments.AmpDiscoveredPodHints()
encoded = argument.toString(self.example)
self.assertThat(encoded, IsInstance(bytes))
decoded = argument.fromString(encoded)
self.assertThat(decoded, Equals(self.example))
def test_pod_hints(self):
cores = random.randint(1, 8)
cpu_speed = random.randint(1000, 2000)
memory = random.randint(4096, 8192)
local_storage = random.randint(4096, 8192)
hints = DiscoveredPodHints(
cores=cores,
cpu_speed=cpu_speed,
memory=memory,
local_storage=local_storage,
)
self.assertEqual(cores, hints.cores)
self.assertEqual(cpu_speed, hints.cpu_speed)
self.assertEqual(memory, hints.memory)
self.assertEqual(local_storage, hints.local_storage)
def test_works_when_driver_returns_hints(self):
hints = DiscoveredPodHints(
cores=random.randint(1, 8),
cpu_speed=random.randint(1000, 2000),
memory=random.randint(1024, 8192),
local_storage=0,
)
fake_driver = MagicMock()
fake_driver.name = factory.make_name("pod")
fake_driver.decompose.return_value = succeed(hints)
pod_id = random.randint(1, 10)
pod_name = factory.make_name("pod")
self.patch(PodDriverRegistry, "get_item").return_value = fake_driver
result = yield pods.decompose_machine(
fake_driver.name, {}, pod_id=pod_id, name=pod_name
)
self.assertEqual({"hints": hints}, result)
def make_compose_machine_result(pod):
composed_machine = DiscoveredMachine(
hostname=factory.make_name("hostname"),
architecture=pod.architectures[0],
cores=1,
memory=1024,
cpu_speed=300,
power_parameters={"instance_name": factory.make_name("instance")},
block_devices=[],
interfaces=[],
)
pod_hints = DiscoveredPodHints(
cores=random.randint(0, 10),
memory=random.randint(1024, 4096),
cpu_speed=random.randint(1000, 3000),
local_storage=0,
)
return composed_machine, pod_hints
def get_pod_hints(self):
"""Gets the discovered pod hints."""
discovered_pod_hints = DiscoveredPodHints(
cores=0, cpu_speed=0, memory=0, local_storage=0)
# You can always create a domain up to the size of total cores,
# memory, and cpu_speed even if that amount is already in use.
# Not a good idea, but possible.
discovered_pod_hints.cores = self.get_pod_cpu_count()
discovered_pod_hints.cpu_speed = self.get_pod_cpu_speed()
discovered_pod_hints.memory = self.get_pod_memory()
discovered_pod_hints.local_storage = (
self.get_pod_available_local_storage())
return discovered_pod_hints
def test_handles_driver_resolving_to_DiscoveredPod(self):
fake_driver = MagicMock()
fake_driver.name = factory.make_name("pod")
discovered_pod = DiscoveredPod(
architectures=['amd64/generic'],
cores=random.randint(1, 8),
cpu_speed=random.randint(1000, 3000),
memory=random.randint(1024, 8192),
local_storage=0,
hints=DiscoveredPodHints(cores=random.randint(1, 8),
cpu_speed=random.randint(1000, 2000),
memory=random.randint(1024, 8192),
local_storage=0),
machines=[])
fake_driver.discover.return_value = succeed(discovered_pod)
self.patch(PodDriverRegistry, "get_item").return_value = fake_driver
result = yield pods.discover_pod(fake_driver.name, {})
self.assertEquals({
"pod": discovered_pod,
}, result)
def test_pod_asdict(self):
hostname = factory.make_name("hostname")
cores = random.randint(1, 8)
cpu_speed = random.randint(1000, 2000)
memory = random.randint(4096, 8192)
local_storage = random.randint(4096, 8192)
local_disks = random.randint(1, 8)
iscsi_storage = random.randint(4096, 8192)
hints = DiscoveredPodHints(
cores=random.randint(1, 8),
cpu_speed=random.randint(1000, 2000),
memory=random.randint(4096, 8192),
local_storage=random.randint(4096, 8192),
local_disks=random.randint(1, 8),
iscsi_storage=random.randint(4096, 8192),
)
machines = []
tags = [factory.make_name("tag") for _ in range(3)]
storage_pools = [
DiscoveredPodStoragePool(
id=factory.make_name("id"),
name=factory.make_name("name"),
storage=random.randint(4096, 8192),
type="dir",
path="/var/%s" % factory.make_name("dir"),
) for _ in range(3)
]
for _ in range(3):
cores = random.randint(1, 8)
cpu_speed = random.randint(1000, 2000)
memory = random.randint(4096, 8192)
power_state = factory.make_name("unknown")
power_parameters = {"power_id": factory.make_name("power_id")}
interfaces = [
DiscoveredMachineInterface(
mac_address=factory.make_mac_address()) for _ in range(3)
]
block_devices = [
DiscoveredMachineBlockDevice(
model=factory.make_name("model"),
serial=factory.make_name("serial"),
size=random.randint(512, 1024),
id_path=factory.make_name("/dev/vda"),
) for _ in range(3)
]
for _ in range(3):
block_devices.append(
DiscoveredMachineBlockDevice(
model=None,
serial=None,
size=random.randint(512, 1024),
type=BlockDeviceType.ISCSI,
iscsi_target=self.make_iscsi_target(),
storage_pool=factory.make_name("pool"),
))
tags = [factory.make_name("tag") for _ in range(3)]
machines.append(
DiscoveredMachine(
hostname=hostname,
architecture="amd64/generic",
cores=cores,
cpu_speed=cpu_speed,
memory=memory,
power_state=power_state,
power_parameters=power_parameters,
interfaces=interfaces,
block_devices=block_devices,
tags=tags,
))
pod = DiscoveredPod(
architectures=["amd64/generic"],
cores=cores,
cpu_speed=cpu_speed,
memory=memory,
local_storage=local_storage,
local_disks=local_disks,
iscsi_storage=iscsi_storage,
hints=hints,
machines=machines,
tags=tags,
storage_pools=storage_pools,
)
self.assertThat(
pod.asdict(),
MatchesDict({
"architectures":
Equals(["amd64/generic"]),
"name":
Equals(None),
"cores":
Equals(cores),
"cpu_speed":
Equals(cpu_speed),
"memory":
Equals(memory),
"local_storage":
Equals(local_storage),
"local_disks":
Equals(local_disks),
"iscsi_storage":
Equals(iscsi_storage),
"mac_addresses":
Equals([]),
"capabilities":
Equals(pod.capabilities),
"hints":
MatchesDict({
"cores": Equals(hints.cores),
"cpu_speed": Equals(hints.cpu_speed),
"memory": Equals(hints.memory),
"local_storage": Equals(hints.local_storage),
"local_disks": Equals(hints.local_disks),
"iscsi_storage": Equals(hints.iscsi_storage),
}),
"machines":
MatchesListwise([
MatchesDict({
"hostname":
Equals(machine.hostname),
"architecture":
Equals("amd64/generic"),
"cores":
Equals(machine.cores),
"cpu_speed":
Equals(machine.cpu_speed),
"memory":
Equals(machine.memory),
"power_state":
Equals(machine.power_state),
"power_parameters":
Equals(machine.power_parameters),
"interfaces":
MatchesListwise([
MatchesDict({
"mac_address":
Equals(interface.mac_address),
"vid":
Equals(interface.vid),
"tags":
Equals(interface.tags),
"boot":
Equals(False),
"attach_type":
Is(None),
"attach_name":
Is(None),
}) for interface in machine.interfaces
]),
"block_devices":
MatchesListwise([
MatchesDict({
"model":
Equals(block_device.model),
"serial":
Equals(block_device.serial),
"size":
Equals(block_device.size),
"block_size":
Equals(block_device.block_size),
"tags":
Equals(block_device.tags),
"id_path":
Equals(block_device.id_path),
"type":
Equals(block_device.type),
"iscsi_target":
Equals(block_device.iscsi_target),
"storage_pool":
Equals(block_device.storage_pool),
}) for block_device in machine.block_devices
]),
"tags":
Equals(machine.tags),
}) for machine in machines
]),
"tags":
Equals(tags),
"storage_pools":
MatchesListwise([
MatchesDict({
"id": Equals(pool.id),
"name": Equals(pool.name),
"storage": Equals(pool.storage),
"type": Equals(pool.type),
"path": Equals(pool.path),
}) for pool in storage_pools
]),
}),
)
def test_pod(self):
hostname = factory.make_name("hostname")
cores = random.randint(1, 8)
cpu_speed = random.randint(1000, 2000)
memory = random.randint(4096, 8192)
local_storage = random.randint(4096, 8192)
iscsi_storage = random.randint(4096, 8192)
hints = DiscoveredPodHints(
cores=random.randint(1, 8),
cpu_speed=random.randint(1000, 2000),
memory=random.randint(4096, 8192),
local_storage=random.randint(4096, 8192),
iscsi_storage=random.randint(4096, 8192),
)
machines = []
for _ in range(3):
cores = random.randint(1, 8)
cpu_speed = random.randint(1000, 2000)
memory = random.randint(4096, 8192)
power_state = factory.make_name("unknown")
power_parameters = {"power_id": factory.make_name("power_id")}
interfaces = [
DiscoveredMachineInterface(
mac_address=factory.make_mac_address()) for _ in range(3)
]
block_devices = [
DiscoveredMachineBlockDevice(
model=factory.make_name("model"),
serial=factory.make_name("serial"),
size=random.randint(512, 1024),
) for _ in range(3)
]
for _ in range(3):
block_devices.append(
DiscoveredMachineBlockDevice(
model=None,
serial=None,
size=random.randint(512, 1024),
type=BlockDeviceType.ISCSI,
iscsi_target=self.make_iscsi_target(),
))
tags = [factory.make_name("tag") for _ in range(3)]
machines.append(
DiscoveredMachine(
hostname=hostname,
architecture="amd64/generic",
cores=cores,
cpu_speed=cpu_speed,
memory=memory,
power_state=power_state,
power_parameters=power_parameters,
interfaces=interfaces,
block_devices=block_devices,
tags=tags,
))
pod = DiscoveredPod(
architectures=["amd64/generic"],
cores=cores,
cpu_speed=cpu_speed,
memory=memory,
local_storage=local_storage,
iscsi_storage=iscsi_storage,
hints=hints,
machines=machines,
)
self.assertEquals(cores, pod.cores)
self.assertEquals(cpu_speed, pod.cpu_speed)
self.assertEquals(memory, pod.memory)
self.assertEquals(local_storage, pod.local_storage)
self.assertEquals(iscsi_storage, pod.iscsi_storage)
self.assertEquals(machines, pod.machines)
def compose(self, pod_id: str, context: dict, request: RequestedMachine):
"""Compose a virtual machine."""
client = yield self.get_client(pod_id, context)
# Check to see if there is a maas profile. If not, use the default.
try:
profile = yield deferToThread(client.profiles.get, "maas")
except NotFound:
# Fall back to default
try:
profile = yield deferToThread(client.profiles.get, "default")
except NotFound:
raise LXDPodError(
f"Pod {pod_id}: MAAS needs LXD to have either a 'maas' "
"profile or a 'default' profile, defined.")
resources = yield deferToThread(lambda: client.resources)
definition = get_lxd_machine_definition(request, profile.name)
# Add disk to the definition.
# XXX: LXD VMs currently only support one virtual block device.
# Loop will need to be modified once LXD has multiple virtual
# block device support.
devices = {}
storage_pools = yield deferToThread(client.storage_pools.all)
default_storage_pool = context.get("default_storage_pool_id",
context.get("default_storage_pool"))
for idx, disk in enumerate(request.block_devices):
usable_pool = self.get_usable_storage_pool(disk, storage_pools,
default_storage_pool)
devices["root"] = {
"path": "/",
"type": "disk",
"pool": usable_pool,
"size": str(disk.size),
"boot.priority": "0",
}
# Create and attach interfaces to the machine.
# The reason we are doing this after the machine is created
# is because pylxd doesn't have a way to override the devices
# that are defined in the profile. Since the profile is provided
# by the user, we have no idea how many interfaces are defined.
#
# Currently, only the bridged type is supported with virtual machines.
# https://lxd.readthedocs.io/en/latest/instances/#device-types
nic_devices = {}
profile_devices = profile.devices
device_names = []
boot = True
for interface in request.interfaces:
if interface.ifname is None:
# No interface constraints sent so use the best
# nic device from the profile's devices.
device_name, device = self.get_best_nic_device_from_profile(
profile_devices)
nic_devices[device_name] = device
if "boot.priority" not in device and boot:
nic_devices[device_name]["boot.priority"] = "1"
boot = False
device_names.append(device_name)
else:
nic_devices[interface.ifname] = get_lxd_nic_device(interface)
# Set to boot from the first nic
if boot:
nic_devices[interface.ifname]["boot.priority"] = "1"
boot = False
device_names.append(interface.ifname)
# Iterate over all of the profile's devices with type=nic
# and set to type=none if not nic_device. This overrides
# the device settings on the profile used by the machine.
for dk, dv in profile_devices.items():
if dk not in device_names and dv["type"] == "nic":
nic_devices[dk] = {"type": "none"}
# Merge the devices and attach the devices to the defintion.
for k, v in nic_devices.items():
devices[k] = v
definition["devices"] = devices
# Create the machine.
machine = yield deferToThread(client.virtual_machines.create,
definition,
wait=True)
# Pod hints are updated on the region after the machine
# is composed.
discovered_machine = yield ensureDeferred(
self.get_discovered_machine(client,
machine,
storage_pools,
request=request))
# Update the machine cpu speed.
discovered_machine.cpu_speed = lxd_cpu_speed(resources)
return discovered_machine, DiscoveredPodHints()
def fromString(self, inString):
# Circular imports.
from provisioningserver.drivers.pod import DiscoveredPodHints
data = super(AmpDiscoveredPodHints, self).fromString(inString)
return DiscoveredPodHints.fromdict(data)
