class CobblerServer:
def __init__(self, fqdn, domain):
self._fqdn = fqdn
self._conn = None
self._domain = domain
self._cobbler_path = ServerConfig.objects.by_key("cobbler.command")
def connect(self):
"""Connect to DHCP server via SSH."""
if not self._conn:
self._conn = SSH(self._fqdn)
self._conn.connect()
def close(self):
"""Close connection to DHCP server."""
if self._conn:
self._conn.close()
def deploy(self):
self.connect()
if not self.is_installed():
raise CobblerException("No Cobbler service found: {}".format(
self._fqdn))
if not self.is_running():
raise CobblerException("Cobbler server is not running: {}".format(
self._fqdn))
machines = Machine.active_machines.filter(fqdn_domain=self._domain.pk)
cobbler_machines = self.get_machines()
cobbler_commands = []
for machine in machines:
if machine.fqdn in cobbler_machines:
cobbler_commands.append(
get_cobbler_update_command(machine, self._cobbler_path))
else:
cobbler_commands.append(
get_cobbler_add_command(machine, self._cobbler_path))
for command in cobbler_commands: # TODO: Convert this to a single ssh call (performance)
_, stderr, exitcode = self._conn.execute(command)
if exitcode:
logger.error("failed to execute %s on %s with error %s",
command, self._fqdn, stderr)
self.close()
def is_installed(self):
"""Check if Cobbler server is available."""
if self._conn.check_path(self._cobbler_path, '-x'):
return True
return False
def is_running(self):
"""Check if the Cobbler daemon is running via the cobbler version command."""
command = "{} version".format(self._cobbler_path)
_, _, exitstatus = self._conn.execute(command)
if exitstatus == 0:
return True
return False
def get_machines(self):
stdout, stderr, exitstatus = self._conn.execute(
"{cobbler} system list".format(cobbler=self._cobbler_path))
if exitstatus:
logger.warning("system list failed on %s with %s", self._fqdn,
stderr)
raise CobblerException(
"system list failed on {server}".format(server=self._fqdn))
clean_out = [system.strip(' \n\t') for system in stdout]
return clean_out
@staticmethod
def profile_normalize(string):
'''
This method replaces the second colon (:) of a string with a dash (-)
This is to convert:
x86_64:SLE-12-SP4-Server-LATEST:install
to
x86_64:SLE-12-SP4-Server-LATEST-install
until cobbler returns profiles where arch:distro:profile are all separated via :
'''
return string.replace(':', '-', 2).replace('-', ':', 1)
def setup(self, machine: Machine, choice: str):
logger.info("setup called for %s with %s on cobbler server %s ",
machine.fqdn, self._fqdn, choice)
cobbler_profile = "{arch}:{profile}".format(arch=machine.architecture,
profile=choice)
# ToDo: Revert this after renaming cobbler profiles
cobbler_profile = CobblerServer.profile_normalize(cobbler_profile)
command = "{cobbler} system edit --name={machine} --profile={profile} --netboot=True"\
.format(cobbler=self._cobbler_path, machine=machine.fqdn, profile=cobbler_profile)
logger.debug("command for setup: %s", command)
self.connect()
try:
stdout, stderr, exitstatus = self._conn.execute(command)
if exitstatus:
logger.warning("setup of %s with %s failed on %s with %s",
machine.fqdn, cobbler_profile, self._fqdn,
stderr)
raise CobblerException(
"setup of {machine} with {profile} failed on {server} with {error}"
.format(machine=machine.fqdn,
arch=cobbler_profile,
server=self._fqdn))
except:
pass
finally:
self.close()
class CobblerServer:
def __init__(self, fqdn, domain):
self._fqdn = fqdn
self._conn = None
self._domain = domain
self._cobbler_path = ServerConfig.objects.by_key("cobbler.command")
def connect(self):
"""Connect to DHCP server via SSH."""
if not self._conn:
self._conn = SSH(self._fqdn)
self._conn.connect()
def close(self):
"""Close connection to DHCP server."""
if self._conn:
self._conn.close()
def deploy(self):
self.connect()
if not self.is_installed():
raise CobblerException("No Cobbler service found: {}".format(self._fqdn))
if not self.is_running():
raise CobblerException("Cobbler server is not running: {}".format(self._fqdn))
machines = Machine.active_machines.filter(fqdn_domain=self._domain.pk)
cobbler_machines = self.get_machines()
cobbler_commands = []
for machine in machines:
if machine.fqdn in cobbler_machines:
cobbler_commands.append(get_cobbler_update_command(machine, self._cobbler_path))
else:
cobbler_commands.append(get_cobbler_add_command(machine, self._cobbler_path))
for command in cobbler_commands: # TODO: Convert this to a single ssh call (performance)
_, stderr, exitcode = self._conn.execute(command)
if exitcode:
logger.error("failed to execute %s on %s with error %s",
command, self._fqdn, stderr)
self.close()
def is_installed(self):
"""Check if Cobbler server is available."""
if self._conn.check_path(self._cobbler_path, '-x'):
return True
return False
def is_running(self):
"""Check if the Cobbler daemon is running via the cobbler version command."""
command = "{} version".format(self._cobbler_path)
_, _, exitstatus = self._conn.execute(command)
if exitstatus == 0:
return True
return False
def get_machines(self):
stdout, stderr, exitstatus = self._conn.execute(
"{cobbler} system list".format(cobbler=self._cobbler_path))
if exitstatus:
logger.warning("system list failed on %s with %s", self._fqdn, stderr)
raise CobblerException("system list failed on {server}".format(server=self._fqdn))
clean_out = [system.strip(' \n\t') for system in stdout]
return clean_out
class CobblerServer:
def __init__(self, fqdn, domain):
self._fqdn = fqdn
self._conn = None
self._domain = domain
self._cobbler_path = ServerConfig.objects.by_key("cobbler.command")
@staticmethod
def from_machine(machine: Machine):
"""
Return the cobbler server associated to a machine
:param machine: Machine object which is managed by the cobbler server to fetch
:returns: The corresponding cobbler server or None
"""
domain = machine.fqdn_domain
server = domain.cobbler_server.all()
if server and server[0]:
return CobblerServer(server[0].fqdn, domain)
return None
def connect(self):
"""Connect to DHCP server via SSH."""
if not self._conn:
self._conn = SSH(self._fqdn)
self._conn.connect()
def close(self):
"""Close connection to DHCP server."""
if self._conn:
self._conn.close()
def deploy(self):
self.connect()
if not self.is_installed():
raise CobblerException("No Cobbler service found: {}".format(self._fqdn))
if not self.is_running():
raise CobblerException("Cobbler server is not running: {}".format(self._fqdn))
machines = Machine.active_machines.filter(fqdn_domain=self._domain.pk)
cobbler_machines = self.get_machines()
cobbler_commands = []
for machine in machines:
if machine.fqdn in cobbler_machines:
cobbler_commands.append(get_cobbler_update_command(machine, self._cobbler_path))
else:
cobbler_commands.append(get_cobbler_add_command(machine, self._cobbler_path))
if hasattr(machine, 'bmc') and machine.bmc:
cobbler_commands.append(get_bmc_command(machine, self._cobbler_path))
for command in cobbler_commands: # TODO: Convert this to a single ssh call (performance)
logger.debug("executing %s ", command)
_, stderr, exitcode = self._conn.execute(command)
if exitcode:
logger.error("failed to execute %s on %s with error %s",
command, self._fqdn, stderr)
self.close()
def update(self, machine: Machine):
self.connect()
self._check()
command = get_cobbler_update_command(machine, self._cobbler_path)
_, stderr, exitcode = self._conn.execute(command)
if exitcode:
raise CobblerException("Updating {machine} failed with {err}".format(
machine.fqdn, stderr))
def remove(self, machine: Machine):
#ToDo: We do not remove machines from cobbler server actively in orthos2 yet
logging.warning("cobbler remove is switched off")
return
self.connect()
if not self.is_installed():
raise CobblerException("No Cobbler service found: {}".format(self._fqdn))
if not self.is_running():
raise CobblerException("Cobbler server is not running: {}".format(self._fqdn))
command = "{cobbler} system remove --name {fqdn}".format(
cobbler=self._cobbler_path, fqdn=machine.fqdn)
_, stderr, exitcode = self._conn.execute(command)
if(exitcode):
logging.error("Removing %s failed with '%s'", machine.fqdn, stderr)
def is_installed(self):
"""Check if Cobbler server is available."""
if self._conn.check_path(self._cobbler_path, '-x'):
return True
return False
def is_running(self):
"""Check if the Cobbler daemon is running via the cobbler version command."""
command = "{} version".format(self._cobbler_path)
_, _, exitstatus = self._conn.execute(command)
if exitstatus == 0:
return True
return False
def get_machines(self):
stdout, stderr, exitstatus = self._conn.execute(
"{cobbler} system list".format(cobbler=self._cobbler_path))
if exitstatus:
logger.warning("system list failed on %s with %s", self._fqdn, stderr)
raise CobblerException("system list failed on {server}".format(server=self._fqdn))
clean_out = [system.strip(' \n\t') for system in stdout]
return clean_out
def setup(self, machine: Machine, choice: str):
logger.info("setup called for %s with %s on cobbler server %s ", machine.fqdn, self._fqdn,
choice)
if choice:
cobbler_profile = "{arch}:{profile}".format(arch=machine.architecture, profile=choice)
else:
cobbler_profile = get_default_profile(machine)
command = "{cobbler} system edit --name={machine} --profile={profile} --netboot=True"\
.format(cobbler=self._cobbler_path, machine=machine.fqdn, profile=cobbler_profile)
logger.debug("command for setup: %s", command)
self.connect()
try:
stdout, stderr, exitstatus = self._conn.execute(command)
if exitstatus:
logger.warning("setup of %s with %s failed on %s with %s", machine.fqdn,
cobbler_profile, self._fqdn, stderr)
raise CobblerException(
"setup of {machine} with {profile} failed on {server} with {error}".format(
machine=machine.fqdn, arch=cobbler_profile, server=self._fqdn))
except:
pass
finally:
self.close()
def powerswitch(self, machine: Machine, action: str):
logger.debug("powerswitching of %s called with action %s", machine.fqdn, action)
self.connect()
cobbler_action = ""
if action == "reboot":
cobbler_action = "reboot"
else:
cobbler_action = "power" + action
command = "{cobbler} system {action} --name {fqdn}".format(cobbler=self._cobbler_path,
action=cobbler_action,
fqdn=machine.fqdn)
out, stderr, exitcode = self._conn.execute(command)
if exitcode:
logger.warning("Powerswitching of %s with %s failed on %s with %s", machine.fqdn,
command, self._fqdn, stderr)
raise CobblerException(
"Powerswitching of {machine} with {command} failed on {server} with {error}".format(
machine=machine.fqdn, command=command, server=self._fqdn))
return out
def _check(self):
if not self.is_installed():
raise CobblerException("No Cobbler service found: {}".format(self._fqdn))
if not self.is_running():
raise CobblerException("Cobbler server is not running: {}".format(self._fqdn))
class Libvirt(VirtualizationAPI):
class Meta:
proxy = True
VIRSH = 'virsh -c qemu:///system'
IGNORE_STDERR = ['domain is not running', 'no domain with matching name']
QEMU_IMG_CONVERT = '/usr/bin/qemu-img convert -O qcow2 -o preallocation=metadata {0}.tmp {0}'
def __init__(self):
self.conn = None
def get_image_list(self):
"""
Return the available architectures and the full image list (over all available
architectures).
Return format:
(
['<arch1>', '<arch2>', ...], [('<value>', '<option>'), ...]
)
"""
from orthos2.data.models import ServerConfig
architectures = [self.host.architecture.name]
image_directory = ServerConfig.objects.by_key(
'virtualization.libvirt.images.directory')
image_list = []
try:
for architecture in architectures:
directory = '{}/{}/'.format(image_directory.rstrip('/'),
architecture)
for image in os.listdir(directory):
path = directory + image
size = os.path.getsize(path)
atime = str(date.fromtimestamp(os.path.getmtime(path)))
if size < (1024**3):
size = int(size / (1024**2))
size = '{}M'.format(size)
else:
size = int(size / (1024**3))
size = '{}G'.format(size)
pretty_image = image.split('.')[0]
image_list.append(
(image, '{} ({} {})'.format(pretty_image, atime,
size)))
except FileNotFoundError as e:
logger.exception(e)
return (architectures, image_list)
def connect(function):
"""Create SSH connection if needed."""
def decorator(self, *args, **kwargs):
from orthos2.utils.ssh import SSH
if not self.conn:
self.conn = SSH(self.host.fqdn)
self.conn.connect()
return function(self, *args, **kwargs)
return decorator
@connect
def _execute(self, command):
return self.conn.execute(command)
def check_connection(self):
"""Check libvirt connection (running libvirt)."""
stdout, stderr, exitstatus = self._execute('{} version'.format(
self.VIRSH))
if exitstatus == 0:
return True
return False
def get_list(self, parameters='--all'):
"""Return `virsh list` output."""
stdout, stderr, exitstatus = self._execute('{} list {}'.format(
self.VIRSH, parameters))
if exitstatus == 0:
return ''.join(stdout)
else:
raise Exception(''.join(stderr))
return False
def check_network_bridge(self, bridge='br0'):
"""
Execute `create_bridge.sh` script remotely and try to set up bridge if it doesn't exist.
Returns true if the bridge is available, false otherwise.
"""
stdout, stderr, exitstatus = self.conn.execute_script_remote(
'create_bridge.sh')
if exitstatus != 0:
raise Exception(''.join(stderr))
stdout, stderr, exitstatus = self.conn.execute('brctl show')
if exitstatus != 0:
raise Exception(''.join(stderr))
for line in stdout:
if line.startswith(bridge):
return True
raise False
def generate_hostname(self):
"""
Generate domain name (hostname).
Check hostnames against Orthos machines and libvirt `virsh list`.
"""
hostname = None
occupied_hostnames = {
vm.hostname
for vm in self.host.get_virtual_machines()
}
libvirt_list = self.get_list()
for line in libvirt_list.split('\n')[2:]:
columns = line.strip().split()
if columns:
domain_name = columns[1]
occupied_hostnames.add(domain_name)
for i in range(1, self.host.vm_max + 1):
hostname_ = '{}-{}'.format(self.host.hostname, i)
if hostname_ not in occupied_hostnames:
hostname = hostname_
break
if hostname is None:
raise Exception("All hostnames (domain names) busy!")
return hostname
def generate_networkinterfaces(self,
amount=1,
bridge='br0',
model='virtio'):
"""Generate networkinterfaces."""
from orthos2.data.models import NetworkInterface
networkinterfaces = []
for i in range(amount):
mac_address = get_random_mac_address()
while NetworkInterface.objects.filter(
mac_address=mac_address).count() != 0:
mac_address = get_random_mac_address()
networkinterface = NetworkInterface(mac_address=mac_address)
networkinterface.bridge = bridge
networkinterface.model = model
networkinterfaces.append(networkinterface)
return networkinterfaces
def copy_image(self, image, disk_image):
"""Copy and allocate disk image."""
stdout, stderr, exitstatus = self.conn.execute('cp {} {}.tmp'.format(
image, disk_image))
if exitstatus != 0:
return False
stdout, stderr, exitstatus = self.conn.execute(
self.QEMU_IMG_CONVERT.format(disk_image))
if exitstatus != 0:
return False
stdout, stderr, exitstatus = self.conn.execute(
'rm -rf {}.tmp'.format(disk_image))
if exitstatus != 0:
return False
return True
def delete_disk_image(self, disk_image):
"""Delete the old disk image."""
stdout, stderr, exitstatus = self.conn.execute(
'rm -rf {}'.format(disk_image))
if exitstatus != 0:
return False
return True
def calculate_vcpu(self):
"""Return virtual CPU amount."""
vcpu = 1
host_cpu_cores = self.host.cpu_cores
if host_cpu_cores is not None:
vcpu = int((host_cpu_cores - 2) / self.host.vm_max)
if vcpu == 0:
vcpu = 1
return vcpu
def check_memory(self, memory_amount):
"""
Check if memory amount for VM is available on host.
Reserve 2GB of memory for host system.
"""
host_ram_amount = self.host.ram_amount
host_reserved_ram_amount = 2048
if host_ram_amount:
if memory_amount > (host_ram_amount - host_reserved_ram_amount):
raise Exception("Host system has only {}MB of memory!".format(
memory_amount))
else:
raise Exception(
"Can't detect memory size of host system '{}'".format(
self.host))
return True
def execute_virt_install(self, *args, dry_run=True, **kwargs):
"""Run `virt-install` command."""
command = '/usr/bin/virt-install '
command += '--name {hostname} '
command += '--vcpus {vcpu} '
command += '--memory {memory} '
disk_ = '--disk {},'.format(kwargs['disk']['image'])
disk_ += 'size={},'.format(kwargs['disk']['size'])
disk_ += 'format={},'.format(kwargs['disk']['format'])
disk_ += 'sparse={},'.format(kwargs['disk']['sparse'])
disk_ += 'bus={} '.format(kwargs['disk']['bus'])
command += disk_
for networkinterface in kwargs.get('networkinterfaces', []):
networkinterface_ = '--network model={},'.format(
networkinterface.model)
networkinterface_ += 'bridge={},'.format(networkinterface.bridge)
networkinterface_ += 'mac={} '.format(networkinterface.mac_address)
command += networkinterface_
command += '{boot} '
vnc = kwargs.get('vnc', None)
if vnc and vnc['enabled']:
command += '--graphics vnc,listen=0.0.0.0,port={} '.format(
vnc['port'])
command += kwargs.get('parameters', '')
if dry_run:
command += '--dry-run'
command = command.format(**kwargs)
logger.debug(command)
stdout, stderr, exitstatus = self.conn.execute(command)
if exitstatus != 0:
raise Exception(''.join(stderr))
return True
def _create(self, vm, *args, **kwargs):
"""
Wrapper function for creating a VM.
Steps:
- check connection to host
- check maxinmum VM number limit
- check network bridge
- check image source directory (if needed)
- check Open Virtual Machine Firmware (OVMF) binary (if needed)
- check memory size
- generate hostname (=domain name)
- copy image to disk image (if needed)
- run `virt-install`
"""
from orthos2.data.models import NetworkInterface, ServerConfig
bridge = ServerConfig.objects.by_key('virtualization.libvirt.bridge')
image_directory = ServerConfig.objects.by_key(
'virtualization.libvirt.images.directory')
disk_image_directory = '/abuild/orthos-vm-images/'
disk_image = '{}/{}.qcow2'.format(disk_image_directory.rstrip('/'),
'{}')
ovmf = ServerConfig.objects.by_key('virtualization.libvirt.ovmf.path')
image_directory = '{}/{}/'.format(image_directory.rstrip('/'),
kwargs['architecture'])
if not self.check_connection():
raise Exception("Host system not reachable!")
if self.host.get_virtual_machines().count() >= self.host.vm_max:
raise Exception("Maximum number of VMs reached!")
if not self.check_network_bridge(bridge=bridge):
raise Exception("Network bridge setup failed!")
if kwargs['image'] is not None:
if not self.conn.check_path(image_directory, '-e'):
raise Exception(
"Image source directory missing on host system!")
if not self.conn.check_path(disk_image_directory, '-w'):
raise Exception(
"Image disk directory missing on host system: {}!".format(
disk_image_directory))
if kwargs['uefi_boot']:
if not self.conn.check_path(ovmf, '-e'):
raise Exception("OVMF file not found: '{}'!".format(ovmf))
boot = '--boot loader={},network'.format(ovmf)
else:
boot = '--boot network,hd,menu=off,bios.useserial=on'
self.check_memory(kwargs['ram_amount'])
vm.hostname = self.generate_hostname()
vm.hypervisor = self.host
vm.fqdn = '{}.{}'.format(vm.hostname, self.host.fqdn_domain.name)
vnc_port = 5900 + int(vm.hostname.split('-')[1])
vm.vnc = {'enabled': kwargs['vnc'], 'port': vnc_port}
vm.cpu_cores = self.calculate_vcpu()
vm.ram_amount = kwargs['ram_amount']
disk_image = disk_image.format(vm.hostname)
if kwargs['image'] is not None:
image = '{}/{}'.format(image_directory.rstrip('/'),
kwargs['image'])
if not self.copy_image(image, disk_image):
raise Exception("Couldn't copy image: {} > {}!".format(
image, disk_image))
else:
self.delete_disk_image(disk_image)
disk = {
'image': disk_image,
'size': kwargs['disk_size'],
'format': 'qcow2',
'sparse': True,
'bus': 'virtio'
}
networkinterfaces = self.generate_networkinterfaces(
amount=kwargs['networkinterfaces'], bridge=bridge)
parameters = '--events on_reboot=restart,on_poweroff=destroy '
parameters += '--import '
parameters += '--noautoconsole '
parameters += '--autostart '
parameters += kwargs['parameters']
self.execute_virt_install(
hostname=vm.hostname,
vcpu=vm.cpu_cores,
memory=vm.ram_amount,
disk=disk,
networkinterfaces=networkinterfaces,
boot=boot,
vnc=vm.vnc,
parameters=parameters,
)
self.execute_virt_install(hostname=vm.hostname,
vcpu=vm.cpu_cores,
memory=vm.ram_amount,
disk=disk,
networkinterfaces=networkinterfaces,
boot=boot,
vnc=vm.vnc,
parameters=parameters,
dry_run=False)
vm.unsaved_networkinterfaces = []
for networkinterface in networkinterfaces:
vm.unsaved_networkinterfaces.append(networkinterface)
return True
def _remove(self, vm):
"""Wrapper function for removing a VM (destroy domain > undefine domain)."""
if not self.check_connection():
raise Exception("Host system not reachable!")
self.destroy(vm)
self.undefine(vm)
return True
def destroy(self, vm):
"""Destroy VM on host system. Ignore `domain is not running` error and proceed."""
stdout, stderr, exitstatus = self._execute('{} destroy {}'.format(
self.VIRSH, vm.hostname))
if exitstatus != 0:
stderr = ''.join(stderr)
if not any(line in stderr for line in self.IGNORE_STDERR):
raise Exception(stderr)
return True
def undefine(self, vm):
"""Undefine VM on host system."""
stdout, stderr, exitstatus = self._execute('{} undefine {}'.format(
self.VIRSH, vm.hostname))
if exitstatus != 0:
stderr = ''.join(stderr)
if not any(line in stderr for line in self.IGNORE_STDERR):
raise Exception(stderr)
return True
