def get_installations(fqdn): """Retrieve information of the installations.""" try: machine = Machine.objects.get(fqdn=fqdn) except Machine.DoesNotExist: logger.warning("Machine '{}' does not exist".format(fqdn)) return False conn = None timer = None try: conn = SSH(fqdn) conn.connect() timer = threading.Timer(5 * 60, conn.close) timer.start() # Installations logger.debug("Collect installations...") installations = [] output, stderr, exitstatus = conn.execute_script_remote( 'machine_get_installations.sh') if output: for line in output: if line.startswith('--'): installation = Installation(machine=machine) installations.append(installation) elif line.startswith('ARCH='): installation.architecture = line.split('=')[1].strip() elif line.startswith('KERNEL='): installation.kernelversion = line.split('=')[1].strip() elif line.startswith('RUNNING='): installation.active = line.startswith('RUNNING=1') elif line.startswith('DIST='): installation.distribution = line.split('=')[1].strip() elif line.startswith('PART='): installation.partition = line.split('=')[1].strip() return installations except Exception as e: logger.exception("{} ({})".format(fqdn, e)) return False finally: if conn: conn.close() if timer: timer.cancel() return None
def execute(self): """ Executes the task. """ if not ServerConfig.objects.bool_by_key('orthos.debug.motd.write'): logger.warning("Disabled: set 'orthos.debug.motd.write' to 'true'") return BEGIN = '-' * 69 + ' Orthos{ --' LINE = '-' * 80 END = '-' * 69 + ' Orthos} --' try: machine = Machine.objects.get(fqdn=self.fqdn) except Machine.DoesNotExist: logger.error("Machine does not exist: fqdn={}".format(self.fqdn)) return conn = None try: conn = SSH(machine.fqdn) conn.connect() motd = conn.get_file('/etc/motd.orthos', 'w') print(BEGIN, file=motd) print( "Machine of the ARCH team. Contact <{}> for problems.".format( machine.get_support_contact()), file=motd) if machine.comment: print("INFO: " + machine.comment, file=motd) if machine.administrative: print( "This machine is an administrative machine. DON\'T TOUCH!", file=motd) if machine.reserved_by: print(LINE, file=motd) if machine.reserved_until == timezone.ZERO: print("This machine is RESERVED by {}.".format( machine.reserved_by), file=motd) else: print("This machine is RESERVED by {} until {}.".format( machine.reserved_by, machine.reserved_until), file=motd) print('', file=motd) print(wrap80(machine.reserved_reason), file=motd) print(END, file=motd) motd.close() stdout, stderr, exitstatus = conn.execute_script_remote( 'machine_sync_motd.sh') if exitstatus != 0: logger.exception("({}) {}".format(machine.fqdn, stderr)) raise Exception(stderr) except SSH.Exception as e: logger.error("({}) {}".format(machine.fqdn, e)) return False except IOError as e: logger.error("({}) {}".format(machine.fqdn, e)) return False finally: if conn: conn.close()
def get_hardware_information(fqdn): """Retrieve information of the system.""" try: machine = Machine.objects.get(fqdn=fqdn) except Machine.DoesNotExist: logger.warning("Machine '{}' does not exist".format(fqdn)) return # set needed values for several checks from original machine machine_ = Machine( architecture=machine.architecture ) conn = None timer = None try: conn = SSH(fqdn) conn.connect() timer = threading.Timer(5 * 60, conn.close) timer.start() # CPUs logger.debug("Get CPU number...") output, stderr, exitstatus = conn.execute_script_remote('machine_get_cpu_number.sh') if output: for line in output: if line.startswith('SOCKETS'): machine_.cpu_physical = int(line.split('=')[1]) elif line.startswith('CORES'): machine_.cpu_cores = int(line.split('=')[1]) elif line.startswith('THREADS'): machine_.cpu_threads = int(line.split('=')[1]) logger.debug("Get CPU type...") output, stderr, exitstatus = conn.execute_script_remote('machine_get_cpu_type.sh') if output and output[0]: machine_.cpu_model = output[0].strip() logger.debug("Get CPU flags...") output, stderr, exitstatus = conn.execute_script_remote('machine_get_cpu_flags.sh') if output and output[0]: machine_.cpu_flags = output[0].strip() logger.debug("Get CPU speed...") output, stderr, exitstatus = conn.execute_script_remote('machine_get_cpu_speed.sh') if output and output[0]: machine_.cpu_speed = Decimal(int(output[0].strip()) / 1000000) logger.debug("Get CPU ID...") output, stderr, exitstatus = conn.execute_script_remote('machine_get_cpu_id.sh') if output and output[0]: machine_.cpu_id = output[0].strip() # EFI logger.debug("Check for EFI...") try: efi_file = conn.get_file('/sys/firmware/efi', 'r') efi_file.close() machine_.efi = True except IOError: machine_.efi = False # Memory logger.debug("Get RAM amount...") for line in conn.read_file('/proc/meminfo'): if line.startswith('MemTotal'): machine_.ram_amount = int(int(line.split()[1]) / 1024) # Virtualization capability VM_HOST_MIN_RAM_MB = 7000 machine_.vm_capable = False # Virtualization: x86 logger.debug("Check for VM capability...") if machine_.architecture_id == Architecture.Type.X86_64: cpu_flags = machine_.cpu_flags if cpu_flags: cpu_flags = cpu_flags.upper() if ((cpu_flags.find('VMX') >= 0 or cpu_flags.find('SVM') >= 0) and int(machine_.ram_amount) > VM_HOST_MIN_RAM_MB): machine_.vm_capable = True # Virtualization: ppc64le if machine_.architecture_id == Architecture.Type.PPC64LE: for line in conn.read_file('/proc/cpuinfo'): if line.startswith('firmware') and 'OPAL' in line: machine_.vm_capable = True # Disk logger.debug("Get disk information...") stdout, stderr, exitstatus = conn.execute('hwinfo --disk') for line in stdout: line = line.strip() if line.startswith('Size:'): machine_.disk_primary_size = int(int(line.split()[1]) / 2 / 1024 ** 2) elif line.startswith('Attached to:'): opening_bracket = line.find('(') closing_bracket = line.find(')') if opening_bracket > 0 and closing_bracket > 0: machine_.disk_type = line[opening_bracket + 1:closing_bracket] else: machine_.disk_type = 'Unknown disk type' break # lsmod logger.debug("Get 'lsmod'...") stdout, stderr, exitstatus = conn.execute('lsmod') machine_.lsmod = normalize_ascii("".join(stdout)) # lspci logger.debug("Get 'lspci'...") stdout, stderr, exitstatus = conn.execute('lspci -vvv -nn') machine_.lspci = normalize_ascii("".join(stdout)) # last logger.debug("Get 'last'...") output, stderr, exitstatus = conn.execute('last | grep -v reboot | head -n 1') string = ''.join(output) result = string[0:8] + string[38:49] machine_.last = normalize_ascii("".join(result)) # hwinfo logger.debug("Get 'hwinfo' (full)...") stdout, stderr, exitstatus = conn.execute( 'hwinfo --bios ' + '--block --bridge --cdrom --cpu --disk --floppy --framebuffer ' + '--gfxcard --hub --ide --isapnp --isdn --keyboard --memory ' + '--monitor --mouse --netcard --network --partition --pci --pcmcia ' + '--scsi --smp --sound --sys --tape --tv --usb --usb-ctrl --wlan' ) machine_.hwinfo = normalize_ascii("".join(stdout)) # dmidecode logger.debug("Get 'dmidecode'...") stdout, stderr, exitstatus = conn.execute('dmidecode') machine_.dmidecode = normalize_ascii("".join(stdout)) # dmesg logger.debug("Get 'dmesg'...") stdout, stderr, exitstatus = conn.execute( 'if [ -e /var/log/boot.msg ]; then ' + 'cat /var/log/boot.msg; else journalctl -xl | head -n200; ' + 'fi' ) machine_.dmesg = normalize_ascii("".join(stdout)) # lsscsi logger.debug("Get 'lsscsi'...") stdout, stderr, exitstatus = conn.execute('lsscsi -s') machine_.lsscsi = normalize_ascii("".join(stdout)) # lsusb logger.debug("Get 'lsusb'...") stdout, stderr, exitstatus = conn.execute('lsusb') machine_.lsusb = normalize_ascii("".join(stdout)) # IPMI logger.debug("Check for IPMI...") machine_.ipmi = machine_.dmidecode.find('IPMI') >= 0 # Firmware script logger.debug("Get BIOS version...") output, stderr, exitstatus = conn.execute_script_remote('machine_get_firmware.sh') if output and output[0]: machine_.bios_version = output[0].strip() return machine_ except Exception as e: logger.error("{} ({})".format(fqdn, e)) return False finally: if conn: conn.close() if timer: timer.cancel() return None
class Libvirt(VirtualizationAPI): class Meta: proxy = True VIRSH = 'virsh -c qemu:///system' IGNORE_STDERR = ['domain is not running', 'no domain with matching name'] QEMU_IMG_CONVERT = '/usr/bin/qemu-img convert -O qcow2 -o preallocation=metadata {0}.tmp {0}' def __init__(self): self.conn = None def get_image_list(self): """ Return the available architectures and the full image list (over all available architectures). Return format: ( ['<arch1>', '<arch2>', ...], [('<value>', '<option>'), ...] ) """ from orthos2.data.models import ServerConfig architectures = [self.host.architecture.name] image_directory = ServerConfig.objects.by_key( 'virtualization.libvirt.images.directory') image_list = [] try: for architecture in architectures: directory = '{}/{}/'.format(image_directory.rstrip('/'), architecture) for image in os.listdir(directory): path = directory + image size = os.path.getsize(path) atime = str(date.fromtimestamp(os.path.getmtime(path))) if size < (1024**3): size = int(size / (1024**2)) size = '{}M'.format(size) else: size = int(size / (1024**3)) size = '{}G'.format(size) pretty_image = image.split('.')[0] image_list.append( (image, '{} ({} {})'.format(pretty_image, atime, size))) except FileNotFoundError as e: logger.exception(e) return (architectures, image_list) def connect(function): """Create SSH connection if needed.""" def decorator(self, *args, **kwargs): from orthos2.utils.ssh import SSH if not self.conn: self.conn = SSH(self.host.fqdn) self.conn.connect() return function(self, *args, **kwargs) return decorator @connect def _execute(self, command): return self.conn.execute(command) def check_connection(self): """Check libvirt connection (running libvirt).""" stdout, stderr, exitstatus = self._execute('{} version'.format( self.VIRSH)) if exitstatus == 0: return True return False def get_list(self, parameters='--all'): """Return `virsh list` output.""" stdout, stderr, exitstatus = self._execute('{} list {}'.format( self.VIRSH, parameters)) if exitstatus == 0: return ''.join(stdout) else: raise Exception(''.join(stderr)) return False def check_network_bridge(self, bridge='br0'): """ Execute `create_bridge.sh` script remotely and try to set up bridge if it doesn't exist. Returns true if the bridge is available, false otherwise. """ stdout, stderr, exitstatus = self.conn.execute_script_remote( 'create_bridge.sh') if exitstatus != 0: raise Exception(''.join(stderr)) stdout, stderr, exitstatus = self.conn.execute('brctl show') if exitstatus != 0: raise Exception(''.join(stderr)) for line in stdout: if line.startswith(bridge): return True raise False def generate_hostname(self): """ Generate domain name (hostname). Check hostnames against Orthos machines and libvirt `virsh list`. """ hostname = None occupied_hostnames = { vm.hostname for vm in self.host.get_virtual_machines() } libvirt_list = self.get_list() for line in libvirt_list.split('\n')[2:]: columns = line.strip().split() if columns: domain_name = columns[1] occupied_hostnames.add(domain_name) for i in range(1, self.host.vm_max + 1): hostname_ = '{}-{}'.format(self.host.hostname, i) if hostname_ not in occupied_hostnames: hostname = hostname_ break if hostname is None: raise Exception("All hostnames (domain names) busy!") return hostname def generate_networkinterfaces(self, amount=1, bridge='br0', model='virtio'): """Generate networkinterfaces.""" from orthos2.data.models import NetworkInterface networkinterfaces = [] for i in range(amount): mac_address = get_random_mac_address() while NetworkInterface.objects.filter( mac_address=mac_address).count() != 0: mac_address = get_random_mac_address() networkinterface = NetworkInterface(mac_address=mac_address) networkinterface.bridge = bridge networkinterface.model = model networkinterfaces.append(networkinterface) return networkinterfaces def copy_image(self, image, disk_image): """Copy and allocate disk image.""" stdout, stderr, exitstatus = self.conn.execute('cp {} {}.tmp'.format( image, disk_image)) if exitstatus != 0: return False stdout, stderr, exitstatus = self.conn.execute( self.QEMU_IMG_CONVERT.format(disk_image)) if exitstatus != 0: return False stdout, stderr, exitstatus = self.conn.execute( 'rm -rf {}.tmp'.format(disk_image)) if exitstatus != 0: return False return True def delete_disk_image(self, disk_image): """Delete the old disk image.""" stdout, stderr, exitstatus = self.conn.execute( 'rm -rf {}'.format(disk_image)) if exitstatus != 0: return False return True def calculate_vcpu(self): """Return virtual CPU amount.""" vcpu = 1 host_cpu_cores = self.host.cpu_cores if host_cpu_cores is not None: vcpu = int((host_cpu_cores - 2) / self.host.vm_max) if vcpu == 0: vcpu = 1 return vcpu def check_memory(self, memory_amount): """ Check if memory amount for VM is available on host. Reserve 2GB of memory for host system. """ host_ram_amount = self.host.ram_amount host_reserved_ram_amount = 2048 if host_ram_amount: if memory_amount > (host_ram_amount - host_reserved_ram_amount): raise Exception("Host system has only {}MB of memory!".format( memory_amount)) else: raise Exception( "Can't detect memory size of host system '{}'".format( self.host)) return True def execute_virt_install(self, *args, dry_run=True, **kwargs): """Run `virt-install` command.""" command = '/usr/bin/virt-install ' command += '--name {hostname} ' command += '--vcpus {vcpu} ' command += '--memory {memory} ' disk_ = '--disk {},'.format(kwargs['disk']['image']) disk_ += 'size={},'.format(kwargs['disk']['size']) disk_ += 'format={},'.format(kwargs['disk']['format']) disk_ += 'sparse={},'.format(kwargs['disk']['sparse']) disk_ += 'bus={} '.format(kwargs['disk']['bus']) command += disk_ for networkinterface in kwargs.get('networkinterfaces', []): networkinterface_ = '--network model={},'.format( networkinterface.model) networkinterface_ += 'bridge={},'.format(networkinterface.bridge) networkinterface_ += 'mac={} '.format(networkinterface.mac_address) command += networkinterface_ command += '{boot} ' vnc = kwargs.get('vnc', None) if vnc and vnc['enabled']: command += '--graphics vnc,listen=0.0.0.0,port={} '.format( vnc['port']) command += kwargs.get('parameters', '') if dry_run: command += '--dry-run' command = command.format(**kwargs) logger.debug(command) stdout, stderr, exitstatus = self.conn.execute(command) if exitstatus != 0: raise Exception(''.join(stderr)) return True def _create(self, vm, *args, **kwargs): """ Wrapper function for creating a VM. Steps: - check connection to host - check maxinmum VM number limit - check network bridge - check image source directory (if needed) - check Open Virtual Machine Firmware (OVMF) binary (if needed) - check memory size - generate hostname (=domain name) - copy image to disk image (if needed) - run `virt-install` """ from orthos2.data.models import NetworkInterface, ServerConfig bridge = ServerConfig.objects.by_key('virtualization.libvirt.bridge') image_directory = ServerConfig.objects.by_key( 'virtualization.libvirt.images.directory') disk_image_directory = '/abuild/orthos-vm-images/' disk_image = '{}/{}.qcow2'.format(disk_image_directory.rstrip('/'), '{}') ovmf = ServerConfig.objects.by_key('virtualization.libvirt.ovmf.path') image_directory = '{}/{}/'.format(image_directory.rstrip('/'), kwargs['architecture']) if not self.check_connection(): raise Exception("Host system not reachable!") if self.host.get_virtual_machines().count() >= self.host.vm_max: raise Exception("Maximum number of VMs reached!") if not self.check_network_bridge(bridge=bridge): raise Exception("Network bridge setup failed!") if kwargs['image'] is not None: if not self.conn.check_path(image_directory, '-e'): raise Exception( "Image source directory missing on host system!") if not self.conn.check_path(disk_image_directory, '-w'): raise Exception( "Image disk directory missing on host system: {}!".format( disk_image_directory)) if kwargs['uefi_boot']: if not self.conn.check_path(ovmf, '-e'): raise Exception("OVMF file not found: '{}'!".format(ovmf)) boot = '--boot loader={},network'.format(ovmf) else: boot = '--boot network,hd,menu=off,bios.useserial=on' self.check_memory(kwargs['ram_amount']) vm.hostname = self.generate_hostname() vm.hypervisor = self.host vm.fqdn = '{}.{}'.format(vm.hostname, self.host.fqdn_domain.name) vnc_port = 5900 + int(vm.hostname.split('-')[1]) vm.vnc = {'enabled': kwargs['vnc'], 'port': vnc_port} vm.cpu_cores = self.calculate_vcpu() vm.ram_amount = kwargs['ram_amount'] disk_image = disk_image.format(vm.hostname) if kwargs['image'] is not None: image = '{}/{}'.format(image_directory.rstrip('/'), kwargs['image']) if not self.copy_image(image, disk_image): raise Exception("Couldn't copy image: {} > {}!".format( image, disk_image)) else: self.delete_disk_image(disk_image) disk = { 'image': disk_image, 'size': kwargs['disk_size'], 'format': 'qcow2', 'sparse': True, 'bus': 'virtio' } networkinterfaces = self.generate_networkinterfaces( amount=kwargs['networkinterfaces'], bridge=bridge) parameters = '--events on_reboot=restart,on_poweroff=destroy ' parameters += '--import ' parameters += '--noautoconsole ' parameters += '--autostart ' parameters += kwargs['parameters'] self.execute_virt_install( hostname=vm.hostname, vcpu=vm.cpu_cores, memory=vm.ram_amount, disk=disk, networkinterfaces=networkinterfaces, boot=boot, vnc=vm.vnc, parameters=parameters, ) self.execute_virt_install(hostname=vm.hostname, vcpu=vm.cpu_cores, memory=vm.ram_amount, disk=disk, networkinterfaces=networkinterfaces, boot=boot, vnc=vm.vnc, parameters=parameters, dry_run=False) vm.unsaved_networkinterfaces = [] for networkinterface in networkinterfaces: vm.unsaved_networkinterfaces.append(networkinterface) return True def _remove(self, vm): """Wrapper function for removing a VM (destroy domain > undefine domain).""" if not self.check_connection(): raise Exception("Host system not reachable!") self.destroy(vm) self.undefine(vm) return True def destroy(self, vm): """Destroy VM on host system. Ignore `domain is not running` error and proceed.""" stdout, stderr, exitstatus = self._execute('{} destroy {}'.format( self.VIRSH, vm.hostname)) if exitstatus != 0: stderr = ''.join(stderr) if not any(line in stderr for line in self.IGNORE_STDERR): raise Exception(stderr) return True def undefine(self, vm): """Undefine VM on host system.""" stdout, stderr, exitstatus = self._execute('{} undefine {}'.format( self.VIRSH, vm.hostname)) if exitstatus != 0: stderr = ''.join(stderr) if not any(line in stderr for line in self.IGNORE_STDERR): raise Exception(stderr) return True