def execute(self): """Execute the task.""" if not ServerConfig.objects.bool_by_key('orthos.debug.setup.execute'): logger.warning("Disabled: set 'orthos.debug.setup.execute' to 'true'") return logger.debug('Calling setup script...') try: machine = Machine.objects.get(fqdn=self.fqdn) tftp_server = machine.fqdn_domain.tftp_server if not tftp_server: logger.warning( "No TFTP server available for '{}'".format(machine.fqdn_domain.name) ) return command_template = ServerConfig.objects.by_key('setup.execute.command') context = Context({ 'architecture': machine.architecture, 'machine': machine, 'choice': self.choice }) command = Template(command_template).render(context) logger.debug("Initialize setup {}@{}: {}:{}".format( self.choice, machine.fqdn, tftp_server.fqdn, command )) tftp_server = SSH(tftp_server.fqdn) tftp_server.connect() stdout, stderr, exitstatus = tftp_server.execute(command) tftp_server.close() if exitstatus != 0: logger.warning("Creating setup configuration failed for '{}'".format(machine)) return # reboot machine finally machine.reboot() except SSH.Exception as exception: logger.error(exception) except Machine.DoesNotExist: logger.error("Machine does not exist: fqdn={}".format(self.fqdn)) except Exception as e: logger.exception(e)
def login_test(fqdn): """Check if it's possible to login via SSH.""" conn = None try: conn = SSH(fqdn) conn.connect() except Exception as e: logger.warning("SSH login failed for '{}': {}".format(fqdn, e)) return False finally: if conn: conn.close() return True
def get_installations(fqdn): """Retrieve information of the installations.""" try: machine = Machine.objects.get(fqdn=fqdn) except Machine.DoesNotExist: logger.warning("Machine '{}' does not exist".format(fqdn)) return False conn = None timer = None try: conn = SSH(fqdn) conn.connect() timer = threading.Timer(5 * 60, conn.close) timer.start() # Installations logger.debug("Collect installations...") installations = [] output, stderr, exitstatus = conn.execute_script_remote( 'machine_get_installations.sh') if output: for line in output: if line.startswith('--'): installation = Installation(machine=machine) installations.append(installation) elif line.startswith('ARCH='): installation.architecture = line.split('=')[1].strip() elif line.startswith('KERNEL='): installation.kernelversion = line.split('=')[1].strip() elif line.startswith('RUNNING='): installation.active = line.startswith('RUNNING=1') elif line.startswith('DIST='): installation.distribution = line.split('=')[1].strip() elif line.startswith('PART='): installation.partition = line.split('=')[1].strip() return installations except Exception as e: logger.exception("{} ({})".format(fqdn, e)) return False finally: if conn: conn.close() if timer: timer.cancel() return None
def abuild_test(fqdn): """Check if Autobuild is running.""" conn = None try: conn = SSH(fqdn) conn.connect() pids, stderr, exitstatus = conn.execute( r"ps -e -o pid,cmd | awk '/.*\/usr\/sbin\/autobuild.*/{print $1}'" ) if pids: return True except Exception as e: logger.warning("SSH login failed for '{}': {}".format(fqdn, e)) return False finally: if conn: conn.close() return False
def ssh_shutdown(self, user=None, reboot=False): """Power off/reboot the machine using SSH.""" from orthos2.utils.ssh import SSH if reboot: option = '--reboot' else: option = '--poweroff' machine = SSH(self.fqdn) machine.connect() command = 'shutdown {} now'.format(option) stdout, stderr, exitstatus = machine.execute(command, retry=False) machine.close() if exitstatus != 0: return False return True
class CobblerServer: def __init__(self, fqdn, domain): self._fqdn = fqdn self._conn = None self._domain = domain self._cobbler_path = ServerConfig.objects.by_key("cobbler.command") def connect(self): """Connect to DHCP server via SSH.""" if not self._conn: self._conn = SSH(self._fqdn) self._conn.connect() def close(self): """Close connection to DHCP server.""" if self._conn: self._conn.close() def deploy(self): self.connect() if not self.is_installed(): raise CobblerException("No Cobbler service found: {}".format( self._fqdn)) if not self.is_running(): raise CobblerException("Cobbler server is not running: {}".format( self._fqdn)) machines = Machine.active_machines.filter(fqdn_domain=self._domain.pk) cobbler_machines = self.get_machines() cobbler_commands = [] for machine in machines: if machine.fqdn in cobbler_machines: cobbler_commands.append( get_cobbler_update_command(machine, self._cobbler_path)) else: cobbler_commands.append( get_cobbler_add_command(machine, self._cobbler_path)) for command in cobbler_commands: # TODO: Convert this to a single ssh call (performance) _, stderr, exitcode = self._conn.execute(command) if exitcode: logger.error("failed to execute %s on %s with error %s", command, self._fqdn, stderr) self.close() def is_installed(self): """Check if Cobbler server is available.""" if self._conn.check_path(self._cobbler_path, '-x'): return True return False def is_running(self): """Check if the Cobbler daemon is running via the cobbler version command.""" command = "{} version".format(self._cobbler_path) _, _, exitstatus = self._conn.execute(command) if exitstatus == 0: return True return False def get_machines(self): stdout, stderr, exitstatus = self._conn.execute( "{cobbler} system list".format(cobbler=self._cobbler_path)) if exitstatus: logger.warning("system list failed on %s with %s", self._fqdn, stderr) raise CobblerException( "system list failed on {server}".format(server=self._fqdn)) clean_out = [system.strip(' \n\t') for system in stdout] return clean_out @staticmethod def profile_normalize(string): ''' This method replaces the second colon (:) of a string with a dash (-) This is to convert: x86_64:SLE-12-SP4-Server-LATEST:install to x86_64:SLE-12-SP4-Server-LATEST-install until cobbler returns profiles where arch:distro:profile are all separated via : ''' return string.replace(':', '-', 2).replace('-', ':', 1) def setup(self, machine: Machine, choice: str): logger.info("setup called for %s with %s on cobbler server %s ", machine.fqdn, self._fqdn, choice) cobbler_profile = "{arch}:{profile}".format(arch=machine.architecture, profile=choice) # ToDo: Revert this after renaming cobbler profiles cobbler_profile = CobblerServer.profile_normalize(cobbler_profile) command = "{cobbler} system edit --name={machine} --profile={profile} --netboot=True"\ .format(cobbler=self._cobbler_path, machine=machine.fqdn, profile=cobbler_profile) logger.debug("command for setup: %s", command) self.connect() try: stdout, stderr, exitstatus = self._conn.execute(command) if exitstatus: logger.warning("setup of %s with %s failed on %s with %s", machine.fqdn, cobbler_profile, self._fqdn, stderr) raise CobblerException( "setup of {machine} with {profile} failed on {server} with {error}" .format(machine=machine.fqdn, arch=cobbler_profile, server=self._fqdn)) except: pass finally: self.close()
def get_setup_records(self, architecture, machinegroup=None, grouped=True, delimiter=':'): """ Collect domain and architecture or machine group specific setup records. Each domain has one optional TFTP server providing available records for machine setup. If `grouped` is False, a list of all records gets returned (no grouping). Expects as return value when executing 'setup.list.command' below: ['<architecture|machinegroup:>DISTRIBUTION:FLAVOUR', ...] e.g.: [ 'x86_64:SLES12-SP3:install\n', 'x86_64:SLES12-SP3:install-ssh\n', ... 'x86_64:SLES12-SP2:install\n', 'x86_64:SLES12-SP2:rescue\n', ..., 'x86_64:local\n', ..., ] Returns (grouped is `True`): OrderedDict([ ('SLES12-SP3', [ 'install', 'install-ssh', ... ]), ('SLES12-SP2', [ 'install', 'rescue', ... ]), ('local', [ '' ]) ) Returns (grouped is `False`): [ 'SLES12-SP3:install', 'SLES12-SP3:install-ssh', ... 'SLES12-SP2:install', 'SLES12-SP2:rescue', ..., 'local', ..., ] """ from orthos2.utils.ssh import SSH def grouping(records): """Group records for HTML form.""" if not self.tftp_server: logger.warning("No TFTP server available for '{}'".format(self.name)) return {} list_command_template = ServerConfig.objects.by_key('setup.list.command') context = Context({ 'architecture': architecture, 'machinegroup': machinegroup }) list_command = Template(list_command_template).render(context) try: conn = SSH(self.tftp_server.fqdn) conn.connect() logger.debug("Fetch setup records: {}:{}".format(self.tftp_server.fqdn, list_command)) stdout, stderr, exitstatus = conn.execute(list_command) conn.close() if exitstatus != 0: logger.warning(str(stderr)) return {} logger.debug("Found {} setup records on {}".format(len(stdout), self.tftp_server.fqdn)) except Exception as e: logger.warning("Couldn't fetch record list for setup: {}".format(str(e))) return {} finally: if conn: conn.close() records = [record.strip('\n') for record in stdout] logger.debug("Records:\n{}".format(records)) if grouped: groups = {} for record in records: delim_c = record.count(delimiter) # <distro>:<profile> if delim_c == 1: (distro, profile) = record.split(delimiter) # <arch>:<distro>:<profile> elif delim_c == 2: (arch, distro, profile) = record.split(delimiter) else: logger.debug("Setup record has invalid format: '{}'".format(record)) continue if distro not in groups: groups[distro] = [] groups[distro].append(profile) records = collections.OrderedDict(sorted(groups.items())) logger.debug("Grouped and parsed:\n{}".format(records)) else: delim_c = records[0].count(delimiter) if delim_c == 1: # <distro>:<profile> pass elif delim_c == 2: # <arch>:<distro>:<profile> records = [record.split(delimiter, maxsplit=1)[1] for record in records] logger.debug("Not grouped and parsed:\n{}".format(records)) return records
def _perform(self, action): """Common implementation for on, off and reset.""" name = self.machine.hostname virsh = 'virsh -c lxc:///' conn = None result = False if not self.machine.hypervisor.fqdn: logger.error("No hypervisor system found") raise Exception("No hypervisor found") conn = SSH(self.machine.hypervisor.fqdn) conn.connect() if action == 'status': stdout, stderr, exitstatus = conn.execute('{} list --all'.format(virsh)) if exitstatus != 0: logger.error(''.join(stderr)) raise Exception(''.join(stderr)) for line in stdout[2:]: columns = line.strip().split() if columns[1] == name: return { 'running': self.Status.ON, 'shut': self.Status.OFF, 'paused': self.Status.PAUSED }.get(columns[2], 0) raise Exception("Couldn't find domain '{}'!".format(name)) elif action == 'off': stdout, stderr, exitstatus = conn.execute('{} destroy {}'.format(virsh, name)) if exitstatus == 0: logger.debug("Virtual machine '{}' stopped".format(name)) result = True else: logger.error(''.join(stderr)) raise Exception(''.join(stderr)) elif action == 'on': stdout, stderr, exitstatus = conn.execute('{} start {}'.format(virsh, name)) if exitstatus == 0: logger.debug("Virtual machine '{}' started".format(name)) result = True else: logger.error(''.join(stderr)) raise Exception(''.join(stderr)) else: logger.warning("Action '{}' does not exist".format(action)) result = False if conn: conn.close() return result
def execute(self): """ Executes the task. """ if not ServerConfig.objects.bool_by_key('orthos.debug.motd.write'): logger.warning("Disabled: set 'orthos.debug.motd.write' to 'true'") return BEGIN = '-' * 69 + ' Orthos{ --' LINE = '-' * 80 END = '-' * 69 + ' Orthos} --' try: machine = Machine.objects.get(fqdn=self.fqdn) except Machine.DoesNotExist: logger.error("Machine does not exist: fqdn={}".format(self.fqdn)) return conn = None try: conn = SSH(machine.fqdn) conn.connect() motd = conn.get_file('/etc/motd.orthos', 'w') print(BEGIN, file=motd) print( "Machine of the ARCH team. Contact <{}> for problems.".format( machine.get_support_contact()), file=motd) if machine.comment: print("INFO: " + machine.comment, file=motd) if machine.administrative: print( "This machine is an administrative machine. DON\'T TOUCH!", file=motd) if machine.reserved_by: print(LINE, file=motd) if machine.reserved_until == timezone.ZERO: print("This machine is RESERVED by {}.".format( machine.reserved_by), file=motd) else: print("This machine is RESERVED by {} until {}.".format( machine.reserved_by, machine.reserved_until), file=motd) print('', file=motd) print(wrap80(machine.reserved_reason), file=motd) print(END, file=motd) motd.close() stdout, stderr, exitstatus = conn.execute_script_remote( 'machine_sync_motd.sh') if exitstatus != 0: logger.exception("({}) {}".format(machine.fqdn, stderr)) raise Exception(stderr) except SSH.Exception as e: logger.error("({}) {}".format(machine.fqdn, e)) return False except IOError as e: logger.error("({}) {}".format(machine.fqdn, e)) return False finally: if conn: conn.close()
class CobblerServer: def __init__(self, fqdn, domain): self._fqdn = fqdn self._conn = None self._domain = domain self._cobbler_path = ServerConfig.objects.by_key("cobbler.command") @staticmethod def from_machine(machine: Machine): """ Return the cobbler server associated to a machine :param machine: Machine object which is managed by the cobbler server to fetch :returns: The corresponding cobbler server or None """ domain = machine.fqdn_domain server = domain.cobbler_server.all() if server and server[0]: return CobblerServer(server[0].fqdn, domain) return None def connect(self): """Connect to DHCP server via SSH.""" if not self._conn: self._conn = SSH(self._fqdn) self._conn.connect() def close(self): """Close connection to DHCP server.""" if self._conn: self._conn.close() def deploy(self): self.connect() if not self.is_installed(): raise CobblerException("No Cobbler service found: {}".format(self._fqdn)) if not self.is_running(): raise CobblerException("Cobbler server is not running: {}".format(self._fqdn)) machines = Machine.active_machines.filter(fqdn_domain=self._domain.pk) cobbler_machines = self.get_machines() cobbler_commands = [] for machine in machines: if machine.fqdn in cobbler_machines: cobbler_commands.append(get_cobbler_update_command(machine, self._cobbler_path)) else: cobbler_commands.append(get_cobbler_add_command(machine, self._cobbler_path)) if hasattr(machine, 'bmc') and machine.bmc: cobbler_commands.append(get_bmc_command(machine, self._cobbler_path)) for command in cobbler_commands: # TODO: Convert this to a single ssh call (performance) logger.debug("executing %s ", command) _, stderr, exitcode = self._conn.execute(command) if exitcode: logger.error("failed to execute %s on %s with error %s", command, self._fqdn, stderr) self.close() def update(self, machine: Machine): self.connect() self._check() command = get_cobbler_update_command(machine, self._cobbler_path) _, stderr, exitcode = self._conn.execute(command) if exitcode: raise CobblerException("Updating {machine} failed with {err}".format( machine.fqdn, stderr)) def remove(self, machine: Machine): #ToDo: We do not remove machines from cobbler server actively in orthos2 yet logging.warning("cobbler remove is switched off") return self.connect() if not self.is_installed(): raise CobblerException("No Cobbler service found: {}".format(self._fqdn)) if not self.is_running(): raise CobblerException("Cobbler server is not running: {}".format(self._fqdn)) command = "{cobbler} system remove --name {fqdn}".format( cobbler=self._cobbler_path, fqdn=machine.fqdn) _, stderr, exitcode = self._conn.execute(command) if(exitcode): logging.error("Removing %s failed with '%s'", machine.fqdn, stderr) def is_installed(self): """Check if Cobbler server is available.""" if self._conn.check_path(self._cobbler_path, '-x'): return True return False def is_running(self): """Check if the Cobbler daemon is running via the cobbler version command.""" command = "{} version".format(self._cobbler_path) _, _, exitstatus = self._conn.execute(command) if exitstatus == 0: return True return False def get_machines(self): stdout, stderr, exitstatus = self._conn.execute( "{cobbler} system list".format(cobbler=self._cobbler_path)) if exitstatus: logger.warning("system list failed on %s with %s", self._fqdn, stderr) raise CobblerException("system list failed on {server}".format(server=self._fqdn)) clean_out = [system.strip(' \n\t') for system in stdout] return clean_out def setup(self, machine: Machine, choice: str): logger.info("setup called for %s with %s on cobbler server %s ", machine.fqdn, self._fqdn, choice) if choice: cobbler_profile = "{arch}:{profile}".format(arch=machine.architecture, profile=choice) else: cobbler_profile = get_default_profile(machine) command = "{cobbler} system edit --name={machine} --profile={profile} --netboot=True"\ .format(cobbler=self._cobbler_path, machine=machine.fqdn, profile=cobbler_profile) logger.debug("command for setup: %s", command) self.connect() try: stdout, stderr, exitstatus = self._conn.execute(command) if exitstatus: logger.warning("setup of %s with %s failed on %s with %s", machine.fqdn, cobbler_profile, self._fqdn, stderr) raise CobblerException( "setup of {machine} with {profile} failed on {server} with {error}".format( machine=machine.fqdn, arch=cobbler_profile, server=self._fqdn)) except: pass finally: self.close() def powerswitch(self, machine: Machine, action: str): logger.debug("powerswitching of %s called with action %s", machine.fqdn, action) self.connect() cobbler_action = "" if action == "reboot": cobbler_action = "reboot" else: cobbler_action = "power" + action command = "{cobbler} system {action} --name {fqdn}".format(cobbler=self._cobbler_path, action=cobbler_action, fqdn=machine.fqdn) out, stderr, exitcode = self._conn.execute(command) if exitcode: logger.warning("Powerswitching of %s with %s failed on %s with %s", machine.fqdn, command, self._fqdn, stderr) raise CobblerException( "Powerswitching of {machine} with {command} failed on {server} with {error}".format( machine=machine.fqdn, command=command, server=self._fqdn)) return out def _check(self): if not self.is_installed(): raise CobblerException("No Cobbler service found: {}".format(self._fqdn)) if not self.is_running(): raise CobblerException("Cobbler server is not running: {}".format(self._fqdn))
class CobblerServer: def __init__(self, fqdn, domain): self._fqdn = fqdn self._conn = None self._domain = domain self._cobbler_path = ServerConfig.objects.by_key("cobbler.command") def connect(self): """Connect to DHCP server via SSH.""" if not self._conn: self._conn = SSH(self._fqdn) self._conn.connect() def close(self): """Close connection to DHCP server.""" if self._conn: self._conn.close() def deploy(self): self.connect() if not self.is_installed(): raise CobblerException("No Cobbler service found: {}".format(self._fqdn)) if not self.is_running(): raise CobblerException("Cobbler server is not running: {}".format(self._fqdn)) machines = Machine.active_machines.filter(fqdn_domain=self._domain.pk) cobbler_machines = self.get_machines() cobbler_commands = [] for machine in machines: if machine.fqdn in cobbler_machines: cobbler_commands.append(get_cobbler_update_command(machine, self._cobbler_path)) else: cobbler_commands.append(get_cobbler_add_command(machine, self._cobbler_path)) for command in cobbler_commands: # TODO: Convert this to a single ssh call (performance) _, stderr, exitcode = self._conn.execute(command) if exitcode: logger.error("failed to execute %s on %s with error %s", command, self._fqdn, stderr) self.close() def is_installed(self): """Check if Cobbler server is available.""" if self._conn.check_path(self._cobbler_path, '-x'): return True return False def is_running(self): """Check if the Cobbler daemon is running via the cobbler version command.""" command = "{} version".format(self._cobbler_path) _, _, exitstatus = self._conn.execute(command) if exitstatus == 0: return True return False def get_machines(self): stdout, stderr, exitstatus = self._conn.execute( "{cobbler} system list".format(cobbler=self._cobbler_path)) if exitstatus: logger.warning("system list failed on %s with %s", self._fqdn, stderr) raise CobblerException("system list failed on {server}".format(server=self._fqdn)) clean_out = [system.strip(' \n\t') for system in stdout] return clean_out
def execute(self): """Execute the task.""" from orthos2.data.models import Machine, SerialConsole if not ServerConfig.objects.bool_by_key( 'orthos.debug.serialconsole.write'): logger.warning( "Disabled: set 'orthos.debug.serialconsole.write' to 'true'") return try: cscreen_server = Machine.objects.get(fqdn=self.fqdn) except Machine.DoesNotExist: logger.warning("Serial console server does not exist: {}".format( self.fqdn)) conn = None try: conn = SSH(cscreen_server.fqdn) conn.connect() stdout, stderr, exitstatus = conn.execute( 'sudo touch /etc/cscreenrc_allow_update') if exitstatus != 0: raise Exception( "Couldn't lock cscreen ('touch /etc/cscreenrc_allow_update')" ) new_content = '' for serialconsole in SerialConsole.cscreen.get(cscreen_server): new_content += serialconsole.get_comment_record() + '\n' new_content += serialconsole.get_command_record() + '\n' screenrc_file = '/etc/cscreenrc' # create `/etc/cscreenrc` if it doesn't exist stdout, stderr, exitstatus = conn.execute( '[ -e "{}"]'.format(screenrc_file)) orthos_inline_begin = ServerConfig.objects.by_key( 'orthos.configuration.inline.begin') orthos_inline_end = ServerConfig.objects.by_key( 'orthos.configuration.inline.end') if exitstatus != 0: stdout, stderr, exitstatus = conn.execute( 'echo "{}\n{}" > {}'.format(orthos_inline_begin, screenrc_file, orthos_inline_end)) if exitstatus != 0: raise Exception("Couldn't create CScreen file ('{}')".format( screenrc_file)) # Save backup file which is used later by an invoked script # to determine the changes and update the running screen # session (add, remove or restart modified entries). stdout, stderr, exitstatus = conn.execute( 'sudo cp {} {}.old'.format(screenrc_file, screenrc_file)) cscreen = conn.get_file(screenrc_file, 'r') buffer = '' in_replace = False for line in cscreen.readlines(): if not in_replace and line.startswith(orthos_inline_begin): buffer += line + new_content in_replace = True elif in_replace and line.startswith(orthos_inline_end): buffer += line in_replace = False elif not in_replace: buffer += line cscreen.close() cscreen = conn.get_file(screenrc_file, 'w') buffer = buffer.strip('\n') print(buffer, file=cscreen) cscreen.close() stdout, stderr, exitstatus = conn.execute( 'sudo /usr/bin/cscreen -u') logger.info("CScreen update exited with: {}".format(exitstatus)) stdout, stderr, exitstatus = conn.execute( 'sudo rm -f /etc/cscreenrc_allow_update') if exitstatus != 0: raise Exception( "Couldn't unlock CScreen ('rm /etc/cscreenrc_allow_update')" ) except SSH.Exception as exception: logger.error(exception) except IOError as exception: logger.error(exception) finally: if conn: conn.close()
def get_hardware_information(fqdn): """Retrieve information of the system.""" try: machine = Machine.objects.get(fqdn=fqdn) except Machine.DoesNotExist: logger.warning("Machine '{}' does not exist".format(fqdn)) return # set needed values for several checks from original machine machine_ = Machine( architecture=machine.architecture ) conn = None timer = None try: conn = SSH(fqdn) conn.connect() timer = threading.Timer(5 * 60, conn.close) timer.start() # CPUs logger.debug("Get CPU number...") output, stderr, exitstatus = conn.execute_script_remote('machine_get_cpu_number.sh') if output: for line in output: if line.startswith('SOCKETS'): machine_.cpu_physical = int(line.split('=')[1]) elif line.startswith('CORES'): machine_.cpu_cores = int(line.split('=')[1]) elif line.startswith('THREADS'): machine_.cpu_threads = int(line.split('=')[1]) logger.debug("Get CPU type...") output, stderr, exitstatus = conn.execute_script_remote('machine_get_cpu_type.sh') if output and output[0]: machine_.cpu_model = output[0].strip() logger.debug("Get CPU flags...") output, stderr, exitstatus = conn.execute_script_remote('machine_get_cpu_flags.sh') if output and output[0]: machine_.cpu_flags = output[0].strip() logger.debug("Get CPU speed...") output, stderr, exitstatus = conn.execute_script_remote('machine_get_cpu_speed.sh') if output and output[0]: machine_.cpu_speed = Decimal(int(output[0].strip()) / 1000000) logger.debug("Get CPU ID...") output, stderr, exitstatus = conn.execute_script_remote('machine_get_cpu_id.sh') if output and output[0]: machine_.cpu_id = output[0].strip() # EFI logger.debug("Check for EFI...") try: efi_file = conn.get_file('/sys/firmware/efi', 'r') efi_file.close() machine_.efi = True except IOError: machine_.efi = False # Memory logger.debug("Get RAM amount...") for line in conn.read_file('/proc/meminfo'): if line.startswith('MemTotal'): machine_.ram_amount = int(int(line.split()[1]) / 1024) # Virtualization capability VM_HOST_MIN_RAM_MB = 7000 machine_.vm_capable = False # Virtualization: x86 logger.debug("Check for VM capability...") if machine_.architecture_id == Architecture.Type.X86_64: cpu_flags = machine_.cpu_flags if cpu_flags: cpu_flags = cpu_flags.upper() if ((cpu_flags.find('VMX') >= 0 or cpu_flags.find('SVM') >= 0) and int(machine_.ram_amount) > VM_HOST_MIN_RAM_MB): machine_.vm_capable = True # Virtualization: ppc64le if machine_.architecture_id == Architecture.Type.PPC64LE: for line in conn.read_file('/proc/cpuinfo'): if line.startswith('firmware') and 'OPAL' in line: machine_.vm_capable = True # Disk logger.debug("Get disk information...") stdout, stderr, exitstatus = conn.execute('hwinfo --disk') for line in stdout: line = line.strip() if line.startswith('Size:'): machine_.disk_primary_size = int(int(line.split()[1]) / 2 / 1024 ** 2) elif line.startswith('Attached to:'): opening_bracket = line.find('(') closing_bracket = line.find(')') if opening_bracket > 0 and closing_bracket > 0: machine_.disk_type = line[opening_bracket + 1:closing_bracket] else: machine_.disk_type = 'Unknown disk type' break # lsmod logger.debug("Get 'lsmod'...") stdout, stderr, exitstatus = conn.execute('lsmod') machine_.lsmod = normalize_ascii("".join(stdout)) # lspci logger.debug("Get 'lspci'...") stdout, stderr, exitstatus = conn.execute('lspci -vvv -nn') machine_.lspci = normalize_ascii("".join(stdout)) # last logger.debug("Get 'last'...") output, stderr, exitstatus = conn.execute('last | grep -v reboot | head -n 1') string = ''.join(output) result = string[0:8] + string[38:49] machine_.last = normalize_ascii("".join(result)) # hwinfo logger.debug("Get 'hwinfo' (full)...") stdout, stderr, exitstatus = conn.execute( 'hwinfo --bios ' + '--block --bridge --cdrom --cpu --disk --floppy --framebuffer ' + '--gfxcard --hub --ide --isapnp --isdn --keyboard --memory ' + '--monitor --mouse --netcard --network --partition --pci --pcmcia ' + '--scsi --smp --sound --sys --tape --tv --usb --usb-ctrl --wlan' ) machine_.hwinfo = normalize_ascii("".join(stdout)) # dmidecode logger.debug("Get 'dmidecode'...") stdout, stderr, exitstatus = conn.execute('dmidecode') machine_.dmidecode = normalize_ascii("".join(stdout)) # dmesg logger.debug("Get 'dmesg'...") stdout, stderr, exitstatus = conn.execute( 'if [ -e /var/log/boot.msg ]; then ' + 'cat /var/log/boot.msg; else journalctl -xl | head -n200; ' + 'fi' ) machine_.dmesg = normalize_ascii("".join(stdout)) # lsscsi logger.debug("Get 'lsscsi'...") stdout, stderr, exitstatus = conn.execute('lsscsi -s') machine_.lsscsi = normalize_ascii("".join(stdout)) # lsusb logger.debug("Get 'lsusb'...") stdout, stderr, exitstatus = conn.execute('lsusb') machine_.lsusb = normalize_ascii("".join(stdout)) # IPMI logger.debug("Check for IPMI...") machine_.ipmi = machine_.dmidecode.find('IPMI') >= 0 # Firmware script logger.debug("Get BIOS version...") output, stderr, exitstatus = conn.execute_script_remote('machine_get_firmware.sh') if output and output[0]: machine_.bios_version = output[0].strip() return machine_ except Exception as e: logger.error("{} ({})".format(fqdn, e)) return False finally: if conn: conn.close() if timer: timer.cancel() return None
def get_pci_devices(fqdn): """Retrieve all PCI devices.""" def get_pci_device_by_slot(pci_devices, slot): """Return the PCI device by slot.""" for dev in pci_devices: pci_slot = dev.slot if not pci_slot: continue pci_slot = pci_slot.strip() # pci domain hacks if len(pci_slot) < 8: pci_slot = '0000:' + pci_slot if len(slot) < 8: slot = '0000:' + slot if pci_slot == slot: return dev return None from orthos2.data.models import PCIDevice try: machine = Machine.objects.get(fqdn=fqdn) except Machine.DoesNotExist: logger.warning("Machine '{}' does not exist".format(fqdn)) return False conn = None timer = None try: conn = SSH(fqdn) conn.connect() timer = threading.Timer(5 * 60, conn.close) timer.start() logger.debug("Collect PCI devices for '{}'...".format(machine.fqdn)) pci_devices = [] chunk = '' stdout, stderr, exitstatus = conn.execute('lspci -mmvn') for line in stdout: if line.strip(): chunk += line else: pci_devices.append(PCIDevice.from_lspci_mmnv(chunk)) chunk = '' # drivers for PCI devices from hwinfo in_pci_device = False current_busid = None if machine.hwinfo: for line in machine.hwinfo.splitlines(): if re.match(r'^\d+: PCI', line): in_pci_device = True continue if not line.strip(): in_pci_device = False current_busid = None continue if not in_pci_device: continue match = re.match(r' SysFS BusID: ([0-9a-fA-F.:]+)', line) if match: current_busid = match.group(1) match = re.match(r' Driver: "([^"]*)"', line) if match and current_busid: pcidev = get_pci_device_by_slot(pci_devices, current_busid) if pcidev: pcidev.drivermodule = match.group(1) match = re.match(r' Driver Modules: "([^"]*)"', line) if match and current_busid: pcidev = get_pci_device_by_slot(pci_devices, current_busid) if pcidev: pcidev.drivermodule = match.group(1) for pci_device in pci_devices: pci_device.machine = machine logger.debug("Collected {} PCI devices for '{}'".format(len(pci_devices), machine.fqdn)) return pci_devices except Exception as e: logger.exception("{} ({})".format(fqdn, e)) return False finally: if conn: conn.close() if timer: timer.cancel()
def get_status_ip(fqdn): """Retrieve information of the systems IPv4/IPv6 status.""" try: machine = Machine.objects.get(fqdn=fqdn) except Machine.DoesNotExist: logger.warning("Machine '{}' does not exist".format(fqdn)) return False machine_ = Machine() conn = None timer = None try: conn = SSH(machine.fqdn) conn.connect() timer = threading.Timer(5 * 60, conn.close) timer.start() logger.debug("Check IPv4/IPv6 status...") stdout, stderr, exitstatus = conn.execute('/sbin/ip a') devices = {} current_device = None addresses = {'inet': [], 'inet6': []} for line in stdout: match = re.match(r'^\d+:\s+([a-zA-Z0-9]+):\s+<.*>\s(.*)\n', line) if match: current_device = match.group(1) devices[current_device] = { 'mac_address': None, 'inet': None, 'inet6': None, 'flags': None } devices[current_device]['flags'] = match.group(2).split() continue line = line.lstrip() match = re.match(r'inet ([0-9.]{7,15})\/.*scope', line) if match: if devices[current_device]['inet'] is None: devices[current_device]['inet'] = [] devices[current_device]['inet'].append(match.group(1)) continue match = re.match(r'inet6 ([a-f0-9:]*)\/[0-9]+ scope', line) if match: if devices[current_device]['inet6'] is None: devices[current_device]['inet6'] = [] devices[current_device]['inet6'].append(match.group(1)) continue match = re.match('link/ether ([a-f0-9:]{17}) brd', line) if match: devices[current_device]['mac_address'] = match.group(1).upper() for device, values in devices.items(): if values['mac_address'] is None: continue # ignore device if hooking up another if any(device in values['flags'] for device in devices.keys()): continue if values['mac_address'] == machine.mac_address: if values['inet'] is None: machine_.status_ipv4 = Machine.StatusIP.AF_DISABLED elif machine.ipv4 not in values['inet']: machine_.status_ipv4 = Machine.StatusIP.NO_ADDRESS if [ipv4 for ipv4 in values['inet'] if not ipv4.startswith('127.0.0.1')]: machine_.status_ipv4 = Machine.StatusIP.ADDRESS_MISMATCH elif machine.ipv4 in values['inet']: machine_.status_ipv4 = Machine.StatusIP.CONFIRMED else: machine_.status_ipv4 = Machine.StatusIP.MISSING if values['inet6'] is None: machine_.status_ipv6 = Machine.StatusIP.AF_DISABLED elif machine.ipv6 not in values['inet6']: machine_.status_ipv6 = Machine.StatusIP.NO_ADDRESS if [ipv6 for ipv6 in values['inet6'] if not ipv6.startswith('fe80::')]: machine_.status_ipv6 = Machine.StatusIP.ADDRESS_MISMATCH elif machine.ipv6 in values['inet6']: machine_.status_ipv6 = Machine.StatusIP.CONFIRMED addresses['inet'].append(values['inet']) addresses['inet6'].append(values['inet6']) if machine_.status_ipv4 == Machine.StatusIP.NO_ADDRESS: if machine.ipv4 in addresses['inet']: machine_.status_ipv4 = Machine.StatusIP.MAC_MISMATCH if machine_.status_ipv6 == Machine.StatusIP.NO_ADDRESS: if machine.ipv6 in addresses['inet6']: machine_.status_ipv6 = Machine.StatusIP.MAC_MISMATCH return machine_ except Exception as e: logger.error("{} ({})".format(fqdn, e)) return False finally: if conn: conn.close() if timer: timer.cancel() return None
def get_networkinterfaces(fqdn): """Retrieve information of the systems network interfaces.""" try: machine = Machine.objects.get(fqdn=fqdn) except Machine.DoesNotExist: logger.warning("Machine '{}' does not exist".format(fqdn)) return False conn = None timer = None try: conn = SSH(machine.fqdn) conn.connect() timer = threading.Timer(5 * 60, conn.close) timer.start() # Network interfaces logger.debug("Collect network interfaces...") stdout, stderr, exitstatus = conn.execute('hwinfo --network') interfaces = [] interface = None for line in stdout: if line and line[0] != ' ' and line[0] != '\t': if interface and interface.mac_address and\ interface.driver_module not in {'bridge', 'tun'}: interfaces.append(interface) interface = NetworkInterface() else: match = re.match(r'\s+Driver: "(\w+)"', line) if match: interface.driver_module = match.group(1) continue match = re.match(r'\s+SysFS ID: ([/\w.]+)', line) if match: interface.sysfs = match.group(1) continue match = re.match(r'\s+HW Address: ([0-9a-fA-F:]+)', line) if match: interface.mac_address = match.group(1).upper() continue match = re.match(r'\s+Device File: ([\w.]+)', line) if match: interface.name = match.group(1) continue if interface and interface.mac_address and\ interface.driver_module not in {'bridge', 'tun'}: interfaces.append(interface) for interface in interfaces: if interface.sysfs is None: continue path = '/sys/{}/type'.format(interface.sysfs) arp_type = ''.join(conn.read_file(path)) if arp_type == ARPHRD_IEEE80211: continue stdout, stderr, exitstatus = conn.execute('ethtool {}'.format(interface.name)) for line in stdout: match = re.match(r'\s+Port: (.+)', line) if match: interface.ethernet_type = match.group(1) return interfaces except Exception as e: logger.error("{} ({})".format(fqdn, e)) return False finally: if conn: conn.close() if timer: timer.cancel() return None
def get_setup_records(self, architecture, machinegroup=None, grouped=True, delimiter=':'): """ Collect domain and architecture or machine group specific setup records. Each domain has one optional TFTP server providing available records for machine setup. If `grouped` is False, a list of all records gets returned (no grouping). Expects stdout: ['DISTRIBUTION-<architecture|machinegroup>-FLAVOUR', ...] [ 'SLES12-SP3-x86_64-install\n', 'SLES12-SP3-x86_64-install-ssh\n', ... 'SLES12-SP2-x86_64-install\n', 'SLES12-SP2-x86_64-rescue\n', ..., 'local-x86_64\n', ..., ] Returns (grouped is `True`): OrderedDict([ ('SLES12-SP3', [ 'SLES12-SP3-x86_64-install', 'SLES12-SP3-x86_64-install-ssh', ... ]), ('SLES12-SP2', [ 'SLES12-SP2-x86_64-install', 'SLES12-SP2-x86_64-rescue', ... ]), ('local', [ 'local-x86_64' ]) ) Returns (grouped is `False`): [ 'SLES12-SP3-x86_64-install', 'SLES12-SP3-x86_64-install-ssh', ... 'SLES12-SP2-x86_64-install', 'SLES12-SP2-x86_64-rescue', ..., 'local-x86_64', ..., ] """ from orthos2.utils.ssh import SSH def grouping(records): """Group records for HTML form.""" groups = {} for record in records: record_ = record.split(delimiter) if len(record_) == 2: prefix = record_[0] suffix = record_[1] else: logger.debug( "Setup record has invalid format: '{}'".format(record)) continue if prefix not in groups: groups[prefix] = [] groups[prefix].append(suffix) return collections.OrderedDict(sorted(groups.items())) if not self.tftp_server: logger.warning("No TFTP server available for '{}'".format( self.name)) return {} list_command_template = ServerConfig.objects.by_key( 'setup.list.command') context = Context({ 'architecture': architecture, 'machinegroup': machinegroup }) list_command = Template(list_command_template).render(context) try: conn = SSH(self.tftp_server.fqdn) conn.connect() logger.debug("Fetch setup records: {}:{}".format( self.tftp_server.fqdn, list_command)) stdout, stderr, exitstatus = conn.execute(list_command) conn.close() if exitstatus != 0: logger.warning(str(stderr)) return {} logger.debug("Found {} setup records on {}".format( len(stdout), self.tftp_server.fqdn)) except Exception as e: logger.warning("Couldn't fetch record list for setup: {}".format( str(e))) return {} finally: if conn: conn.close() records = list(map(lambda record: record.strip('\n'), stdout)) if grouped: records = grouping(records) else: records = list( map(lambda record: record.split(delimiter)[1], records)) return records
def execute(self): """Execute the task.""" from orthos2.data.models import Machine, SerialConsole if not ServerConfig.objects.bool_by_key( 'orthos.debug.serialconsole.write'): logger.warning( "Disabled: set 'orthos.debug.serialconsole.write' to 'true'") return try: cscreen_server = Machine.objects.get(fqdn=self.fqdn) except Machine.DoesNotExist: logger.warning("Serial console server does not exist: {}".format( self.fqdn)) conn = None try: conn = SSH(cscreen_server.fqdn) conn.connect(user='******') stdout, stderr, exitstatus = conn.execute( 'touch /dev/shm/.cscreenrc_allow_update') if exitstatus != 0: raise Exception( "Couldn't lock cscreen ('touch /dev/shm/.cscreenrc_allow_update')" ) new_content = '' for serialconsole in SerialConsole.cscreen.get(cscreen_server): new_content += serialconsole.get_comment_record() + '\n' new_content += serialconsole.get_command_record() + '\n' screenrc_file = '/etc/cscreenrc' orthos_inline_begin = ServerConfig.objects.by_key( 'orthos.configuration.inline.begin') orthos_inline_end = ServerConfig.objects.by_key( 'orthos.configuration.inline.end') buffer = '' file_found = True try: cscreen = conn.get_file(screenrc_file, 'r') in_replace = False marker_found = False for line in cscreen.readlines(): if not in_replace and line.startswith(orthos_inline_begin): buffer += line + new_content in_replace = True marker_found = True elif in_replace and line.startswith(orthos_inline_end): buffer += line in_replace = False elif not in_replace: buffer += line # orthos start marker was not found... Add it. if marker_found is False: logging.info("CSCREEN: Orthos marker not found, adding...") buffer += orthos_inline_begin + '\n' + new_content + orthos_inline_end cscreen.close() except IOError as e: _errno, _strerror = e.args import errno if _errno == errno.ENOENT: file_found = False logging.warning("{}:{} not found - creating...".format( cscreen_server.fqdn, screenrc_file)) else: raise (e) # Create an empty file with just markers, this will get the .old file # to diff against for new entries via cscreen -u if not file_found: buffer = orthos_inline_begin + '\n' + orthos_inline_end cscreen = conn.get_file(screenrc_file, 'w') buffer = buffer.strip('\n') print(buffer, file=cscreen) cscreen.close() buffer = orthos_inline_begin + '\n' + new_content + orthos_inline_end # Save backup file which is used later by an invoked script # to determine the changes and update the running screen # session (add, remove or restart modified entries). stdout, stderr, exitstatus = conn.execute('cp {} {}.old'.format( screenrc_file, screenrc_file)) cscreen = conn.get_file(screenrc_file, 'w') buffer = buffer.strip('\n') print(buffer, file=cscreen) cscreen.close() stdout, stderr, exitstatus = conn.execute('/usr/bin/cscreen -u') if exitstatus != 0: logger.warning(stderr) stdout, stderr, exitstatus = conn.execute( 'rm -f /dev/shm/.cscreenrc_allow_update') if exitstatus != 0: raise Exception( "Couldn't unlock CScreen ('rm /dev/shm/.cscreenrc_allow_update)" ) logger.info("CScreen update for %s finished", cscreen_server.fqdn) except SSH.Exception as exception: logger.exception(exception) except IOError as exception: logger.exception(exception) finally: if conn: conn.close()