def execute(self):
"""Execute the task."""
if not ServerConfig.objects.bool_by_key('orthos.debug.setup.execute'):
logger.warning("Disabled: set 'orthos.debug.setup.execute' to 'true'")
return
logger.debug('Calling setup script...')
try:
machine = Machine.objects.get(fqdn=self.fqdn)
tftp_server = machine.fqdn_domain.tftp_server
if not tftp_server:
logger.warning(
"No TFTP server available for '{}'".format(machine.fqdn_domain.name)
)
return
command_template = ServerConfig.objects.by_key('setup.execute.command')
context = Context({
'architecture': machine.architecture,
'machine': machine,
'choice': self.choice
})
command = Template(command_template).render(context)
logger.debug("Initialize setup {}@{}: {}:{}".format(
self.choice,
machine.fqdn,
tftp_server.fqdn,
command
))
tftp_server = SSH(tftp_server.fqdn)
tftp_server.connect()
stdout, stderr, exitstatus = tftp_server.execute(command)
tftp_server.close()
if exitstatus != 0:
logger.warning("Creating setup configuration failed for '{}'".format(machine))
return
# reboot machine finally
machine.reboot()
except SSH.Exception as exception:
logger.error(exception)
except Machine.DoesNotExist:
logger.error("Machine does not exist: fqdn={}".format(self.fqdn))
except Exception as e:
logger.exception(e)
def login_test(fqdn):
"""Check if it's possible to login via SSH."""
conn = None
try:
conn = SSH(fqdn)
conn.connect()
except Exception as e:
logger.warning("SSH login failed for '{}': {}".format(fqdn, e))
return False
finally:
if conn:
conn.close()
return True
def get_installations(fqdn):
"""Retrieve information of the installations."""
try:
machine = Machine.objects.get(fqdn=fqdn)
except Machine.DoesNotExist:
logger.warning("Machine '{}' does not exist".format(fqdn))
return False
conn = None
timer = None
try:
conn = SSH(fqdn)
conn.connect()
timer = threading.Timer(5 * 60, conn.close)
timer.start()
# Installations
logger.debug("Collect installations...")
installations = []
output, stderr, exitstatus = conn.execute_script_remote(
'machine_get_installations.sh')
if output:
for line in output:
if line.startswith('--'):
installation = Installation(machine=machine)
installations.append(installation)
elif line.startswith('ARCH='):
installation.architecture = line.split('=')[1].strip()
elif line.startswith('KERNEL='):
installation.kernelversion = line.split('=')[1].strip()
elif line.startswith('RUNNING='):
installation.active = line.startswith('RUNNING=1')
elif line.startswith('DIST='):
installation.distribution = line.split('=')[1].strip()
elif line.startswith('PART='):
installation.partition = line.split('=')[1].strip()
return installations
except Exception as e:
logger.exception("{} ({})".format(fqdn, e))
return False
finally:
if conn:
conn.close()
if timer:
timer.cancel()
return None
def abuild_test(fqdn):
"""Check if Autobuild is running."""
conn = None
try:
conn = SSH(fqdn)
conn.connect()
pids, stderr, exitstatus = conn.execute(
r"ps -e -o pid,cmd | awk '/.*\/usr\/sbin\/autobuild.*/{print $1}'"
)
if pids:
return True
except Exception as e:
logger.warning("SSH login failed for '{}': {}".format(fqdn, e))
return False
finally:
if conn:
conn.close()
return False
def ssh_shutdown(self, user=None, reboot=False):
"""Power off/reboot the machine using SSH."""
from orthos2.utils.ssh import SSH
if reboot:
option = '--reboot'
else:
option = '--poweroff'
machine = SSH(self.fqdn)
machine.connect()
command = 'shutdown {} now'.format(option)
stdout, stderr, exitstatus = machine.execute(command, retry=False)
machine.close()
if exitstatus != 0:
return False
return True
class CobblerServer:
def __init__(self, fqdn, domain):
self._fqdn = fqdn
self._conn = None
self._domain = domain
self._cobbler_path = ServerConfig.objects.by_key("cobbler.command")
def connect(self):
"""Connect to DHCP server via SSH."""
if not self._conn:
self._conn = SSH(self._fqdn)
self._conn.connect()
def close(self):
"""Close connection to DHCP server."""
if self._conn:
self._conn.close()
def deploy(self):
self.connect()
if not self.is_installed():
raise CobblerException("No Cobbler service found: {}".format(
self._fqdn))
if not self.is_running():
raise CobblerException("Cobbler server is not running: {}".format(
self._fqdn))
machines = Machine.active_machines.filter(fqdn_domain=self._domain.pk)
cobbler_machines = self.get_machines()
cobbler_commands = []
for machine in machines:
if machine.fqdn in cobbler_machines:
cobbler_commands.append(
get_cobbler_update_command(machine, self._cobbler_path))
else:
cobbler_commands.append(
get_cobbler_add_command(machine, self._cobbler_path))
for command in cobbler_commands: # TODO: Convert this to a single ssh call (performance)
_, stderr, exitcode = self._conn.execute(command)
if exitcode:
logger.error("failed to execute %s on %s with error %s",
command, self._fqdn, stderr)
self.close()
def is_installed(self):
"""Check if Cobbler server is available."""
if self._conn.check_path(self._cobbler_path, '-x'):
return True
return False
def is_running(self):
"""Check if the Cobbler daemon is running via the cobbler version command."""
command = "{} version".format(self._cobbler_path)
_, _, exitstatus = self._conn.execute(command)
if exitstatus == 0:
return True
return False
def get_machines(self):
stdout, stderr, exitstatus = self._conn.execute(
"{cobbler} system list".format(cobbler=self._cobbler_path))
if exitstatus:
logger.warning("system list failed on %s with %s", self._fqdn,
stderr)
raise CobblerException(
"system list failed on {server}".format(server=self._fqdn))
clean_out = [system.strip(' \n\t') for system in stdout]
return clean_out
@staticmethod
def profile_normalize(string):
'''
This method replaces the second colon (:) of a string with a dash (-)
This is to convert:
x86_64:SLE-12-SP4-Server-LATEST:install
to
x86_64:SLE-12-SP4-Server-LATEST-install
until cobbler returns profiles where arch:distro:profile are all separated via :
'''
return string.replace(':', '-', 2).replace('-', ':', 1)
def setup(self, machine: Machine, choice: str):
logger.info("setup called for %s with %s on cobbler server %s ",
machine.fqdn, self._fqdn, choice)
cobbler_profile = "{arch}:{profile}".format(arch=machine.architecture,
profile=choice)
# ToDo: Revert this after renaming cobbler profiles
cobbler_profile = CobblerServer.profile_normalize(cobbler_profile)
command = "{cobbler} system edit --name={machine} --profile={profile} --netboot=True"\
.format(cobbler=self._cobbler_path, machine=machine.fqdn, profile=cobbler_profile)
logger.debug("command for setup: %s", command)
self.connect()
try:
stdout, stderr, exitstatus = self._conn.execute(command)
if exitstatus:
logger.warning("setup of %s with %s failed on %s with %s",
machine.fqdn, cobbler_profile, self._fqdn,
stderr)
raise CobblerException(
"setup of {machine} with {profile} failed on {server} with {error}"
.format(machine=machine.fqdn,
arch=cobbler_profile,
server=self._fqdn))
except:
pass
finally:
self.close()
def get_setup_records(self, architecture, machinegroup=None, grouped=True, delimiter=':'):
"""
Collect domain and architecture or machine group specific setup records.
Each domain has one optional TFTP server providing available records for machine setup.
If `grouped` is False, a list of all records gets returned (no grouping).
Expects as return value when executing 'setup.list.command' below:
['<architecture|machinegroup:>DISTRIBUTION:FLAVOUR', ...]
e.g.:
[
'x86_64:SLES12-SP3:install\n',
'x86_64:SLES12-SP3:install-ssh\n',
...
'x86_64:SLES12-SP2:install\n',
'x86_64:SLES12-SP2:rescue\n',
...,
'x86_64:local\n',
...,
]
Returns (grouped is `True`):
OrderedDict([
('SLES12-SP3', [
'install',
'install-ssh',
...
]),
('SLES12-SP2', [
'install',
'rescue',
...
]),
('local', [
''
])
)
Returns (grouped is `False`):
[
'SLES12-SP3:install',
'SLES12-SP3:install-ssh',
...
'SLES12-SP2:install',
'SLES12-SP2:rescue',
...,
'local',
...,
]
"""
from orthos2.utils.ssh import SSH
def grouping(records):
"""Group records for HTML form."""
if not self.tftp_server:
logger.warning("No TFTP server available for '{}'".format(self.name))
return {}
list_command_template = ServerConfig.objects.by_key('setup.list.command')
context = Context({
'architecture': architecture,
'machinegroup': machinegroup
})
list_command = Template(list_command_template).render(context)
try:
conn = SSH(self.tftp_server.fqdn)
conn.connect()
logger.debug("Fetch setup records: {}:{}".format(self.tftp_server.fqdn, list_command))
stdout, stderr, exitstatus = conn.execute(list_command)
conn.close()
if exitstatus != 0:
logger.warning(str(stderr))
return {}
logger.debug("Found {} setup records on {}".format(len(stdout), self.tftp_server.fqdn))
except Exception as e:
logger.warning("Couldn't fetch record list for setup: {}".format(str(e)))
return {}
finally:
if conn:
conn.close()
records = [record.strip('\n') for record in stdout]
logger.debug("Records:\n{}".format(records))
if grouped:
groups = {}
for record in records:
delim_c = record.count(delimiter)
# <distro>:<profile>
if delim_c == 1:
(distro, profile) = record.split(delimiter)
# <arch>:<distro>:<profile>
elif delim_c == 2:
(arch, distro, profile) = record.split(delimiter)
else:
logger.debug("Setup record has invalid format: '{}'".format(record))
continue
if distro not in groups:
groups[distro] = []
groups[distro].append(profile)
records = collections.OrderedDict(sorted(groups.items()))
logger.debug("Grouped and parsed:\n{}".format(records))
else:
delim_c = records[0].count(delimiter)
if delim_c == 1:
# <distro>:<profile>
pass
elif delim_c == 2:
# <arch>:<distro>:<profile>
records = [record.split(delimiter, maxsplit=1)[1] for record in records]
logger.debug("Not grouped and parsed:\n{}".format(records))
return records
def _perform(self, action):
"""Common implementation for on, off and reset."""
name = self.machine.hostname
virsh = 'virsh -c lxc:///'
conn = None
result = False
if not self.machine.hypervisor.fqdn:
logger.error("No hypervisor system found")
raise Exception("No hypervisor found")
conn = SSH(self.machine.hypervisor.fqdn)
conn.connect()
if action == 'status':
stdout, stderr, exitstatus = conn.execute('{} list --all'.format(virsh))
if exitstatus != 0:
logger.error(''.join(stderr))
raise Exception(''.join(stderr))
for line in stdout[2:]:
columns = line.strip().split()
if columns[1] == name:
return {
'running': self.Status.ON,
'shut': self.Status.OFF,
'paused': self.Status.PAUSED
}.get(columns[2], 0)
raise Exception("Couldn't find domain '{}'!".format(name))
elif action == 'off':
stdout, stderr, exitstatus = conn.execute('{} destroy {}'.format(virsh, name))
if exitstatus == 0:
logger.debug("Virtual machine '{}' stopped".format(name))
result = True
else:
logger.error(''.join(stderr))
raise Exception(''.join(stderr))
elif action == 'on':
stdout, stderr, exitstatus = conn.execute('{} start {}'.format(virsh, name))
if exitstatus == 0:
logger.debug("Virtual machine '{}' started".format(name))
result = True
else:
logger.error(''.join(stderr))
raise Exception(''.join(stderr))
else:
logger.warning("Action '{}' does not exist".format(action))
result = False
if conn:
conn.close()
return result
def execute(self):
"""
Executes the task.
"""
if not ServerConfig.objects.bool_by_key('orthos.debug.motd.write'):
logger.warning("Disabled: set 'orthos.debug.motd.write' to 'true'")
return
BEGIN = '-' * 69 + ' Orthos{ --'
LINE = '-' * 80
END = '-' * 69 + ' Orthos} --'
try:
machine = Machine.objects.get(fqdn=self.fqdn)
except Machine.DoesNotExist:
logger.error("Machine does not exist: fqdn={}".format(self.fqdn))
return
conn = None
try:
conn = SSH(machine.fqdn)
conn.connect()
motd = conn.get_file('/etc/motd.orthos', 'w')
print(BEGIN, file=motd)
print(
"Machine of the ARCH team. Contact <{}> for problems.".format(
machine.get_support_contact()),
file=motd)
if machine.comment:
print("INFO: " + machine.comment, file=motd)
if machine.administrative:
print(
"This machine is an administrative machine. DON\'T TOUCH!",
file=motd)
if machine.reserved_by:
print(LINE, file=motd)
if machine.reserved_until == timezone.ZERO:
print("This machine is RESERVED by {}.".format(
machine.reserved_by),
file=motd)
else:
print("This machine is RESERVED by {} until {}.".format(
machine.reserved_by, machine.reserved_until),
file=motd)
print('', file=motd)
print(wrap80(machine.reserved_reason), file=motd)
print(END, file=motd)
motd.close()
stdout, stderr, exitstatus = conn.execute_script_remote(
'machine_sync_motd.sh')
if exitstatus != 0:
logger.exception("({}) {}".format(machine.fqdn, stderr))
raise Exception(stderr)
except SSH.Exception as e:
logger.error("({}) {}".format(machine.fqdn, e))
return False
except IOError as e:
logger.error("({}) {}".format(machine.fqdn, e))
return False
finally:
if conn:
conn.close()
class CobblerServer:
def __init__(self, fqdn, domain):
self._fqdn = fqdn
self._conn = None
self._domain = domain
self._cobbler_path = ServerConfig.objects.by_key("cobbler.command")
@staticmethod
def from_machine(machine: Machine):
"""
Return the cobbler server associated to a machine
:param machine: Machine object which is managed by the cobbler server to fetch
:returns: The corresponding cobbler server or None
"""
domain = machine.fqdn_domain
server = domain.cobbler_server.all()
if server and server[0]:
return CobblerServer(server[0].fqdn, domain)
return None
def connect(self):
"""Connect to DHCP server via SSH."""
if not self._conn:
self._conn = SSH(self._fqdn)
self._conn.connect()
def close(self):
"""Close connection to DHCP server."""
if self._conn:
self._conn.close()
def deploy(self):
self.connect()
if not self.is_installed():
raise CobblerException("No Cobbler service found: {}".format(self._fqdn))
if not self.is_running():
raise CobblerException("Cobbler server is not running: {}".format(self._fqdn))
machines = Machine.active_machines.filter(fqdn_domain=self._domain.pk)
cobbler_machines = self.get_machines()
cobbler_commands = []
for machine in machines:
if machine.fqdn in cobbler_machines:
cobbler_commands.append(get_cobbler_update_command(machine, self._cobbler_path))
else:
cobbler_commands.append(get_cobbler_add_command(machine, self._cobbler_path))
if hasattr(machine, 'bmc') and machine.bmc:
cobbler_commands.append(get_bmc_command(machine, self._cobbler_path))
for command in cobbler_commands: # TODO: Convert this to a single ssh call (performance)
logger.debug("executing %s ", command)
_, stderr, exitcode = self._conn.execute(command)
if exitcode:
logger.error("failed to execute %s on %s with error %s",
command, self._fqdn, stderr)
self.close()
def update(self, machine: Machine):
self.connect()
self._check()
command = get_cobbler_update_command(machine, self._cobbler_path)
_, stderr, exitcode = self._conn.execute(command)
if exitcode:
raise CobblerException("Updating {machine} failed with {err}".format(
machine.fqdn, stderr))
def remove(self, machine: Machine):
#ToDo: We do not remove machines from cobbler server actively in orthos2 yet
logging.warning("cobbler remove is switched off")
return
self.connect()
if not self.is_installed():
raise CobblerException("No Cobbler service found: {}".format(self._fqdn))
if not self.is_running():
raise CobblerException("Cobbler server is not running: {}".format(self._fqdn))
command = "{cobbler} system remove --name {fqdn}".format(
cobbler=self._cobbler_path, fqdn=machine.fqdn)
_, stderr, exitcode = self._conn.execute(command)
if(exitcode):
logging.error("Removing %s failed with '%s'", machine.fqdn, stderr)
def is_installed(self):
"""Check if Cobbler server is available."""
if self._conn.check_path(self._cobbler_path, '-x'):
return True
return False
def is_running(self):
"""Check if the Cobbler daemon is running via the cobbler version command."""
command = "{} version".format(self._cobbler_path)
_, _, exitstatus = self._conn.execute(command)
if exitstatus == 0:
return True
return False
def get_machines(self):
stdout, stderr, exitstatus = self._conn.execute(
"{cobbler} system list".format(cobbler=self._cobbler_path))
if exitstatus:
logger.warning("system list failed on %s with %s", self._fqdn, stderr)
raise CobblerException("system list failed on {server}".format(server=self._fqdn))
clean_out = [system.strip(' \n\t') for system in stdout]
return clean_out
def setup(self, machine: Machine, choice: str):
logger.info("setup called for %s with %s on cobbler server %s ", machine.fqdn, self._fqdn,
choice)
if choice:
cobbler_profile = "{arch}:{profile}".format(arch=machine.architecture, profile=choice)
else:
cobbler_profile = get_default_profile(machine)
command = "{cobbler} system edit --name={machine} --profile={profile} --netboot=True"\
.format(cobbler=self._cobbler_path, machine=machine.fqdn, profile=cobbler_profile)
logger.debug("command for setup: %s", command)
self.connect()
try:
stdout, stderr, exitstatus = self._conn.execute(command)
if exitstatus:
logger.warning("setup of %s with %s failed on %s with %s", machine.fqdn,
cobbler_profile, self._fqdn, stderr)
raise CobblerException(
"setup of {machine} with {profile} failed on {server} with {error}".format(
machine=machine.fqdn, arch=cobbler_profile, server=self._fqdn))
except:
pass
finally:
self.close()
def powerswitch(self, machine: Machine, action: str):
logger.debug("powerswitching of %s called with action %s", machine.fqdn, action)
self.connect()
cobbler_action = ""
if action == "reboot":
cobbler_action = "reboot"
else:
cobbler_action = "power" + action
command = "{cobbler} system {action} --name {fqdn}".format(cobbler=self._cobbler_path,
action=cobbler_action,
fqdn=machine.fqdn)
out, stderr, exitcode = self._conn.execute(command)
if exitcode:
logger.warning("Powerswitching of %s with %s failed on %s with %s", machine.fqdn,
command, self._fqdn, stderr)
raise CobblerException(
"Powerswitching of {machine} with {command} failed on {server} with {error}".format(
machine=machine.fqdn, command=command, server=self._fqdn))
return out
def _check(self):
if not self.is_installed():
raise CobblerException("No Cobbler service found: {}".format(self._fqdn))
if not self.is_running():
raise CobblerException("Cobbler server is not running: {}".format(self._fqdn))
class CobblerServer:
def __init__(self, fqdn, domain):
self._fqdn = fqdn
self._conn = None
self._domain = domain
self._cobbler_path = ServerConfig.objects.by_key("cobbler.command")
def connect(self):
"""Connect to DHCP server via SSH."""
if not self._conn:
self._conn = SSH(self._fqdn)
self._conn.connect()
def close(self):
"""Close connection to DHCP server."""
if self._conn:
self._conn.close()
def deploy(self):
self.connect()
if not self.is_installed():
raise CobblerException("No Cobbler service found: {}".format(self._fqdn))
if not self.is_running():
raise CobblerException("Cobbler server is not running: {}".format(self._fqdn))
machines = Machine.active_machines.filter(fqdn_domain=self._domain.pk)
cobbler_machines = self.get_machines()
cobbler_commands = []
for machine in machines:
if machine.fqdn in cobbler_machines:
cobbler_commands.append(get_cobbler_update_command(machine, self._cobbler_path))
else:
cobbler_commands.append(get_cobbler_add_command(machine, self._cobbler_path))
for command in cobbler_commands: # TODO: Convert this to a single ssh call (performance)
_, stderr, exitcode = self._conn.execute(command)
if exitcode:
logger.error("failed to execute %s on %s with error %s",
command, self._fqdn, stderr)
self.close()
def is_installed(self):
"""Check if Cobbler server is available."""
if self._conn.check_path(self._cobbler_path, '-x'):
return True
return False
def is_running(self):
"""Check if the Cobbler daemon is running via the cobbler version command."""
command = "{} version".format(self._cobbler_path)
_, _, exitstatus = self._conn.execute(command)
if exitstatus == 0:
return True
return False
def get_machines(self):
stdout, stderr, exitstatus = self._conn.execute(
"{cobbler} system list".format(cobbler=self._cobbler_path))
if exitstatus:
logger.warning("system list failed on %s with %s", self._fqdn, stderr)
raise CobblerException("system list failed on {server}".format(server=self._fqdn))
clean_out = [system.strip(' \n\t') for system in stdout]
return clean_out
def execute(self):
"""Execute the task."""
from orthos2.data.models import Machine, SerialConsole
if not ServerConfig.objects.bool_by_key(
'orthos.debug.serialconsole.write'):
logger.warning(
"Disabled: set 'orthos.debug.serialconsole.write' to 'true'")
return
try:
cscreen_server = Machine.objects.get(fqdn=self.fqdn)
except Machine.DoesNotExist:
logger.warning("Serial console server does not exist: {}".format(
self.fqdn))
conn = None
try:
conn = SSH(cscreen_server.fqdn)
conn.connect()
stdout, stderr, exitstatus = conn.execute(
'sudo touch /etc/cscreenrc_allow_update')
if exitstatus != 0:
raise Exception(
"Couldn't lock cscreen ('touch /etc/cscreenrc_allow_update')"
)
new_content = ''
for serialconsole in SerialConsole.cscreen.get(cscreen_server):
new_content += serialconsole.get_comment_record() + '\n'
new_content += serialconsole.get_command_record() + '\n'
screenrc_file = '/etc/cscreenrc'
# create `/etc/cscreenrc` if it doesn't exist
stdout, stderr, exitstatus = conn.execute(
'[ -e "{}"]'.format(screenrc_file))
orthos_inline_begin = ServerConfig.objects.by_key(
'orthos.configuration.inline.begin')
orthos_inline_end = ServerConfig.objects.by_key(
'orthos.configuration.inline.end')
if exitstatus != 0:
stdout, stderr, exitstatus = conn.execute(
'echo "{}\n{}" > {}'.format(orthos_inline_begin,
screenrc_file,
orthos_inline_end))
if exitstatus != 0:
raise Exception("Couldn't create CScreen file ('{}')".format(
screenrc_file))
# Save backup file which is used later by an invoked script
# to determine the changes and update the running screen
# session (add, remove or restart modified entries).
stdout, stderr, exitstatus = conn.execute(
'sudo cp {} {}.old'.format(screenrc_file, screenrc_file))
cscreen = conn.get_file(screenrc_file, 'r')
buffer = ''
in_replace = False
for line in cscreen.readlines():
if not in_replace and line.startswith(orthos_inline_begin):
buffer += line + new_content
in_replace = True
elif in_replace and line.startswith(orthos_inline_end):
buffer += line
in_replace = False
elif not in_replace:
buffer += line
cscreen.close()
cscreen = conn.get_file(screenrc_file, 'w')
buffer = buffer.strip('\n')
print(buffer, file=cscreen)
cscreen.close()
stdout, stderr, exitstatus = conn.execute(
'sudo /usr/bin/cscreen -u')
logger.info("CScreen update exited with: {}".format(exitstatus))
stdout, stderr, exitstatus = conn.execute(
'sudo rm -f /etc/cscreenrc_allow_update')
if exitstatus != 0:
raise Exception(
"Couldn't unlock CScreen ('rm /etc/cscreenrc_allow_update')"
)
except SSH.Exception as exception:
logger.error(exception)
except IOError as exception:
logger.error(exception)
finally:
if conn:
conn.close()
def get_hardware_information(fqdn):
"""Retrieve information of the system."""
try:
machine = Machine.objects.get(fqdn=fqdn)
except Machine.DoesNotExist:
logger.warning("Machine '{}' does not exist".format(fqdn))
return
# set needed values for several checks from original machine
machine_ = Machine(
architecture=machine.architecture
)
conn = None
timer = None
try:
conn = SSH(fqdn)
conn.connect()
timer = threading.Timer(5 * 60, conn.close)
timer.start()
# CPUs
logger.debug("Get CPU number...")
output, stderr, exitstatus = conn.execute_script_remote('machine_get_cpu_number.sh')
if output:
for line in output:
if line.startswith('SOCKETS'):
machine_.cpu_physical = int(line.split('=')[1])
elif line.startswith('CORES'):
machine_.cpu_cores = int(line.split('=')[1])
elif line.startswith('THREADS'):
machine_.cpu_threads = int(line.split('=')[1])
logger.debug("Get CPU type...")
output, stderr, exitstatus = conn.execute_script_remote('machine_get_cpu_type.sh')
if output and output[0]:
machine_.cpu_model = output[0].strip()
logger.debug("Get CPU flags...")
output, stderr, exitstatus = conn.execute_script_remote('machine_get_cpu_flags.sh')
if output and output[0]:
machine_.cpu_flags = output[0].strip()
logger.debug("Get CPU speed...")
output, stderr, exitstatus = conn.execute_script_remote('machine_get_cpu_speed.sh')
if output and output[0]:
machine_.cpu_speed = Decimal(int(output[0].strip()) / 1000000)
logger.debug("Get CPU ID...")
output, stderr, exitstatus = conn.execute_script_remote('machine_get_cpu_id.sh')
if output and output[0]:
machine_.cpu_id = output[0].strip()
# EFI
logger.debug("Check for EFI...")
try:
efi_file = conn.get_file('/sys/firmware/efi', 'r')
efi_file.close()
machine_.efi = True
except IOError:
machine_.efi = False
# Memory
logger.debug("Get RAM amount...")
for line in conn.read_file('/proc/meminfo'):
if line.startswith('MemTotal'):
machine_.ram_amount = int(int(line.split()[1]) / 1024)
# Virtualization capability
VM_HOST_MIN_RAM_MB = 7000
machine_.vm_capable = False
# Virtualization: x86
logger.debug("Check for VM capability...")
if machine_.architecture_id == Architecture.Type.X86_64:
cpu_flags = machine_.cpu_flags
if cpu_flags:
cpu_flags = cpu_flags.upper()
if ((cpu_flags.find('VMX') >= 0 or cpu_flags.find('SVM') >= 0) and
int(machine_.ram_amount) > VM_HOST_MIN_RAM_MB):
machine_.vm_capable = True
# Virtualization: ppc64le
if machine_.architecture_id == Architecture.Type.PPC64LE:
for line in conn.read_file('/proc/cpuinfo'):
if line.startswith('firmware') and 'OPAL' in line:
machine_.vm_capable = True
# Disk
logger.debug("Get disk information...")
stdout, stderr, exitstatus = conn.execute('hwinfo --disk')
for line in stdout:
line = line.strip()
if line.startswith('Size:'):
machine_.disk_primary_size = int(int(line.split()[1]) / 2 / 1024 ** 2)
elif line.startswith('Attached to:'):
opening_bracket = line.find('(')
closing_bracket = line.find(')')
if opening_bracket > 0 and closing_bracket > 0:
machine_.disk_type = line[opening_bracket + 1:closing_bracket]
else:
machine_.disk_type = 'Unknown disk type'
break
# lsmod
logger.debug("Get 'lsmod'...")
stdout, stderr, exitstatus = conn.execute('lsmod')
machine_.lsmod = normalize_ascii("".join(stdout))
# lspci
logger.debug("Get 'lspci'...")
stdout, stderr, exitstatus = conn.execute('lspci -vvv -nn')
machine_.lspci = normalize_ascii("".join(stdout))
# last
logger.debug("Get 'last'...")
output, stderr, exitstatus = conn.execute('last | grep -v reboot | head -n 1')
string = ''.join(output)
result = string[0:8] + string[38:49]
machine_.last = normalize_ascii("".join(result))
# hwinfo
logger.debug("Get 'hwinfo' (full)...")
stdout, stderr, exitstatus = conn.execute(
'hwinfo --bios ' +
'--block --bridge --cdrom --cpu --disk --floppy --framebuffer ' +
'--gfxcard --hub --ide --isapnp --isdn --keyboard --memory ' +
'--monitor --mouse --netcard --network --partition --pci --pcmcia ' +
'--scsi --smp --sound --sys --tape --tv --usb --usb-ctrl --wlan'
)
machine_.hwinfo = normalize_ascii("".join(stdout))
# dmidecode
logger.debug("Get 'dmidecode'...")
stdout, stderr, exitstatus = conn.execute('dmidecode')
machine_.dmidecode = normalize_ascii("".join(stdout))
# dmesg
logger.debug("Get 'dmesg'...")
stdout, stderr, exitstatus = conn.execute(
'if [ -e /var/log/boot.msg ]; then ' +
'cat /var/log/boot.msg; else journalctl -xl | head -n200; ' +
'fi'
)
machine_.dmesg = normalize_ascii("".join(stdout))
# lsscsi
logger.debug("Get 'lsscsi'...")
stdout, stderr, exitstatus = conn.execute('lsscsi -s')
machine_.lsscsi = normalize_ascii("".join(stdout))
# lsusb
logger.debug("Get 'lsusb'...")
stdout, stderr, exitstatus = conn.execute('lsusb')
machine_.lsusb = normalize_ascii("".join(stdout))
# IPMI
logger.debug("Check for IPMI...")
machine_.ipmi = machine_.dmidecode.find('IPMI') >= 0
# Firmware script
logger.debug("Get BIOS version...")
output, stderr, exitstatus = conn.execute_script_remote('machine_get_firmware.sh')
if output and output[0]:
machine_.bios_version = output[0].strip()
return machine_
except Exception as e:
logger.error("{} ({})".format(fqdn, e))
return False
finally:
if conn:
conn.close()
if timer:
timer.cancel()
return None
def get_pci_devices(fqdn):
"""Retrieve all PCI devices."""
def get_pci_device_by_slot(pci_devices, slot):
"""Return the PCI device by slot."""
for dev in pci_devices:
pci_slot = dev.slot
if not pci_slot:
continue
pci_slot = pci_slot.strip()
# pci domain hacks
if len(pci_slot) < 8:
pci_slot = '0000:' + pci_slot
if len(slot) < 8:
slot = '0000:' + slot
if pci_slot == slot:
return dev
return None
from orthos2.data.models import PCIDevice
try:
machine = Machine.objects.get(fqdn=fqdn)
except Machine.DoesNotExist:
logger.warning("Machine '{}' does not exist".format(fqdn))
return False
conn = None
timer = None
try:
conn = SSH(fqdn)
conn.connect()
timer = threading.Timer(5 * 60, conn.close)
timer.start()
logger.debug("Collect PCI devices for '{}'...".format(machine.fqdn))
pci_devices = []
chunk = ''
stdout, stderr, exitstatus = conn.execute('lspci -mmvn')
for line in stdout:
if line.strip():
chunk += line
else:
pci_devices.append(PCIDevice.from_lspci_mmnv(chunk))
chunk = ''
# drivers for PCI devices from hwinfo
in_pci_device = False
current_busid = None
if machine.hwinfo:
for line in machine.hwinfo.splitlines():
if re.match(r'^\d+: PCI', line):
in_pci_device = True
continue
if not line.strip():
in_pci_device = False
current_busid = None
continue
if not in_pci_device:
continue
match = re.match(r' SysFS BusID: ([0-9a-fA-F.:]+)', line)
if match:
current_busid = match.group(1)
match = re.match(r' Driver: "([^"]*)"', line)
if match and current_busid:
pcidev = get_pci_device_by_slot(pci_devices, current_busid)
if pcidev:
pcidev.drivermodule = match.group(1)
match = re.match(r' Driver Modules: "([^"]*)"', line)
if match and current_busid:
pcidev = get_pci_device_by_slot(pci_devices, current_busid)
if pcidev:
pcidev.drivermodule = match.group(1)
for pci_device in pci_devices:
pci_device.machine = machine
logger.debug("Collected {} PCI devices for '{}'".format(len(pci_devices), machine.fqdn))
return pci_devices
except Exception as e:
logger.exception("{} ({})".format(fqdn, e))
return False
finally:
if conn:
conn.close()
if timer:
timer.cancel()
def get_status_ip(fqdn):
"""Retrieve information of the systems IPv4/IPv6 status."""
try:
machine = Machine.objects.get(fqdn=fqdn)
except Machine.DoesNotExist:
logger.warning("Machine '{}' does not exist".format(fqdn))
return False
machine_ = Machine()
conn = None
timer = None
try:
conn = SSH(machine.fqdn)
conn.connect()
timer = threading.Timer(5 * 60, conn.close)
timer.start()
logger.debug("Check IPv4/IPv6 status...")
stdout, stderr, exitstatus = conn.execute('/sbin/ip a')
devices = {}
current_device = None
addresses = {'inet': [], 'inet6': []}
for line in stdout:
match = re.match(r'^\d+:\s+([a-zA-Z0-9]+):\s+<.*>\s(.*)\n', line)
if match:
current_device = match.group(1)
devices[current_device] = {
'mac_address': None,
'inet': None,
'inet6': None,
'flags': None
}
devices[current_device]['flags'] = match.group(2).split()
continue
line = line.lstrip()
match = re.match(r'inet ([0-9.]{7,15})\/.*scope', line)
if match:
if devices[current_device]['inet'] is None:
devices[current_device]['inet'] = []
devices[current_device]['inet'].append(match.group(1))
continue
match = re.match(r'inet6 ([a-f0-9:]*)\/[0-9]+ scope', line)
if match:
if devices[current_device]['inet6'] is None:
devices[current_device]['inet6'] = []
devices[current_device]['inet6'].append(match.group(1))
continue
match = re.match('link/ether ([a-f0-9:]{17}) brd', line)
if match:
devices[current_device]['mac_address'] = match.group(1).upper()
for device, values in devices.items():
if values['mac_address'] is None:
continue
# ignore device if hooking up another
if any(device in values['flags'] for device in devices.keys()):
continue
if values['mac_address'] == machine.mac_address:
if values['inet'] is None:
machine_.status_ipv4 = Machine.StatusIP.AF_DISABLED
elif machine.ipv4 not in values['inet']:
machine_.status_ipv4 = Machine.StatusIP.NO_ADDRESS
if [ipv4 for ipv4 in values['inet'] if not ipv4.startswith('127.0.0.1')]:
machine_.status_ipv4 = Machine.StatusIP.ADDRESS_MISMATCH
elif machine.ipv4 in values['inet']:
machine_.status_ipv4 = Machine.StatusIP.CONFIRMED
else:
machine_.status_ipv4 = Machine.StatusIP.MISSING
if values['inet6'] is None:
machine_.status_ipv6 = Machine.StatusIP.AF_DISABLED
elif machine.ipv6 not in values['inet6']:
machine_.status_ipv6 = Machine.StatusIP.NO_ADDRESS
if [ipv6 for ipv6 in values['inet6'] if not ipv6.startswith('fe80::')]:
machine_.status_ipv6 = Machine.StatusIP.ADDRESS_MISMATCH
elif machine.ipv6 in values['inet6']:
machine_.status_ipv6 = Machine.StatusIP.CONFIRMED
addresses['inet'].append(values['inet'])
addresses['inet6'].append(values['inet6'])
if machine_.status_ipv4 == Machine.StatusIP.NO_ADDRESS:
if machine.ipv4 in addresses['inet']:
machine_.status_ipv4 = Machine.StatusIP.MAC_MISMATCH
if machine_.status_ipv6 == Machine.StatusIP.NO_ADDRESS:
if machine.ipv6 in addresses['inet6']:
machine_.status_ipv6 = Machine.StatusIP.MAC_MISMATCH
return machine_
except Exception as e:
logger.error("{} ({})".format(fqdn, e))
return False
finally:
if conn:
conn.close()
if timer:
timer.cancel()
return None
def get_networkinterfaces(fqdn):
"""Retrieve information of the systems network interfaces."""
try:
machine = Machine.objects.get(fqdn=fqdn)
except Machine.DoesNotExist:
logger.warning("Machine '{}' does not exist".format(fqdn))
return False
conn = None
timer = None
try:
conn = SSH(machine.fqdn)
conn.connect()
timer = threading.Timer(5 * 60, conn.close)
timer.start()
# Network interfaces
logger.debug("Collect network interfaces...")
stdout, stderr, exitstatus = conn.execute('hwinfo --network')
interfaces = []
interface = None
for line in stdout:
if line and line[0] != ' ' and line[0] != '\t':
if interface and interface.mac_address and\
interface.driver_module not in {'bridge', 'tun'}:
interfaces.append(interface)
interface = NetworkInterface()
else:
match = re.match(r'\s+Driver: "(\w+)"', line)
if match:
interface.driver_module = match.group(1)
continue
match = re.match(r'\s+SysFS ID: ([/\w.]+)', line)
if match:
interface.sysfs = match.group(1)
continue
match = re.match(r'\s+HW Address: ([0-9a-fA-F:]+)', line)
if match:
interface.mac_address = match.group(1).upper()
continue
match = re.match(r'\s+Device File: ([\w.]+)', line)
if match:
interface.name = match.group(1)
continue
if interface and interface.mac_address and\
interface.driver_module not in {'bridge', 'tun'}:
interfaces.append(interface)
for interface in interfaces:
if interface.sysfs is None:
continue
path = '/sys/{}/type'.format(interface.sysfs)
arp_type = ''.join(conn.read_file(path))
if arp_type == ARPHRD_IEEE80211:
continue
stdout, stderr, exitstatus = conn.execute('ethtool {}'.format(interface.name))
for line in stdout:
match = re.match(r'\s+Port: (.+)', line)
if match:
interface.ethernet_type = match.group(1)
return interfaces
except Exception as e:
logger.error("{} ({})".format(fqdn, e))
return False
finally:
if conn:
conn.close()
if timer:
timer.cancel()
return None
def get_setup_records(self,
architecture,
machinegroup=None,
grouped=True,
delimiter=':'):
"""
Collect domain and architecture or machine group specific setup records.
Each domain has one optional TFTP server providing available records for machine setup.
If `grouped` is False, a list of all records gets returned (no grouping).
Expects stdout:
['DISTRIBUTION-<architecture|machinegroup>-FLAVOUR', ...]
[
'SLES12-SP3-x86_64-install\n',
'SLES12-SP3-x86_64-install-ssh\n',
...
'SLES12-SP2-x86_64-install\n',
'SLES12-SP2-x86_64-rescue\n',
...,
'local-x86_64\n',
...,
]
Returns (grouped is `True`):
OrderedDict([
('SLES12-SP3', [
'SLES12-SP3-x86_64-install',
'SLES12-SP3-x86_64-install-ssh',
...
]),
('SLES12-SP2', [
'SLES12-SP2-x86_64-install',
'SLES12-SP2-x86_64-rescue',
...
]),
('local', [
'local-x86_64'
])
)
Returns (grouped is `False`):
[
'SLES12-SP3-x86_64-install',
'SLES12-SP3-x86_64-install-ssh',
...
'SLES12-SP2-x86_64-install',
'SLES12-SP2-x86_64-rescue',
...,
'local-x86_64',
...,
]
"""
from orthos2.utils.ssh import SSH
def grouping(records):
"""Group records for HTML form."""
groups = {}
for record in records:
record_ = record.split(delimiter)
if len(record_) == 2:
prefix = record_[0]
suffix = record_[1]
else:
logger.debug(
"Setup record has invalid format: '{}'".format(record))
continue
if prefix not in groups:
groups[prefix] = []
groups[prefix].append(suffix)
return collections.OrderedDict(sorted(groups.items()))
if not self.tftp_server:
logger.warning("No TFTP server available for '{}'".format(
self.name))
return {}
list_command_template = ServerConfig.objects.by_key(
'setup.list.command')
context = Context({
'architecture': architecture,
'machinegroup': machinegroup
})
list_command = Template(list_command_template).render(context)
try:
conn = SSH(self.tftp_server.fqdn)
conn.connect()
logger.debug("Fetch setup records: {}:{}".format(
self.tftp_server.fqdn, list_command))
stdout, stderr, exitstatus = conn.execute(list_command)
conn.close()
if exitstatus != 0:
logger.warning(str(stderr))
return {}
logger.debug("Found {} setup records on {}".format(
len(stdout), self.tftp_server.fqdn))
except Exception as e:
logger.warning("Couldn't fetch record list for setup: {}".format(
str(e)))
return {}
finally:
if conn:
conn.close()
records = list(map(lambda record: record.strip('\n'), stdout))
if grouped:
records = grouping(records)
else:
records = list(
map(lambda record: record.split(delimiter)[1], records))
return records
def execute(self):
"""Execute the task."""
from orthos2.data.models import Machine, SerialConsole
if not ServerConfig.objects.bool_by_key(
'orthos.debug.serialconsole.write'):
logger.warning(
"Disabled: set 'orthos.debug.serialconsole.write' to 'true'")
return
try:
cscreen_server = Machine.objects.get(fqdn=self.fqdn)
except Machine.DoesNotExist:
logger.warning("Serial console server does not exist: {}".format(
self.fqdn))
conn = None
try:
conn = SSH(cscreen_server.fqdn)
conn.connect(user='******')
stdout, stderr, exitstatus = conn.execute(
'touch /dev/shm/.cscreenrc_allow_update')
if exitstatus != 0:
raise Exception(
"Couldn't lock cscreen ('touch /dev/shm/.cscreenrc_allow_update')"
)
new_content = ''
for serialconsole in SerialConsole.cscreen.get(cscreen_server):
new_content += serialconsole.get_comment_record() + '\n'
new_content += serialconsole.get_command_record() + '\n'
screenrc_file = '/etc/cscreenrc'
orthos_inline_begin = ServerConfig.objects.by_key(
'orthos.configuration.inline.begin')
orthos_inline_end = ServerConfig.objects.by_key(
'orthos.configuration.inline.end')
buffer = ''
file_found = True
try:
cscreen = conn.get_file(screenrc_file, 'r')
in_replace = False
marker_found = False
for line in cscreen.readlines():
if not in_replace and line.startswith(orthos_inline_begin):
buffer += line + new_content
in_replace = True
marker_found = True
elif in_replace and line.startswith(orthos_inline_end):
buffer += line
in_replace = False
elif not in_replace:
buffer += line
# orthos start marker was not found... Add it.
if marker_found is False:
logging.info("CSCREEN: Orthos marker not found, adding...")
buffer += orthos_inline_begin + '\n' + new_content + orthos_inline_end
cscreen.close()
except IOError as e:
_errno, _strerror = e.args
import errno
if _errno == errno.ENOENT:
file_found = False
logging.warning("{}:{} not found - creating...".format(
cscreen_server.fqdn, screenrc_file))
else:
raise (e)
# Create an empty file with just markers, this will get the .old file
# to diff against for new entries via cscreen -u
if not file_found:
buffer = orthos_inline_begin + '\n' + orthos_inline_end
cscreen = conn.get_file(screenrc_file, 'w')
buffer = buffer.strip('\n')
print(buffer, file=cscreen)
cscreen.close()
buffer = orthos_inline_begin + '\n' + new_content + orthos_inline_end
# Save backup file which is used later by an invoked script
# to determine the changes and update the running screen
# session (add, remove or restart modified entries).
stdout, stderr, exitstatus = conn.execute('cp {} {}.old'.format(
screenrc_file, screenrc_file))
cscreen = conn.get_file(screenrc_file, 'w')
buffer = buffer.strip('\n')
print(buffer, file=cscreen)
cscreen.close()
stdout, stderr, exitstatus = conn.execute('/usr/bin/cscreen -u')
if exitstatus != 0:
logger.warning(stderr)
stdout, stderr, exitstatus = conn.execute(
'rm -f /dev/shm/.cscreenrc_allow_update')
if exitstatus != 0:
raise Exception(
"Couldn't unlock CScreen ('rm /dev/shm/.cscreenrc_allow_update)"
)
logger.info("CScreen update for %s finished", cscreen_server.fqdn)
except SSH.Exception as exception:
logger.exception(exception)
except IOError as exception:
logger.exception(exception)
finally:
if conn:
conn.close()