Esempio n. 1
0
def valid_ssh(hostname, port, username, password):
    try:
        private_key = AppSetting.get('private_key')
        public_key = AppSetting.get('public_key')
    except KeyError:
        private_key, public_key = SSH.generate_key()
        AppSetting.set('private_key', private_key, 'ssh private key')
        AppSetting.set('public_key', public_key, 'ssh public key')
    if password:
        print(password)
        cli = SSH(hostname, port, username, password=password)
        try:
            cli.ping()
        except Exception:
            return False
        code, out = cli.exec_command('mkdir -p -m 700 ~/.ssh && \
                echo %r >> ~/.ssh/authorized_keys && \
                chmod 600 ~/.ssh/authorized_keys' % public_key)
        if code != 0:
            raise Exception(f'add public key error: {out!r}')
    else:
        cli = SSH(hostname, port, username, private_key)

    try:
        cli.ping()
    except Exception:
        return False
    return True
Esempio n. 2
0
def create_ion(tsuite):
    """Create ION file systems.

  Args:
    tsuite: tsuite runtime."""

    for ion in tsuite.sl2objects["ion"]:

        #Create monolithic reference/replace dict
        repl_dict = dict(tsuite.src_dirs, **tsuite.build_dirs)
        repl_dict = dict(repl_dict, **ion)

        #Create remote connection to server
        try:
            user, host = tsuite.user, ion["host"]
            log.debug("Connecting to {0}@{1}".format(user, host))
            ssh = SSH(user, host, '')

            cmd = """
      mkdir -p {datadir}
      mkdir -p {fsroot}
      {slmkfs} -Wi -u {fsuuid} -I {site_id} {fsroot}"""\
            .format(**repl_dict)

            sock_name = "ts.ion." + ion["id"]

            sl2gen.sl_screen_and_wait(tsuite, ssh, cmd, sock_name)

            log.info("Finished creating {0}!".format(ion["name"]))
            ssh.close()

        except SSHException, e:
            log.fatal("Error with remote connection to {0} with res {1}!"\
                .format(ion["host"], ion["name"]))
            tsuite.shutdown()
Esempio n. 3
0
def create_ion(tsuite):
  """Create ION file systems.

  Args:
    tsuite: tsuite runtime."""

  for ion in tsuite.sl2objects["ion"]:

    #Create monolithic reference/replace dict
    repl_dict = dict(tsuite.src_dirs, **tsuite.build_dirs)
    repl_dict = dict(repl_dict, **ion)

    #Create remote connection to server
    try:
      user, host = tsuite.user, ion["host"]
      log.debug("Connecting to {0}@{1}".format(user, host))
      ssh = SSH(user, host, '')

      cmd = """
      mkdir -p {datadir}
      mkdir -p {fsroot}
      {slmkfs} -Wi -u {fsuuid} -I {site_id} {fsroot}"""\
      .format(**repl_dict)

      sock_name = "ts.ion."+ion["id"]

      sl2gen.sl_screen_and_wait(tsuite, ssh, cmd, sock_name)

      log.info("Finished creating {0}!".format(ion["name"]))
      ssh.close()

    except SSHException, e:
      log.fatal("Error with remote connection to {0} with res {1}!"\
          .format(ion["host"], ion["name"]))
      tsuite.shutdown()
Esempio n. 4
0
 def connect(self):
     """
     Connect to DHCP server via SSH.
     """
     if not self._conn:
         self._conn = SSH(self._fqdn)
         self._conn.connect()
Esempio n. 5
0
        def decorator(self, *args, **kwargs):
            from utils.ssh import SSH

            if not self.conn:
                self.conn = SSH(self.host.fqdn)
                self.conn.connect()
            return function(self, *args, **kwargs)
Esempio n. 6
0
 def k8s_apply(self, params, out_func=None):
     cmd = K8S_APPLY_CMD + params['filename']
     ssh = SSH(hostname=params['public_ip'],
               port=22,
               username=params['username'],
               passwd=params['passwd'])
     out, err = ssh.exec_rt(cmd, out_func)
     return out, err
Esempio n. 7
0
def create_mds(tsuite):
    """Initialize MDS resources for testing.

    Args:
      tsuite: tsuite runtime."""

    #Create the MDS systems
    for mds in tsuite.sl2objects["mds"]:

        #Create monolithic reference/replace dict
        repl_dict = dict(tsuite.src_dirs, **tsuite.build_dirs)
        repl_dict = dict(repl_dict, **mds)

        #Create remote connection to server
        try:
            #Can probably avoid doing user, host everytime
            user, host = tsuite.user, mds["host"]
            log.debug("Connecting to {0}@{1}".format(user, host))
            ssh = SSH(user, host, '')

            cmd = """
      $SHELL -c "cd {src} && make printvar-CC >/dev/null"
      pkill zfs-fuse || true
      $SHELL -c "{zfs_fuse} &"
      sleep 2
      {zpool} destroy {zpool_name} || true
      sleep 2
      {zpool} create -m {zpool_path} -f {zpool_name} {zpool_args}
      sleep 2
      {zpool} set cachefile={zpool_cache} {zpool_name}
      sleep 2
      {slmkfs} -u {fsuuid} -I {site_id} {zpool_path}
      sleep 2
      sync
      umount {zpool_path}
      pkill zfs-fuse || true
      sleep 2
      $SHELL -c "{zfs_fuse} &"
      sleep 2
      {zpool} import {zpool_name} || true
      sleep 2
      pkill zfs-fuse || true
      sleep 2
      mkdir -p {datadir}
      {slmkjrnl} -D {datadir} -b {jrnldev} -f -u {fsuuid}""".format(
                **repl_dict)

            screen_name = "ts.mds." + mds["id"]

            sl2gen.sl_screen_and_wait(tsuite, ssh, cmd, screen_name)

            log.info("Finished creating {0}".format(mds["name"]))
            ssh.close()

        except SSHException, e:
            log.fatal("Error with remote connection to {0} with res {1}!"\
                .format(mds["host"], mds["name"]))
            tsuite.shutdown()
Esempio n. 8
0
def create_mds(tsuite):
  """Initialize MDS resources for testing.

    Args:
      tsuite: tsuite runtime."""

  #Create the MDS systems
  for mds in tsuite.sl2objects["mds"]:

    #Create monolithic reference/replace dict
    repl_dict = dict(tsuite.src_dirs, **tsuite.build_dirs)
    repl_dict = dict(repl_dict, **mds)

    #Create remote connection to server
    try:
      #Can probably avoid doing user, host everytime
      user, host = tsuite.user, mds["host"]
      log.debug("Connecting to {0}@{1}".format(user, host))
      ssh = SSH(user, host, '')

      cmd = """
      $SHELL -c "cd {src} && make printvar-CC >/dev/null"
      pkill zfs-fuse || true
      $SHELL -c "{zfs_fuse} &"
      sleep 2
      {zpool} destroy {zpool_name} || true
      sleep 2
      {zpool} create -m {zpool_path} -f {zpool_name} {zpool_args}
      sleep 2
      {zpool} set cachefile={zpool_cache} {zpool_name}
      sleep 2
      {slmkfs} -u {fsuuid} -I {site_id} {zpool_path}
      sleep 2
      sync
      umount {zpool_path}
      pkill zfs-fuse || true
      sleep 2
      $SHELL -c "{zfs_fuse} &"
      sleep 2
      {zpool} import {zpool_name} || true
      sleep 2
      pkill zfs-fuse || true
      sleep 2
      mkdir -p {datadir}
      {slmkjrnl} -D {datadir} -b {jrnldev} -f -u {fsuuid}""".format(**repl_dict)

      screen_name = "ts.mds."+mds["id"]

      sl2gen.sl_screen_and_wait(tsuite, ssh, cmd, screen_name)

      log.info("Finished creating {0}".format(mds["name"]))
      ssh.close()

    except SSHException, e:
      log.fatal("Error with remote connection to {0} with res {1}!"\
          .format(mds["host"], mds["name"]))
      tsuite.shutdown()
Esempio n. 9
0
 def create_image(self, params, out_func=None):
     cmd = CREATE_IMAGE_CMD + ' '.join([
         params['app_name'], params['image_name'],
         params['repos_https_url'], params['branch_name'], params['version']
     ])
     ssh = SSH(hostname=params['public_ip'],
               port=22,
               username=params['username'],
               passwd=params['passwd'])
     out, err = ssh.exec_rt(cmd, out_func)
     return out, err
Esempio n. 10
0
    def check_status(self):
        """Generate general status report for all sl2 objects.

    Returns: {
      "type":[ {"host": ..., "reports": ... } ],
      ...
    }"""
        report = {}

        #Operations based on type
        ops = {
            "all": {
                "load": "cat /proc/loadavg | cut -d' ' -f1,2,3",
                "mem_total": "cat /proc/meminfo | head -n1",
                "mem_free": "sed -n 2,2p /proc/meminfo",
                "uptime": "cat /proc/uptime | head -d' ' -f1",
                "disk_stats": "df -hl"
            },
            "mds": {
                "connections": "{slmctl} -sconnections",
                "iostats": "{slmctl} -siostats"
            },
            "ion": {
                "connections": "{slictl} -sconnections",
                "iostats": "{slictl} -siostats"
            }
        }

        for sl2_restype in self.sl2objects.keys():

            report[sl2_restype] = []

            obj_ops = ops["all"]
            if sl2_obj[sl2_restype] in ops:
                obj_ops = dict(ops["all"].items() + ops[sl2_restype].items())

            for sl2_obj in self.sl2objects[sl2_restype]:
                obj_report = {
                    "host": sl2_obj["host"],
                    "id": sl2_obj["id"],
                    "reports": {}
                }
                user, host = self.user, sl2_obj["host"]
                log.debug("Connecting to {0}@{1}".format(user, host))
                ssh = SSH(user, host, '')
                for op, cmd in obj_ops.items():
                    obj_report["reports"][op] = ssh.run(cmd, timeout=2)

                report[sl2_restype].append(obj_report)
                log.debug("Status check completed for {0} [{1}]".format(
                    host, sl2_restype))
                ssh.close()
        return report
Esempio n. 11
0
 def k8s_delete(self, params, out_func=None):
     if params.get('obj_type') and params.get('obj_name'):
         cmd = K8S_DELETE_CMD + ' '.join(
             [params['obj_type'], params['obj_name']])
     else:
         return '', ''
     ssh = SSH(hostname=params['public_ip'],
               port=22,
               username=params['username'],
               passwd=params['passwd'])
     out, err = ssh.exec_rt(cmd, out_func)
     return out, err
Esempio n. 12
0
def kill_mnt(tsuite):
  """Kill ION daemons.

  Args:
    tsuite: runtime tsuite."""
  
  for client in tsuite.sl2objects["client"]:
    ssh = SSH(tsuite.user, client["host"])
    if not ssh.run("sudo umount {0}".format(tsuite.build_dirs["mp"]))["err"] == []:
      log.critical("Cannot unmount client mountpoint at {0} @ {1}.".format(tsuite.build_dirs["mp"], client["host"]))

  sl2gen.stop_slash2_socks(tsuite, "client", tsuite.sl2objects["client"], "msctl", "mount_slash")
Esempio n. 13
0
 def get_captcha_from_k8s_log(self):
     SSH_CONFIG = HandleConfig(r'{}\ssh_config.yml'.format(
         SharePath.CONFIG_FOLDER)).config
     ssh_config = SSH_CONFIG.get("ssh")
     # ssh_config['hostname'] = "10.151.3.96"
     # 动态传入当前运行环境的ip
     ssh_config['hostname'] = f'{get_run_env()["host"]}'
     ssh = SSH(**ssh_config)
     oauth2_pod_name = ssh.execute_command(
         "kubectl get pods | grep oauth2 | awk '{print $1}'")
     captcha = ssh.execute_command(
         f"kubectl logs {oauth2_pod_name.rstrip()} --tail 2 | grep 生成验证码存入redis | awk -F ' ' '{{print $5}}'"
     )
     return captcha.rstrip()
Esempio n. 14
0
    def get_captcha_from_k8s_log():
        """ 从K8S日志中获取验证码

        返回: 验证码字符串

        """
        ssh_config = LoginPage.SSH_CONFIG.get("ssh")
        ssh_config['hostname'] = sb_config.host
        ssh = SSH(**ssh_config)
        oauth2_pod_name = ssh.execute_command(
            "kubectl get pods | grep oauth2 | awk '{print $1}'")
        captcha = ssh.execute_command(
            f"kubectl logs {oauth2_pod_name.rstrip()} --tail 2 | grep 生成验证码存入redis | awk -F ' ' '{{print $5}}'")
        return captcha.rstrip()
Esempio n. 15
0
    def execute(self, args):
        file = "/etc/dtux/env.yml"
        config = self.read_configuration(file)

        sshobjFromYml = config[args.source]

        sshobj = sshObject(sshobjFromYml['ipAddress'], sshobjFromYml['user'],
                           sshobjFromYml['password'])

        scp = SSH(sshobj)
        scp.scp_put(sshobj, "/etc/dtux/env.yml", "/root/diego.txt")
        command = scp.run_command("w")
        print(command)
        logger.warning("Some message: %s", command)
Esempio n. 16
0
    def check_status(self):
        """Generate general status report for all sl2 objects.

    Returns: {
      "type":[ {"host": ..., "reports": ... } ],
      ...
    }"""
        report = {}

        # Operations based on type
        ops = {
            "all": {
                "load": "cat /proc/loadavg | cut -d' ' -f1,2,3",
                "mem_total": "cat /proc/meminfo | head -n1",
                "mem_free": "sed -n 2,2p /proc/meminfo",
                "uptime": "cat /proc/uptime | head -d' ' -f1",
                "disk_stats": "df -hl",
            },
            "mds": {"connections": "{slmctl} -sconnections", "iostats": "{slmctl} -siostats"},
            "ion": {"connections": "{slictl} -sconnections", "iostats": "{slictl} -siostats"},
        }

        for sl2_restype in self.sl2objects.keys():

            report[sl2_restype] = []

            obj_ops = ops["all"]
            if sl2_obj[sl2_restype] in ops:
                obj_ops = dict(ops["all"].items() + ops[sl2_restype].items())

            for sl2_obj in self.sl2objects[sl2_restype]:
                obj_report = {"host": sl2_obj["host"], "id": sl2_obj["id"], "reports": {}}
                user, host = self.user, sl2_obj["host"]
                log.debug("Connecting to {0}@{1}".format(user, host))
                ssh = SSH(user, host, "")
                for op, cmd in obj_ops.items():
                    obj_report["reports"][op] = ssh.run(cmd, timeout=2)

                report[sl2_restype].append(obj_report)
                log.debug("Status check completed for {0} [{1}]".format(host, sl2_restype))
                ssh.close()
        return report
Esempio n. 17
0
    def create_remote_setups(self):
        """Create the necessary build directories on all slash2 objects."""

        for sl2_obj in self.all_objects():
            try:
                ssh = SSH(self.user, sl2_obj["host"], '')
                log.debug("Creating build directories on {0}@{1}".format(
                    sl2_obj["name"], sl2_obj["host"]))
                for d in self.build_dirs.values():
                    ssh.make_dirs(d)
                    ssh.run("sudo chmod -R 777 \"{0}\"".format(d), quiet=True)
                ssh.close()
            except SSHException:
                log.error(
                    "Unable to connect to {0} to create build directories!".
                    format(sl2_obj["host"]))
Esempio n. 18
0
def get_installations(fqdn):
    """
    Retrieves information of the installations.
    """
    try:
        machine = Machine.objects.get(fqdn=fqdn)
    except Machine.DoesNotExist:
        logger.warning("Machine '{}' does not exist".format(fqdn))
        return False

    conn = None
    timer = None
    try:
        conn = SSH(fqdn)
        conn.connect()
        timer = threading.Timer(5 * 60, conn.close)
        timer.start()

        # Installations
        logger.debug("Collect installations...")

        installations = []
        output, stderr, exitstatus = conn.execute_script_remote(
            'machine_get_installations.sh')
        if output:
            for line in output:
                if line.startswith('--'):
                    installation = Installation(machine=machine)
                    installations.append(installation)
                elif line.startswith('ARCH='):
                    installation.architecture = line.split('=')[1].strip()
                elif line.startswith('KERNEL='):
                    installation.kernelversion = line.split('=')[1].strip()
                elif line.startswith('RUNNING='):
                    installation.active = line.startswith('RUNNING=1')
                elif line.startswith('DIST='):
                    installation.distribution = line.split('=')[1].strip()
                elif line.startswith('PART='):
                    installation.partition = line.split('=')[1].strip()

        return installations

    except Exception as e:
        logger.error("{} ({})".format(fqdn, e))
        return False
    finally:
        if conn:
            conn.close()
        if timer:
            timer.cancel()

    return None
Esempio n. 19
0
    def create_remote_setups(self):
        """Create the necessary build directories on all slash2 objects."""

        for sl2_obj in self.all_objects():
            try:
                ssh = SSH(self.user, sl2_obj["host"], "")
                log.debug("Creating build directories on {0}@{1}".format(sl2_obj["name"], sl2_obj["host"]))
                for d in self.build_dirs.values():
                    ssh.make_dirs(d)
                    ssh.run('sudo chmod -R 777 "{0}"'.format(d), quiet=True)
                ssh.close()
            except SSHException:
                log.error("Unable to connect to {0} to create build directories!".format(sl2_obj["host"]))
Esempio n. 20
0
    def execute(self):
        """
        Executes the task.
        """
        if not ServerConfig.objects.bool_by_key('orthos.debug.setup.execute'):
            logger.warning(
                "Disabled: set 'orthos.debug.setup.execute' to 'true'")
            return

        logger.debug('Calling setup script...')

        try:
            machine = Machine.objects.get(fqdn=self.fqdn)
            tftp_server = machine.fqdn_domain.tftp_server

            if not tftp_server:
                logger.warning("No TFTP server available for '{}'".format(
                    machine.fqdn_domain.name))
                return

            command_template = ServerConfig.objects.by_key(
                'setup.execute.command')

            context = Context({'machine': machine, 'choice': self.choice})

            command = Template(command_template).render(context)

            logger.debug("Initialize setup {}@{}: {}:{}".format(
                self.choice, machine.fqdn, tftp_server.fqdn, command))

            tftp_server = SSH(tftp_server.fqdn)
            tftp_server.connect()
            stdout, stderr, exitstatus = tftp_server.execute(command)
            tftp_server.close()

            if exitstatus != 0:
                logger.warning(
                    "Creating setup configuration failed for '{}'".format(
                        machine))
                return

            # reboot machine finally
            machine.reboot()

        except SSH.Exception as exception:
            logger.error(exception)
        except Machine.DoesNotExist:
            logger.error("Machine does not exist: fqdn={}".format(self.fqdn))
        except Exception as e:
            logger.exception(e)
Esempio n. 21
0
def stop_slash2_socks(tsuite, sock_name, sl2objects, ctl_type, daemon_type):
    """ Terminates all slash2 socks and screen socks on a generic host.
  Args:
    tsuite: tsuite runtime.
    sock_name: name of sl2 sock.
    sl2objects: list of objects to be launched.
    ctl_type: key to ctl path in src_dirs
    daemon_type: key to daemon path in src_dirs"""

    assert (ctl_type in tsuite.src_dirs)
    assert (daemon_type in tsuite.src_dirs)

    for sl2object in sl2objects:
        log.info("Killing {0} @ {1}".format(sl2object["name"],
                                            sl2object["host"]))

        #Remote connection
        user, host = tsuite.user, sl2object["host"]
        log.debug("Connecting to {0}@{1}".format(user, host))

        ssh = None
        try:
            ssh = SSH(user, host, '')
        except Exception:
            log.error("Unable to connect to {0}@{1}".format(user, host))
            return

        cmd = "{0} -S {1}/{2}.{3}.sock stop".format(tsuite.src_dirs[ctl_type],
                                                    tsuite.build_dirs["ctl"],
                                                    daemon_type, host)
        ssh.run(cmd)

        if "id" not in sl2object.keys():
            sl2object["id"] = 0

        screen_sock_name = "sl.{0}.{1}".format(sock_name, sl2object["id"])
        ssh.kill_screens(screen_sock_name, exact_sock=True)

        ssh.close()
Esempio n. 22
0
def login_test(fqdn):
    """
    Checks if it's possible to login via SSH.
    """
    conn = None
    try:
        conn = SSH(fqdn)
        conn.connect()
    except Exception as e:
        logger.warning("SSH login failed for '{}': {}".format(fqdn, e))
        return False
    finally:
        if conn:
            conn.close()

    return True
Esempio n. 23
0
def abuild_test(fqdn):
    """
    Checks if Autobuild is running.
    """
    conn = None
    try:
        conn = SSH(fqdn)
        conn.connect()
        pids, stderr, exitstatus = conn.execute(
            r"ps -e -o pid,cmd | awk '/.*\/usr\/sbin\/autobuild.*/{print $1}'")
        if len(pids) > 0:
            return True
    except Exception as e:
        logger.warning("SSH login failed for '{}': {}".format(fqdn, e))
        return False
    finally:
        if conn:
            conn.close()

    return False
Esempio n. 24
0
    def ssh_shutdown(self, user=None, reboot=False):
        """
        Powers off/reboots the machine using SSH.
        """
        from utils.ssh import SSH

        if not reboot:
            option = '--poweroff'
        else:
            option = '--reboot'

        machine = SSH(self.fqdn)
        machine.connect()
        command = 'shutdown {} now'.format(option)
        stdout, stderr, exitstatus = machine.execute(command, retry=False)
        machine.close()

        if exitstatus != 0:
            return False

        return True
Esempio n. 25
0
    def remote_ssh(self, params):
        """ 远程控制主机
        :param params: dict, 必须{'public_ip', 'username', 'passwd', 'cmd'}, 可选{'rt'(实时输出), 'out_func'}
        """
        try:
            ssh = SSH(hostname=params['public_ip'],
                      username=params['username'],
                      passwd=params['passwd'])

            if params.get('rt'):
                out, err = ssh.exec_rt(params['cmd'], params.get('out_func'))
            else:
                out, err = ssh.exec(params['cmd'])
            ssh.close()

            return out, err
        except Exception as e:
            return [], [str(e)]
Esempio n. 26
0
def stop_slash2_socks(tsuite, sock_name, sl2objects, ctl_type, daemon_type):
    """ Terminates all slash2 socks and screen socks on a generic host.
  Args:
    tsuite: tsuite runtime.
    sock_name: name of sl2 sock.
    sl2objects: list of objects to be launched.
    ctl_type: key to ctl path in src_dirs
    daemon_type: key to daemon path in src_dirs"""

    assert ctl_type in tsuite.src_dirs
    assert daemon_type in tsuite.src_dirs

    for sl2object in sl2objects:
        log.info("Killing {0} @ {1}".format(sl2object["name"], sl2object["host"]))

        # Remote connection
        user, host = tsuite.user, sl2object["host"]
        log.debug("Connecting to {0}@{1}".format(user, host))

        ssh = None
        try:
            ssh = SSH(user, host, "")
        except Exception:
            log.error("Unable to connect to {0}@{1}".format(user, host))
            return

        cmd = "{0} -S {1}/{2}.{3}.sock stop".format(
            tsuite.src_dirs[ctl_type], tsuite.build_dirs["ctl"], daemon_type, host
        )
        ssh.run(cmd)

        if "id" not in sl2object.keys():
            sl2object["id"] = 0

        screen_sock_name = "sl.{0}.{1}".format(sock_name, sl2object["id"])
        ssh.kill_screens(screen_sock_name, exact_sock=True)

        ssh.close()
Esempio n. 27
0
    def parse_slash2_conf(self):
        """Reads and parses slash2 conf for tokens.
    Writes to the base directory; updates slash2 objects in the tsuite."""

        try:
            with open(self.conf["slash2"]["conf"]) as conf:
                new_conf = "#TSuite Slash2 Conf\n"

                res, site_name = None, None
                in_site = False
                site_id, fsuuid = -1, -1
                client = None

                # Regex config parsing for sl2objects
                reg = {
                    "clients": re.compile("^\s*?#\s*clients\s*=\s*(.+?)\s*;\s*$"),
                    "type": re.compile("^\s*?type\s*?=\s*?(\S+?)\s*?;\s*$"),
                    "id": re.compile("^\s*id\s*=\s*(\d+)\s*;\s*$"),
                    "zpool": re.compile(r"^\s*?#\s*?zfspool\s*?=\s*?(\w+?)\s+?(.*?)\s*$"),
                    "zpool_path": re.compile(r"^\s*?#\s*?zfspath\s*?=\s*?(.+?)\s*$"),
                    "prefmds": re.compile(r"\s*?#\s*?prefmds\s*?=\s*?(\w+?@\w+?)\s*$"),
                    "fsuuid": re.compile(r"^\s*set\s*fsuuid\s*=\s*\"?(0x[a-fA-F\d]+|\d+)\"?\s*;\s*$"),
                    "fsroot": re.compile("^\s*?fsroot\s*?=\s*?(\S+?)\s*?;\s*$"),
                    "nids": re.compile("^\s*?nids\s*?=\s*?(.*)$"),
                    "new_res": re.compile("^\s*resource\s+(\w+)\s*{\s*$"),
                    "fin_res": re.compile("^\s*?}\s*$"),
                    "site": re.compile("^\s*?site\s*?@(\w+).*?"),
                    "site_id": re.compile("^\s*site_id\s*=\s*(0x[a-fA-F\d]+|\d+)\s*;\s*$"),
                    "jrnldev": re.compile("^\s*jrnldev\s*=\s*([/\w]+)\s*;\s*$"),
                }

                line = conf.readline()

                while line:
                    # Replace keywords and append to new conf

                    new_conf += repl(self.build_dirs, line)

                    # Iterate through the regexes and return a tuple of
                    # (name, [\1, \2, \3, ...]) for successful matches

                    matches = [(k, reg[k].match(line).groups()) for k in reg if reg[k].match(line)]

                    # Should not be possible to have more than one
                    assert len(matches) <= 1

                    # log.debug("%s %s %s\n->%s" % (matches, in_site, res, line))
                    if matches:
                        (name, groups) = matches[0]

                        if in_site:

                            if name == "site_id":
                                site_id = groups[0]

                            elif res:
                                if name == "type":
                                    res["type"] = groups[0]

                                elif name == "id":
                                    res["id"] = groups[0]

                                elif name == "zpool_path":
                                    res["zpool_path"] = groups[0].strip()

                                elif name == "jrnldev":
                                    res["jrnldev"] = groups[0]

                                elif name == "zpool":
                                    res["zpool_name"] = groups[0]
                                    res["zpool_cache"] = path.join(self.build_dirs["base"], "{0}.zcf".format(groups[0]))
                                    res["zpool_args"] = groups[1]

                                elif name == "prefmds":
                                    res["prefmds"] = groups[0]

                                elif name == "fsroot":
                                    res["fsroot"] = groups[0].strip('"')

                                elif name == "nids":
                                    # Read subsequent lines and get the first host

                                    tmp = groups[0]
                                    while line and ";" not in line:
                                        tmp += line
                                        line = conf.readline()
                                    tmp = re.sub(";\s*$", "", tmp)
                                    res["host"] = re.split("\s*,\s*", tmp, 1)[0].strip(" ")

                                elif name == "fin_res":
                                    # Check for errors finalizing object
                                    res["site_id"] = site_id
                                    res["fsuuid"] = fsuuid

                                    if not res.finalize(self.sl2objects):
                                        self.shutdown()
                                    res = None
                            else:
                                if name == "new_res":
                                    res = SL2Res(groups[0], site_name)
                        else:
                            if name == "clients":
                                for client in [g.strip() for g in groups[0].split(",")]:
                                    client_res = SL2Res(client, None)
                                    client_res["type"] = "client"
                                    client_res["host"] = client
                                    client_res.finalize(self.sl2objects)
                            elif name == "site":
                                site_name = groups[0]
                                in_site = True
                            elif name == "fsuuid":
                                fsuuid = groups[0]

                    line = conf.readline()

                new_conf_path = path.join(self.build_dirs["base"], "slash.conf")

                try:
                    with open(new_conf_path, "w") as new_conf_file:
                        new_conf_file.write(new_conf)
                        log.debug("Successfully wrote build slash2 conf at {0}".format(new_conf_path))
                        for sl2_obj in self.all_objects():
                            try:
                                ssh = SSH(self.user, sl2_obj["host"], "")
                                log.debug("Copying new config to {0}".format(sl2_obj["host"]))
                                try:
                                    ssh.copy_file(new_conf_path, new_conf_path)
                                except IOError:
                                    log.critical("Error copying config file to {0}".format(ssh.host))
                                    self.shutdown()
                                ssh.close()
                            except SSHException:
                                log.error("Unable to copy config file to {0}!".format(sl2_obj["host"]))
                except IOError, e:
                    log.fatal("Unable to write new conf to build directory!")
                    log.fatal(new_conf_path)
                    log.fatal(e)
                    return False
        except IOError, e:
            log.fatal("Unable to read conf file at {0}".format(self.conf["slash2"]["conf"]))
            log.fatal(e)
            return False
Esempio n. 28
0
def launch_gdb_sl(tsuite, sock_name, sl2objects, res_bin_type, gdbcmd_path):
    """Generic slash2 launch service in screen+gdb. Will also copy over authbuf keys.

  Args:
    tsuite: tsuite runtime.
    sock_name: name of sl2 sock.
    sl2objects: list of objects to be launched.
    res_bin_type: key to bin path in src_dirs.
    gdbcmd_path: path to gdbcmd file."""

    # res_bin_type NEEDS to be a path in src_dirs
    assert res_bin_type in tsuite.src_dirs

    for sl2object in sl2objects:
        log.debug("Initializing environment > {0} @ {1}".format(sl2object["name"], sl2object["host"]))

        # Remote connection
        user, host = tsuite.user, sl2object["host"]
        log.debug("Connecting to {0}@{1}".format(user, host))
        ssh = SSH(user, host, "")

        # Acquire and deploy authbuf key
        need_authbuf = handle_authbuf(tsuite, ssh, sl2object["type"])

        ls_cmd = "ls {0}/".format(tsuite.build_dirs["ctl"])
        result = ssh.run(ls_cmd)

        present_socks = [res_bin_type in sock for sock in result["out"]].count(True)

        # Create monolithic reference/replace dict
        repl_dict = dict(tsuite.src_dirs, **tsuite.build_dirs)
        repl_dict = dict(repl_dict, **sl2object)

        if "id" not in sl2object.keys():
            sl2object["id"] = 0

        # Create gdbcmd from template
        gdbcmd_build_path = path.join(
            tsuite.build_dirs["base"], "{0}_{1}".format(sl2object["id"], path.basename(gdbcmd_path))
        )

        new_gdbcmd = repl_file(repl_dict, gdbcmd_path)

        if new_gdbcmd:
            with open(gdbcmd_build_path, "w") as f:
                f.write(new_gdbcmd)
                f.close()
                log.debug("Wrote gdb cmd to {0}".format(gdbcmd_build_path))
                log.debug("Remote copying gdbcmd.")
                ssh.copy_file(gdbcmd_build_path, gdbcmd_build_path)
        else:
            log.fatal("Unable to parse gdb cmd at {0}!".format(gdbcmd_path))
            tsuite.shutdown()

        cmd = "sudo gdb -f -x {0} {1}".format(gdbcmd_build_path, tsuite.src_dirs[res_bin_type])
        screen_sock_name = "sl.{0}.{1}".format(sock_name, sl2object["id"])

        # Launch slash2 in gdb within a screen session
        ssh.run_screen(cmd, screen_sock_name)

        # Wait two seconds to make sure slash2 launched without errors
        time.sleep(2)

        screen_socks = ssh.list_screen_socks()
        if screen_sock_name + "-error" in screen_socks or screen_sock_name not in screen_socks:
            log.fatal(
                "sl2object {0}:{1} launched with an error. Resume to {2} and resolve it.".format(
                    sl2object["name"], sl2object["id"], screen_sock_name + "-error"
                )
            )
            tsuite.shutdown(ignore=sock_name)

        log.debug("Waiting for {0} sock on {1} to appear.".format(sock_name, host))
        count = 0
        while True:
            result = ssh.run(ls_cmd, quiet=True)
            if not all(res_bin_type not in sock for sock in result["out"]):
                break
            time.sleep(1)
            count += 1
            if count == int(tsuite.conf["slash2"]["timeout"]):
                log.fatal(
                    "Cannot find {0} sock on {1}. Resume to {2} and resolve it. ".format(
                        res_bin_type, sl2object["id"], screen_sock_name
                    )
                )
                tsuite.shutdown(ignore=sock_name)

        # grab pid for resouce querying later
        # TODO: do not grab other running instances
        sl2object["pid"] = ssh.run("pgrep {0}".format(res_bin_type))["out"][0].strip()
        log.debug("Found {0} pid on {1} : {2}".format(res_bin_type, host, sl2object["pid"]))

        if need_authbuf:
            pull_authbuf(tsuite, ssh)

        ssh.close()
Esempio n. 29
0
def get_hardware_information(fqdn):
    """
    Retrieves information of the system.
    """
    try:
        machine = Machine.objects.get(fqdn=fqdn)
    except Machine.DoesNotExist:
        logger.warning("Machine '{}' does not exist".format(fqdn))
        return

    # set needed values for several checks from original machine
    machine_ = Machine(architecture=machine.architecture)

    conn = None
    timer = None
    try:
        conn = SSH(fqdn)
        conn.connect()
        timer = threading.Timer(5 * 60, conn.close)
        timer.start()

        # CPUs
        logger.debug("Get CPU number...")
        output, stderr, exitstatus = conn.execute_script_remote(
            'machine_get_cpu_number.sh')
        if output:
            for line in output:
                if line.startswith('SOCKETS'):
                    machine_.cpu_physical = int(line.split('=')[1])
                elif line.startswith('CORES'):
                    machine_.cpu_cores = int(line.split('=')[1])
                elif line.startswith('THREADS'):
                    machine_.cpu_threads = int(line.split('=')[1])

        logger.debug("Get CPU type...")
        output, stderr, exitstatus = conn.execute_script_remote(
            'machine_get_cpu_type.sh')
        if output and output[0]:
            machine_.cpu_model = output[0].strip()

        logger.debug("Get CPU flags...")
        output, stderr, exitstatus = conn.execute_script_remote(
            'machine_get_cpu_flags.sh')
        if output and output[0]:
            machine_.cpu_flags = output[0].strip()

        logger.debug("Get CPU speed...")
        output, stderr, exitstatus = conn.execute_script_remote(
            'machine_get_cpu_speed.sh')
        if output and output[0]:
            machine_.cpu_speed = Decimal(int(output[0].strip()) / 1000000)

        logger.debug("Get CPU ID...")
        output, stderr, exitstatus = conn.execute_script_remote(
            'machine_get_cpu_id.sh')
        if output and output[0]:
            machine_.cpu_id = output[0].strip()

        # EFI
        logger.debug("Check for EFI...")
        try:
            efi_file = conn.get_file('/sys/firmware/efi', 'r')
            efi_file.close()
            machine_.efi = True
        except IOError:
            machine_.efi = False

        # Memory
        logger.debug("Get RAM amount...")
        for line in conn.read_file('/proc/meminfo'):
            if line.startswith('MemTotal'):
                machine_.ram_amount = int(int(line.split()[1]) / 1024)

        # Virtualization capability
        VM_HOST_MIN_RAM_MB = 7000
        machine_.vm_capable = False

        # Virtualization: x86
        logger.debug("Check for VM capability...")
        if machine_.architecture_id == Architecture.Type.X86_64:
            cpu_flags = machine_.cpu_flags
            if cpu_flags:
                cpu_flags = cpu_flags.upper()
                if ((cpu_flags.find('VMX') >= 0 or cpu_flags.find('SVM') >= 0)
                        and int(machine_.ram_amount) > VM_HOST_MIN_RAM_MB):
                    machine_.vm_capable = True

        # Virtualization: ppc64le
        if machine_.architecture_id == Architecture.Type.PPC64LE:
            for line in conn.read_file('/proc/cpuinfo'):
                if line.startswith('firmware') and 'OPAL' in line:
                    machine_.vm_capable = True

        # Disk
        logger.debug("Get disk information...")
        stdout, stderr, exitstatus = conn.execute('hwinfo --disk')
        for line in stdout:
            line = line.strip()
            if line.startswith('Size:'):
                machine_.disk_primary_size = int(
                    int(line.split()[1]) / 2 / 1024**2)
            elif line.startswith('Attached to:'):
                opening_bracket = line.find('(')
                closing_bracket = line.find(')')
                if opening_bracket > 0 and closing_bracket > 0:
                    machine_.disk_type = line[opening_bracket +
                                              1:closing_bracket]
                else:
                    machine_.disk_type = 'Unknown disk type'
                break

        # lsmod
        logger.debug("Get 'lsmod'...")
        stdout, stderr, exitstatus = conn.execute('lsmod')
        machine_.lsmod = normalize_ascii("".join(stdout))

        # lspci
        logger.debug("Get 'lspci'...")
        stdout, stderr, exitstatus = conn.execute('lspci -vvv -nn')
        machine_.lspci = normalize_ascii("".join(stdout))

        # last
        logger.debug("Get 'last'...")
        output, stderr, exitstatus = conn.execute(
            'last | grep -v reboot | head -n 1')
        string = ''.join(output)
        result = string[0:8] + string[38:49]
        machine_.last = normalize_ascii("".join(result))

        # hwinfo
        logger.debug("Get 'hwinfo' (full)...")
        stdout, stderr, exitstatus = conn.execute(
            'hwinfo --bios ' +
            '--block --bridge --cdrom --cpu --disk --floppy --framebuffer ' +
            '--gfxcard --hub --ide --isapnp --isdn --keyboard --memory ' +
            '--monitor --mouse --netcard --network --partition --pci --pcmcia '
            + '--scsi --smp --sound --sys --tape --tv --usb --usb-ctrl --wlan')
        machine_.hwinfo = normalize_ascii("".join(stdout))

        # dmidecode
        logger.debug("Get 'dmidecode'...")
        stdout, stderr, exitstatus = conn.execute('dmidecode')
        machine_.dmidecode = normalize_ascii("".join(stdout))

        # dmesg
        logger.debug("Get 'dmesg'...")
        stdout, stderr, exitstatus = conn.execute(
            'if [ -e /var/log/boot.msg ]; then ' +
            'cat /var/log/boot.msg; else journalctl -xl | head -n200; ' + 'fi')
        machine_.dmesg = normalize_ascii("".join(stdout))

        # lsscsi
        logger.debug("Get 'lsscsi'...")
        stdout, stderr, exitstatus = conn.execute('lsscsi -s')
        machine_.lsscsi = normalize_ascii("".join(stdout))

        # lsusb
        logger.debug("Get 'lsusb'...")
        stdout, stderr, exitstatus = conn.execute('lsusb')
        machine_.lsusb = normalize_ascii("".join(stdout))

        # IPMI
        logger.debug("Check for IPMI...")
        machine_.ipmi = machine_.dmidecode.find('IPMI') >= 0

        # Firmware script
        logger.debug("Get BIOS version...")
        output, stderr, exitstatus = conn.execute_script_remote(
            'machine_get_firmware.sh')
        if output and output[0]:
            machine_.bios_version = output[0].strip()

        return machine_

    except Exception as e:
        logger.error("{} ({})".format(fqdn, e))
        return False
    finally:
        if conn:
            conn.close()
        if timer:
            timer.cancel()

    return None
Esempio n. 30
0
    def execute(self):
        """
        Executes the task.
        """
        if not ServerConfig.objects.bool_by_key('orthos.debug.motd.write'):
            logger.warning("Disabled: set 'orthos.debug.motd.write' to 'true'")
            return

        BEGIN = '-' * 69 + ' Orthos{ --'
        LINE = '-' * 80
        END = '-' * 69 + ' Orthos} --'

        try:
            machine = Machine.objects.get(fqdn=self.fqdn)
        except Machine.DoesNotExist:
            logger.error("Machine does not exist: fqdn={}".format(self.fqdn))
            return

        conn = None
        try:
            conn = SSH(machine.fqdn)
            conn.connect()
            motd = conn.get_file('/etc/motd.orthos', 'w')
            print(BEGIN, file=motd)
            print(
                "Machine of the ARCH team. Contact <{}> for problems.".format(
                    machine.get_support_contact()),
                file=motd)
            if machine.comment:
                print("INFO: " + machine.comment, file=motd)
            if machine.administrative:
                print(
                    "This machine is an administrative machine. DON\'T TOUCH!",
                    file=motd)
            if machine.reserved_by:
                print(LINE, file=motd)
                if machine.reserved_until != timezone.ZERO:
                    print("This machine is RESERVED by {} until {}.".format(
                        machine.reserved_by, machine.reserved_until),
                          file=motd)
                else:
                    print("This machine is RESERVED by {}.".format(
                        machine.reserved_by),
                          file=motd)
                print('', file=motd)
                print(wrap80(machine.reserved_reason), file=motd)
            print(END, file=motd)
            motd.close()
            stdout, stderr, exitstatus = conn.execute_script_remote(
                'machine_sync_motd.sh')

            if exitstatus != 0:
                logger.exception("({}) {}".format(machine.fqdn, stderr))
                raise Exception(stderr)

        except SSH.Exception as e:
            logger.error("({}) {}".format(machine.fqdn, e))
            return False
        except IOError as e:
            logger.error("({}) {}".format(machine.fqdn, e))
            return False
        finally:
            if conn:
                conn.close()
Esempio n. 31
0
def get_networkinterfaces(fqdn):
    """
    Retrieves information of the systems network interfaces.
    """
    try:
        machine = Machine.objects.get(fqdn=fqdn)
    except Machine.DoesNotExist:
        logger.warning("Machine '{}' does not exist".format(fqdn))
        return False

    conn = None
    timer = None
    try:
        conn = SSH(machine.fqdn)
        conn.connect()
        timer = threading.Timer(5 * 60, conn.close)
        timer.start()

        # Network interfaces
        logger.debug("Collect network interfaces...")
        stdout, stderr, exitstatus = conn.execute('hwinfo --network')
        interfaces = []
        interface = None

        for line in stdout:
            if len(line) > 0 and line[0] != ' ' and line[0] != '\t':
                if interface and interface.mac_address and\
                        interface.driver_module not in ('bridge', 'tun'):

                    interfaces.append(interface)
                interface = NetworkInterface()
            else:
                match = re.match(r'\s+Driver: "(\w+)"', line)
                if match:
                    interface.driver_module = match.group(1)
                    continue

                match = re.match(r'\s+SysFS ID: ([/\w.]+)', line)
                if match:
                    interface.sysfs = match.group(1)
                    continue

                match = re.match(r'\s+HW Address: ([0-9a-fA-F:]+)', line)
                if match:
                    interface.mac_address = match.group(1).upper()
                    continue

                match = re.match(r'\s+Device File: ([\w.]+)', line)
                if match:
                    interface.name = match.group(1)
                    continue

        if interface and interface.mac_address and\
                interface.driver_module not in ('bridge', 'tun'):

            interfaces.append(interface)

        for interface in interfaces:
            if interface.sysfs is None:
                continue

            path = '/sys/{}/type'.format(interface.sysfs)
            arp_type = ''.join(conn.read_file(path))

            if arp_type == ARPHRD_IEEE80211:
                continue

            stdout, stderr, exitstatus = conn.execute('ethtool %s' %
                                                      (interface.name))
            for line in stdout:
                match = re.match(r'\s+Port: (.+)', line)
                if match:
                    interface.ethernet_type = match.group(1)

        return interfaces

    except Exception as e:
        logger.error("{} ({})".format(fqdn, e))
        return False
    finally:
        if conn:
            conn.close()
        if timer:
            timer.cancel()

    return None
Esempio n. 32
0
def get_status_ip(fqdn):
    """
    Retrieves information of the systems IPv4/IPv6 status.
    """
    try:
        machine = Machine.objects.get(fqdn=fqdn)
    except Machine.DoesNotExist:
        logger.warning("Machine '{}' does not exist".format(fqdn))
        return False

    machine_ = Machine()

    conn = None
    timer = None
    try:
        conn = SSH(machine.fqdn)
        conn.connect()
        timer = threading.Timer(5 * 60, conn.close)
        timer.start()

        logger.debug("Check IPv4/IPv6 status...")
        stdout, stderr, exitstatus = conn.execute('/sbin/ip a')

        devices = {}
        current_device = None
        addresses = {'inet': [], 'inet6': []}

        for line in stdout:
            match = re.match(r'^\d+:\s+([a-zA-Z0-9]+):\s+<.*>\s(.*)\n', line)
            if match:
                current_device = match.group(1)
                devices[current_device] = {
                    'mac_address': None,
                    'inet': None,
                    'inet6': None,
                    'flags': None
                }
                devices[current_device]['flags'] = match.group(2).split()
                continue

            line = line.lstrip()

            match = re.match(r'inet ([0-9.]{7,15})\/.*scope', line)
            if match:
                if devices[current_device]['inet'] is None:
                    devices[current_device]['inet'] = []
                devices[current_device]['inet'].append(match.group(1))
                continue

            match = re.match(r'inet6 ([a-f0-9:]*)\/[0-9]+ scope', line)
            if match:
                if devices[current_device]['inet6'] is None:
                    devices[current_device]['inet6'] = []
                devices[current_device]['inet6'].append(match.group(1))
                continue

            match = re.match('link/ether ([a-f0-9:]{17}) brd', line)
            if match:
                devices[current_device]['mac_address'] = match.group(1).upper()

        for device, values in devices.items():
            if values['mac_address'] is None:
                continue

            # ignore device if hooking up another
            if any(device in values['flags'] for device in devices.keys()):
                continue

            if values['mac_address'] == machine.mac_address:
                if values['inet'] is None:
                    machine_.status_ipv4 = Machine.StatusIP.AF_DISABLED
                elif machine.ipv4 not in values['inet']:
                    machine_.status_ipv4 = Machine.StatusIP.NO_ADDRESS
                    if [
                            ipv4 for ipv4 in values['inet']
                            if not ipv4.startswith('127.0.0.1')
                    ]:
                        machine_.status_ipv4 = Machine.StatusIP.ADDRESS_MISMATCH
                elif machine.ipv4 in values['inet']:
                    machine_.status_ipv4 = Machine.StatusIP.CONFIRMED
                else:
                    machine_.status_ipv4 = Machine.StatusIP.MISSING

                if values['inet6'] is None:
                    machine_.status_ipv6 = Machine.StatusIP.AF_DISABLED
                elif machine.ipv6 not in values['inet6']:
                    machine_.status_ipv6 = Machine.StatusIP.NO_ADDRESS
                    if [
                            ipv6 for ipv6 in values['inet6']
                            if not ipv6.startswith('fe80::')
                    ]:
                        machine_.status_ipv6 = Machine.StatusIP.ADDRESS_MISMATCH
                elif machine.ipv6 in values['inet6']:
                    machine_.status_ipv6 = Machine.StatusIP.CONFIRMED

            addresses['inet'].append(values['inet'])
            addresses['inet6'].append(values['inet6'])

        if machine_.status_ipv4 == Machine.StatusIP.NO_ADDRESS:
            if machine.ipv4 in addresses['inet']:
                machine_.status_ipv4 = Machine.StatusIP.MAC_MISMATCH

        if machine_.status_ipv6 == Machine.StatusIP.NO_ADDRESS:
            if machine.ipv6 in addresses['inet6']:
                machine_.status_ipv6 = Machine.StatusIP.MAC_MISMATCH

        return machine_

    except Exception as e:
        logger.error("{} ({})".format(fqdn, e))
        return False
    finally:
        if conn:
            conn.close()
        if timer:
            timer.cancel()

    return None
Esempio n. 33
0
def get_pci_devices(fqdn):
    """
    Retrieves all PCI devices.
    """
    def get_pci_device_by_slot(pci_devices, slot):
        """
        Returns the PCI device by slot.
        """
        for dev in pci_devices:
            pci_slot = dev.slot
            if not pci_slot:
                continue
            pci_slot = pci_slot.strip()
            # pci domain hacks
            if len(pci_slot) < 8:
                pci_slot = '0000:' + pci_slot
            if len(slot) < 8:
                slot = '0000:' + slot
            if pci_slot == slot:
                return dev
        return None

    from data.models import PCIDevice

    try:
        machine = Machine.objects.get(fqdn=fqdn)
    except Machine.DoesNotExist:
        logger.warning("Machine '{}' does not exist".format(fqdn))
        return False

    conn = None
    timer = None
    try:
        conn = SSH(fqdn)
        conn.connect()
        timer = threading.Timer(5 * 60, conn.close)
        timer.start()

        logger.debug("Collect PCI devices for '{}'...".format(machine.fqdn))
        pci_devices = []
        chunk = ''
        stdout, stderr, exitstatus = conn.execute('lspci -mmvn')
        for line in stdout:
            if len(line.strip()) == 0:
                pci_devices.append(PCIDevice.from_lspci_mmnv(chunk))
                chunk = ''
            else:
                chunk += line

        # drivers for PCI devices from hwinfo
        in_pci_device = False
        current_busid = None

        if machine.hwinfo:
            for line in machine.hwinfo.splitlines():
                if re.match(r'^\d+: PCI', line):
                    in_pci_device = True
                    continue
                if len(line.strip()) == 0:
                    in_pci_device = False
                    current_busid = None
                    continue
                if not in_pci_device:
                    continue
                match = re.match(r'  SysFS BusID: ([0-9a-fA-F.:]+)', line)
                if match:
                    current_busid = match.group(1)
                match = re.match(r'  Driver: "([^"]*)"', line)
                if match and current_busid:
                    pcidev = get_pci_device_by_slot(pci_devices, current_busid)
                    if pcidev:
                        pcidev.drivermodule = match.group(1)
                match = re.match(r'  Driver Modules: "([^"]*)"', line)
                if match and current_busid:
                    pcidev = get_pci_device_by_slot(pci_devices, current_busid)
                    if pcidev:
                        pcidev.drivermodule = match.group(1)

        for pci_device in pci_devices:
            pci_device.machine = machine

        logger.debug("Collected {} PCI devices for '{}'".format(
            len(pci_devices), machine.fqdn))

        return pci_devices

    except Exception as e:
        logger.exception("{} ({})".format(fqdn, e))
        return False
    finally:
        if conn:
            conn.close()
        if timer:
            timer.cancel()
Esempio n. 34
0
    def _perform(self, action):
        """
        Common implementation for on, off and reset.
        """
        name = self.machine.hostname
        virsh = 'virsh -c lxc:///'
        conn = None
        result = False

        if not self.machine.hypervisor.fqdn:
            logger.error("No hypervisor system found")
            raise Exception("No hypervisor found")

        conn = SSH(self.machine.hypervisor.fqdn)
        conn.connect()

        if action == 'status':

            stdout, stderr, exitstatus = conn.execute(
                '{} list --all'.format(virsh))

            if exitstatus != 0:
                logger.error(''.join(stderr))
                raise Exception(''.join(stderr))

            for line in stdout[2:]:
                columns = line.strip().split()

                if columns[1] == name:
                    return {
                        'running': self.Status.ON,
                        'shut': self.Status.OFF,
                        'paused': self.Status.PAUSED
                    }.get(columns[2], 0)

            raise Exception("Couldn't find domain '{}'!".format(name))

        elif action == 'off':
            stdout, stderr, exitstatus = conn.execute('{} destroy {}'.format(
                virsh, name))

            if exitstatus == 0:
                logger.debug("Virtual machine '{}' stopped".format(name))
                result = True
            else:
                logger.error(''.join(stderr))
                raise Exception(''.join(stderr))

        elif action == 'on':
            stdout, stderr, exitstatus = conn.execute('{} start {}'.format(
                virsh, name))

            if exitstatus == 0:
                logger.debug("Virtual machine '{}' started".format(name))
                result = True
            else:
                logger.error(''.join(stderr))
                raise Exception(''.join(stderr))

        else:
            logger.warning("Action '{}' does not exist".format(action))
            result = False

        if conn:
            conn.close()

        return result
Esempio n. 35
0
    def execute(self):
        """
        Executes the task.
        """
        from data.models import Machine, SerialConsole

        if not ServerConfig.objects.bool_by_key(
                'orthos.debug.serialconsole.write'):
            logger.warning(
                "Disabled: set 'orthos.debug.serialconsole.write' to 'true'")
            return

        try:
            cscreen_server = Machine.objects.get(fqdn=self.fqdn)
        except Machine.DoesNotExist:
            logger.warning("Serial console server does not exist: {}".format(
                self.fqdn))

        conn = None
        try:
            conn = SSH(cscreen_server.fqdn)
            conn.connect()

            stdout, stderr, exitstatus = conn.execute(
                'sudo touch /etc/cscreenrc_allow_update')

            if exitstatus != 0:
                raise Exception(
                    "Couldn't lock cscreen ('touch /etc/cscreenrc_allow_update')"
                )

            new_content = ''
            for serialconsole in SerialConsole.cscreen.get(cscreen_server):
                new_content += serialconsole.get_comment_record() + '\n'
                new_content += serialconsole.get_command_record() + '\n'

            screenrc_file = '/etc/cscreenrc'

            # create `/etc/cscreenrc` if it doesn't exist
            stdout, stderr, exitstatus = conn.execute(
                '[ -e "{}"]'.format(screenrc_file))

            orthos_inline_begin = ServerConfig.objects.by_key(
                'orthos.configuration.inline.begin')
            orthos_inline_end = ServerConfig.objects.by_key(
                'orthos.configuration.inline.end')

            if exitstatus != 0:
                stdout, stderr, exitstatus = conn.execute(
                    'echo "{}\n{}" > {}'.format(orthos_inline_begin,
                                                screenrc_file,
                                                orthos_inline_end))

            if exitstatus != 0:
                raise Exception("Couldn't create CScreen file ('{}')".format(
                    screenrc_file))

            # Save backup file which is used later by an invoked script
            # to determine the changes and update the running screen
            # session (add, remove or restart modified entries).
            stdout, stderr, exitstatus = conn.execute(
                'sudo cp {} {}.old'.format(screenrc_file, screenrc_file))

            cscreen = conn.get_file(screenrc_file, 'r')
            buffer = ''

            in_replace = False
            for line in cscreen.readlines():
                if not in_replace and line.startswith(orthos_inline_begin):
                    buffer += line + new_content
                    in_replace = True
                elif in_replace and line.startswith(orthos_inline_end):
                    buffer += line
                    in_replace = False
                elif not in_replace:
                    buffer += line

            cscreen.close()

            cscreen = conn.get_file(screenrc_file, 'w')
            buffer = buffer.strip('\n')
            print(buffer, file=cscreen)
            cscreen.close()

            stdout, stderr, exitstatus = conn.execute(
                'sudo /usr/bin/cscreen -u')
            logger.info("CScreen update exited with: {}".format(exitstatus))

            stdout, stderr, exitstatus = conn.execute(
                'sudo rm -f /etc/cscreenrc_allow_update')

            if exitstatus != 0:
                raise Exception(
                    "Couldn't unlock CScreen ('rm /etc/cscreenrc_allow_update')"
                )

        except SSH.Exception as exception:
            logger.error(exception)
        except IOError as exception:
            logger.error(exception)
        finally:
            if conn:
                conn.close()
Esempio n. 36
0
class CobblerServer:

    def __init__(self, fqdn, domain):
        self._fqdn = fqdn
        self._conn = None
        self._domain = domain
        self._cobbler_path = ServerConfig.objects.by_key("cobbler.command")

    def connect(self):
        """
        Connect to DHCP server via SSH.
        """
        if not self._conn:
            self._conn = SSH(self._fqdn)
            self._conn.connect()

    def close(self):
        """
        Close connection to DHCP server.
        """
        if self._conn:
            self._conn.close()

    def deploy(self):
        self.connect()
        if not self.is_installed():
            raise SystemError("No Cobbler service found: {}".format(self._fqdn))

        machines = Machine.active_machines.filter(fqdn_domain=self._domain.pk)
        cobbler_machines = self.get_machines()
        cobbler_commands = []
        for machine in machines:
            if machine.fqdn in cobbler_machines:
                cobbler_commands.append(get_cobbler_update_command(machine, self._cobbler_path))
            else:
                cobbler_commands.append(get_cobbler_add_command(machine, self._cobbler_path))
        for command in cobbler_commands:  # TODO: Convert this to a single ssh call (performance)
            _, stderr, exitcode = self._conn.execute(command)
            if exitcode:
                logger.error("failed to execute %s on %s", command, self._fqdn)

        self.close()

    def is_installed(self):
        """
        Check if Cobbler server is available.
        """
        if self._conn.check_path(self._cobbler_path, '-x'):
            return True
        return False

    def is_running(self):
        """
        Check if the Cobbler daemon is running via the cobbler version command
        """

        command = f"{self._cobbler_path} version"
        _, _, exitstatus = self._conn.execute(command)
        if exitstatus == 0:
            return True
        return False

    def get_machines(self):
        stdout, stderr, exitstatus = self._conn.execute(
            "{cobbler} system list".format(cobbler=self._cobbler_path))
        if exitstatus:
            logger.warning("system list failed on %s with %s", self._fqdn, stderr)
            raise CobblerException("system list failed on {server}".format(server=self._fqdn))
        clean_out = [system.strip(' \n\t') for system in stdout]
        return clean_out
Esempio n. 37
0
class Libvirt(VirtualizationAPI):

    class Meta:
        proxy = True

    VIRSH = 'virsh -c qemu:///system'
    IGNORE_STDERR = [
        'domain is not running',
        'no domain with matching name'
    ]
    QEMU_IMG_CONVERT = '/usr/bin/qemu-img convert -O qcow2 -o preallocation=metadata {0}.tmp {0}'

    def __init__(self):
        self.conn = None

    def get_image_list(self):
        """
        Returns the available architectures and the full image list (over all available
        architectures).

        Return format:
            (
                ['<arch1>', '<arch2>', ...], [('<value>', '<option>'), ...]
            )
        """
        from data.models import ServerConfig

        architectures = [self.host.architecture.name]
        image_directory = ServerConfig.objects.by_key('virtualization.libvirt.images.directory')
        image_list = []

        try:
            for architecture in architectures:
                directory = '{}/{}/'.format(image_directory.rstrip('/'), architecture)
                for image in os.listdir(directory):
                    path = directory + image
                    size = os.path.getsize(path)
                    atime = str(date.fromtimestamp(os.path.getmtime(path)))

                    if size < (1024**3):
                        size = int(size / (1024**2))
                        size = '{}M'.format(size)
                    else:
                        size = int(size / (1024**3))
                        size = '{}G'.format(size)

                    pretty_image = image.split('.')[0]

                    image_list.append((image, '{} ({} {})'.format(pretty_image, atime, size)))

        except FileNotFoundError as e:
            logger.exception(e)

        return (architectures, image_list)

    def connect(function):
        """
        Create SSH connection if needed.
        """
        def decorator(self, *args, **kwargs):
            from utils.ssh import SSH

            if not self.conn:
                self.conn = SSH(self.host.fqdn)
                self.conn.connect()
            return function(self, *args, **kwargs)

        return decorator

    @connect
    def _execute(self, command):
        return self.conn.execute(command)

    def check_connection(self):
        """
        Check libvirt connection (running libvirt).
        """
        stdout, stderr, exitstatus = self._execute('{} version'.format(self.VIRSH))
        if exitstatus == 0:
            return True
        return False

    def get_list(self, parameters='--all'):
        """
        Return `virsh list` output.
        """
        stdout, stderr, exitstatus = self._execute('{} list {}'.format(self.VIRSH, parameters))

        if exitstatus == 0:
            return ''.join(stdout)
        else:
            raise Exception(''.join(stderr))

        return False

    def check_network_bridge(self, bridge='br0'):
        """
        Execute `create_bridge.sh` script remotely and try to set up bridge if it doesn't exist.
        Returns true if the bridge is available, false otherwise.
        """
        stdout, stderr, exitstatus = self.conn.execute_script_remote('create_bridge.sh')

        if exitstatus != 0:
            raise Exception(''.join(stderr))

        stdout, stderr, exitstatus = self.conn.execute('brctl show')

        if exitstatus != 0:
            raise Exception(''.join(stderr))

        for line in stdout:
            if line.startswith(bridge):
                return True

        raise False

    def generate_hostname(self):
        """
        Generate domain name (hostname). Check hostnames against Orthos machines and libvirt
        `virsh list`.
        """
        hostname = None
        occupied_hostnames = set(vm.hostname for vm in self.host.get_virtual_machines())

        libvirt_list = self.get_list()
        for line in libvirt_list.split('\n')[2:]:
            columns = line.strip().split()
            if len(columns) > 0:
                domain_name = columns[1]
                occupied_hostnames.add(domain_name)

        for i in range(1, self.host.vm_max + 1):
            hostname_ = '{}-{}'.format(self.host.hostname, i)
            if hostname_ not in occupied_hostnames:
                hostname = hostname_
                break

        if hostname is None:
            raise Exception("All hostnames (domain names) busy!")

        return hostname

    def generate_networkinterfaces(self, amount=1, bridge='br0', model='virtio'):
        """
        Generate networkinterfaces.
        """
        from data.models import NetworkInterface

        networkinterfaces = []
        for i in range(amount):
            mac_address = get_random_mac_address()
            while NetworkInterface.objects.filter(mac_address=mac_address).count() != 0:
                mac_address = get_random_mac_address()

            networkinterface = NetworkInterface(mac_address=mac_address)
            networkinterface.bridge = bridge
            networkinterface.model = model

            networkinterfaces.append(networkinterface)

        return networkinterfaces

    def copy_image(self, image, disk_image):
        """
        Copy and allocate disk image.
        """
        stdout, stderr, exitstatus = self.conn.execute('cp {} {}.tmp'.format(image, disk_image))

        if exitstatus != 0:
            return False

        stdout, stderr, exitstatus = self.conn.execute(self.QEMU_IMG_CONVERT.format(disk_image))

        if exitstatus != 0:
            return False

        stdout, stderr, exitstatus = self.conn.execute('rm -rf {}.tmp'.format(disk_image))

        if exitstatus != 0:
            return False

        return True

    def delete_disk_image(self, disk_image):
        """
        Delete the old disk image.
        """
        stdout, stderr, exitstatus = self.conn.execute('rm -rf {}'.format(disk_image))

        if exitstatus != 0:
            return False

        return True

    def calculate_vcpu(self):
        """
        Return virtual CPU amount.
        """
        vcpu = 1

        host_cpu_cores = self.host.cpu_cores
        if host_cpu_cores is not None:
            vcpu = int((host_cpu_cores - 2) / self.host.vm_max)
            if vcpu == 0:
                vcpu = 1

        return vcpu

    def check_memory(self, memory_amount):
        """
        Check if memory amount for VM is available on host. Reserve 2GB of memory for host system.
        """
        host_ram_amount = self.host.ram_amount
        host_reserved_ram_amount = 2048

        if host_ram_amount:
            if memory_amount > (host_ram_amount - host_reserved_ram_amount):
                raise Exception("Host system has only {}MB of memory!".format(memory_amount))
        else:
            raise Exception("Can't detect memory size of host system '{}'".format(self.host))

        return True

    def execute_virt_install(self, *args, dry_run=True, **kwargs):
        """
        Run `virt-install` command.
        """
        command = '/usr/bin/virt-install '
        command += '--name {hostname} '
        command += '--vcpus {vcpu} '
        command += '--memory {memory} '

        disk_ = '--disk {},'.format(kwargs['disk']['image'])
        disk_ += 'size={},'.format(kwargs['disk']['size'])
        disk_ += 'format={},'.format(kwargs['disk']['format'])
        disk_ += 'sparse={},'.format(kwargs['disk']['sparse'])
        disk_ += 'bus={} '.format(kwargs['disk']['bus'])
        command += disk_

        for networkinterface in kwargs.get('networkinterfaces', []):
            networkinterface_ = '--network model={},'.format(networkinterface.model)
            networkinterface_ += 'bridge={},'.format(networkinterface.bridge)
            networkinterface_ += 'mac={} '.format(networkinterface.mac_address)
            command += networkinterface_

        command += '{boot} '

        vnc = kwargs.get('vnc', None)
        if vnc and vnc['enabled']:
            command += '--graphics vnc,listen=0.0.0.0,port={} '.format(vnc['port'])

        command += kwargs.get('parameters', '')

        if dry_run:
            command += '--dry-run'

        command = command.format(**kwargs)
        logger.debug(command)
        stdout, stderr, exitstatus = self.conn.execute(command)

        if exitstatus != 0:
            raise Exception(''.join(stderr))

        return True

    def _create(self, vm, *args, **kwargs):
        """
        Wrapper function for creating a VM.

        Steps:
            - check connection to host
            - check maxinmum VM number limit
            - check network bridge
            - check image source directory (if needed)
            - check Open Virtual Machine Firmware (OVMF) binary (if needed)
            - check memory size
            - generate hostname (=domain name)
            - copy image to disk image (if needed)
            - run `virt-install`
        """
        from data.models import ServerConfig
        from data.models import NetworkInterface

        bridge = ServerConfig.objects.by_key('virtualization.libvirt.bridge')
        image_directory = ServerConfig.objects.by_key('virtualization.libvirt.images.directory')
        disk_image_directory = '/abuild/orthos-vm-images/'
        disk_image = '{}/{}.qcow2'.format(disk_image_directory.rstrip('/'), '{}')
        ovmf = ServerConfig.objects.by_key('virtualization.libvirt.ovmf.path')

        image_directory = '{}/{}/'.format(
            image_directory.rstrip('/'),
            kwargs['architecture']
        )

        if not self.check_connection():
            raise Exception("Host system not reachable!")

        if self.host.get_virtual_machines().count() >= self.host.vm_max:
            raise Exception("Maximum number of VMs reached!")

        if not self.check_network_bridge(bridge=bridge):
            raise Exception("Network bridge setup failed!")

        if not kwargs['image'] is None:
            if not self.conn.check_path(image_directory, '-e'):
                raise Exception("Image source directory missing on host system!")

        if not self.conn.check_path(disk_image_directory, '-w'):
            raise Exception(
                "Image disk directory missing on host system: {}!".format(disk_image_directory)
            )

        if kwargs['uefi_boot']:
            if not self.conn.check_path(ovmf, '-e'):
                raise Exception("OVMF file not found: '{}'!".format(ovmf))
            boot = '--boot loader={},network'.format(ovmf)
        else:
            boot = '--boot network,hd,menu=off'

        self.check_memory(kwargs['ram_amount'])

        vm.hostname = self.generate_hostname()
        vm.fqdn = '{}.{}'.format(vm.hostname, self.host.fqdn_domain.name)

        vnc_port = 5900 + int(vm.hostname.split('-')[1])
        vm.vnc = {
            'enabled': kwargs['vnc'],
            'port': vnc_port
        }

        vm.cpu_cores = self.calculate_vcpu()

        vm.ram_amount = kwargs['ram_amount']

        disk_image = disk_image.format(vm.hostname)

        if kwargs['image'] is not None:
            image = '{}/{}'.format(image_directory.rstrip('/'), kwargs['image'])

            if not self.copy_image(image, disk_image):
                raise Exception("Couldn't copy image: {} > {}!".format(image, disk_image))
        else:
            self.delete_disk_image(disk_image)

        disk = {
            'image': disk_image,
            'size': kwargs['disk_size'],
            'format': 'qcow2',
            'sparse': True,
            'bus': 'virtio'
        }

        networkinterfaces = self.generate_networkinterfaces(
            amount=kwargs['networkinterfaces'],
            bridge=bridge
        )

        parameters = '--events on_reboot=restart,on_poweroff=destroy '
        parameters += '--import '
        parameters += '--noautoconsole '
        parameters += '--autostart '
        parameters += kwargs['parameters']

        self.execute_virt_install(
            hostname=vm.hostname,
            vcpu=vm.cpu_cores,
            memory=vm.ram_amount,
            disk=disk,
            networkinterfaces=networkinterfaces,
            boot=boot,
            vnc=vm.vnc,
            parameters=parameters,
            dry_run=False
        )

        vm.unsaved_networkinterfaces = []
        for networkinterface in networkinterfaces:
            vm.unsaved_networkinterfaces.append(networkinterface)

        return True

    def _remove(self, vm):
        """
        Wrapper function for removing a VM (destroy domain > undefine domain).
        """
        if not self.check_connection():
            raise Exception("Host system not reachable!")

        self.destroy(vm)
        self.undefine(vm)
        return True

    def destroy(self, vm):
        """
        Destroy VM on host system. Ignore `domain is not running` error and proceed.
        """
        stdout, stderr, exitstatus = self._execute('{} destroy {}'.format(self.VIRSH, vm.hostname))
        if exitstatus != 0:
            stderr = ''.join(stderr)

            if not any(line in stderr for line in self.IGNORE_STDERR):
                raise Exception(stderr)

        return True

    def undefine(self, vm):
        """
        Undefine VM on host system.
        """
        stdout, stderr, exitstatus = self._execute('{} undefine {}'.format(self.VIRSH, vm.hostname))
        if exitstatus != 0:
            stderr = ''.join(stderr)

            if not any(line in stderr for line in self.IGNORE_STDERR):
                raise Exception(stderr)

        return True