Example #1
0
def do_connect(host, j=False, f=False):
    nodeset = ClusterShell.NodeSet.NodeSet(host)
    if (len(nodeset) != 1):
        clara_exit('Only one host allowed for this command')

    pat = re.compile("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}")
    if pat.match(host):
        logging.debug(
            "The host is an IP adddres: {0}. Using ipmitool without conman.".
            format(host))
        do_connect_ipmi(host)
    else:
        conmand = get_from_config("ipmi", "conmand")
        port = int(get_from_config("ipmi", "port"))
        if (len(conmand) == 0):
            clara_exit("You must set the configuration paramenter 'conmand'")
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            s.connect((conmand, port))
            os.environ["CONMAN_ESCAPE"] = '!'

            cmd = ["conman"]
            if j:
                cmd = cmd + ["-j"]
            if f:
                cmd = cmd + ["-f"]
            cmd = cmd + ["-d", conmand, host]
            run(cmd)
        except socket.error as e:
            logging.debug(
                "Conman not running. Message on connect: Errno {0} - {1}".
                format(e.errno, e.strerror))
            do_connect_ipmi(host)

        s.close()
Example #2
0
def extract_image():
    squashfs_file = get_from_config("images", "trg_img", dist)
    if not os.path.isfile(squashfs_file):
        sys.exit("The image {0} does not exist!".format(squashfs_file))

    print "Extracting {0} to {1} ...".format(squashfs_file, work_dir)
    run(["unsquashfs", "-f", "-d", work_dir, squashfs_file])
Example #3
0
def do_connect(host, j=False, f=False):
    nodeset = ClusterShell.NodeSet.NodeSet(host)
    if (len(nodeset) != 1):
        clara_exit('Only one host allowed for this command')

    pat = re.compile("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}")
    if pat.match(host):
        logging.debug("The host is an IP adddres: {0}. Using ipmitool without conman.".format(host))
        ipmi_do(host, True, "sol", "activate")
    else:
        conmand = get_from_config("ipmi", "conmand")
        port = int(get_from_config("ipmi", "port"))
        if (len(conmand) == 0):
            clara_exit("You must set the paramenter 'conmand' in the configuration file")
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            s.connect((conmand, port))
            os.environ["CONMAN_ESCAPE"] = '!'

            cmd = ["conman"]
            if j:
                cmd = cmd + ["-j"]
            if f:
                cmd = cmd + ["-f"]
            cmd = cmd + ["-d", conmand, host]
            run(cmd)
        except socket.error as e:
            logging.debug("Conman not running. Message on connect: {0}" % e)
            ipmi_do(host, True, "sol", "activate")

        s.close()
Example #4
0
def do_reprepro(action, package=None, flags=None):
    repo_dir = get_from_config("repo", "repo_dir", dist)
    reprepro_config = repo_dir + '/conf/distributions'

    if not os.path.isfile(reprepro_config):
        clara_exit(
            "There is not configuration for the local repository for {0}. Run first 'clara repo init <dist>'"
            .format(dist))

    list_flags = ['--silent', '--ask-passphrase']
    if conf.ddebug:
        list_flags = ['-V', '--ask-passphrase']

    if flags is not None:
        list_flags.append(flags)

    cmd = ['reprepro'] + list_flags + \
         ['--basedir', get_from_config("repo", "repo_dir", dist),
         '--outdir', get_from_config("repo", "mirror_local", dist),
         action, dist]

    if package is not None:
        cmd.append(package)

    run(cmd)
Example #5
0
def mktorrent(image):
    if (image is None):
        squashfs_file = get_from_config("images", "trg_img", dist)
    else:
        squashfs_file = image
    seeders = get_from_config("p2p", "seeders", dist)
    trackers_port = get_from_config("p2p", "trackers_port", dist)
    trackers_schema = get_from_config("p2p", "trackers_schema", dist)
    seeding_service = get_from_config("p2p", "seeding_service", dist)
    init_stop = get_from_config("p2p", "init_stop", dist)
    init_start = get_from_config("p2p", "init_start", dist)

    # trackers is a dictionary with pairs nodeset and torrent file
    trackers = {}
    for e in get_from_config("p2p", "trackers", dist).split(";"):
        k, v = e.split(":")
        trackers[k] = v

    if not os.path.isfile(squashfs_file):
        clara_exit("The file {0} doesn't exist".format(squashfs_file))

    # Remove old torrent files
    for f in trackers.values():
        if os.path.isfile(f):
            os.remove(f)

    clush(seeders, init_stop.format(seeding_service))

    for e in trackers.keys():
        announce = []
        for t in list(ClusterShell.NodeSet.NodeSet(e)):
            announce.append("{0}://{1}:{2}/announce".format(trackers_schema, t, trackers_port))
        run(["/usr/bin/mktorrent", "-a", ",".join(announce), "-o", trackers[e], squashfs_file])

    clush(seeders, init_start.format(seeding_service))
def genimg(image):
    if (image is None):
        squashfs_file = get_from_config("images", "trg_img", dist)
    else:
        path_to_image = os.path.dirname(image)
        if not os.path.isdir(path_to_image) and len(path_to_image) != 0:
            os.makedirs(path_to_image)
        squashfs_file = image

    if os.path.isfile(squashfs_file):
        os.rename(squashfs_file, squashfs_file + ".old")
        logging.info("Previous image renamed to {0}.".format(squashfs_file +
                                                             ".old"))

    logging.info("Creating image at {0}".format(squashfs_file))
    run_chroot(["chroot", work_dir, "apt-get", "clean"])
    if conf.ddebug:
        run([
            "mksquashfs", work_dir, squashfs_file, "-no-exports", "-noappend",
            "-info"
        ])
    else:
        run([
            "mksquashfs", work_dir, squashfs_file, "-no-exports", "-noappend"
        ])

    os.chmod(squashfs_file, 0o755)
Example #7
0
def do_init():
    repo_dir = get_from_config("repo", "repo_dir", dist)
    reprepro_config = repo_dir + '/conf/distributions'

    if not os.path.isfile(reprepro_config):
        if not os.path.isdir(repo_dir + '/conf'):
            os.makedirs(repo_dir + '/conf')

        freprepro = open(reprepro_config, 'w')
        freprepro.write("""Origin: {0}
Label: {1}
Suite: {1}
Codename: {1}
Version: {2}
Architectures: amd64 source
Components: main contrib non-free
UDebComponents: main
SignWith: {3}
Description: Depot Local {4}
DebIndices: Packages Release . .gz .bz2
DscIndices: Sources Release . .gz .bz2
""".format(get_from_config("common", "origin", dist),
            dist,
            get_from_config("repo", "version", dist),
            get_from_config("repo", "gpg_key", dist),
            get_from_config("repo", "clustername", dist)))
        freprepro.close()

    os.chdir(repo_dir)
    run(['reprepro', '--ask-passphrase', '--basedir', repo_dir,
         '--outdir', get_from_config("repo", "mirror_local", dist),
         'export', dist])
Example #8
0
def umount_chroot():
    run(["chroot", work_dir, "umount", "/sys"])
    run(["chroot", work_dir, "umount", "/proc"])
    time.sleep(1)  # Wait one second so the system has time to unmount
    with open("/proc/mounts", "r") as file_to_read:
        for line in file_to_read:
            if work_dir in line:
                sys.exit("Something went wrong when umounting in the chroot")
Example #9
0
def ipmi_do(hosts, cmd):
    install_cfg()
    imm_user = value_from_file(get_from_config("common", "master_passwd_file"), "IMMUSER")
    os.environ["IPMI_PASSWORD"] = value_from_file(get_from_config("common", "master_passwd_file"), "PASSWD")
    nodeset = ClusterShell.NodeSet.NodeSet(hosts)
    for host in nodeset:
        print "%s: " % host
        run(["ipmitool", "-I", "lanplus", "-H", "imm" + host,
             "-U", imm_user, "-E", cmd])
Example #10
0
def ipmi_do(hosts, cmd, pty=False):
    nodeset = ClusterShell.NodeSet.NodeSet(hosts)
    for host in nodeset:
        ipmitool = ["ipmitool", "-I", "lanplus", "-H", "imm" + host, "-U", imm_user, "-E", "-e!"]
        ipmitool.extend(cmd)
        if pty:
            run(ipmitool)
        else:
            os.system("echo -n '%s: ' ;" % host + " ".join(ipmitool))
Example #11
0
def geninitrd(path):
    if (path is None):
        trg_dir = get_from_config("images", "trg_dir", dist)
    else:
        trg_dir = path

    if not os.path.isdir(trg_dir):
        os.makedirs(trg_dir)

    squashfs_file = get_from_config("images", "trg_img", dist)
    if not os.path.isfile(squashfs_file):
        clara_exit("The image {0} does not exist!".format(squashfs_file))

    if conf.ddebug:
        run(["unsquashfs", "-li", "-f", "-d", work_dir, squashfs_file])
    else:
        run(["unsquashfs", "-f", "-d", work_dir, squashfs_file])

    mount_chroot()

    # Install the kernel in the image
    kver = get_from_config("images", "kver", dist)
    if len(kver) == 0:
        clara_exit("kver hasn't be set in config.ini")
    else:
        run_chroot(["chroot", work_dir, "apt-get", "update"])
        run_chroot([
            "chroot", work_dir, "apt-get", "install",
            "--no-install-recommends", "--yes", "--force-yes",
            "linux-image-" + kver
        ])

    # Install packages from 'packages_initrd'
    packages_initrd = get_from_config("images", "packages_initrd", dist)
    if len(packages_initrd) == 0:
        logging.warning("packages_initrd hasn't be set in config.ini")
    else:
        pkgs = packages_initrd.split(',')
        run_chroot([
            "chroot", work_dir, "apt-get", "install",
            "--no-install-recommends", "--yes", "--force-yes"
        ] + pkgs)

    # Generate the initrd in the image
    run_chroot(
        ["chroot", work_dir, "mkinitramfs", "-o", "/tmp/initrd-" + kver, kver])
    umount_chroot()

    # Copy the initrd out of the chroot
    shutil.copy(work_dir + "/tmp/initrd-" + kver, trg_dir + "/initrd-" + kver)
    os.chmod(trg_dir + "/initrd-" + kver, 0o644)
    logging.info("Initrd available at " + trg_dir + "/initrd-" + kver)

    # Copy vmlinuz out of the chroot
    shutil.copy(work_dir + "/boot/vmlinuz-" + kver,
                trg_dir + "/vmlinuz-" + kver)
    logging.info("vmlinuz available at " + trg_dir + "/vmlinuz-" + kver)
Example #12
0
def do_push(dist=''):
    push_cmd = get_from_config_or("repo", "push", dist, None)
    if push_cmd:
        push_hosts = get_from_config_or("repo", "hosts", dist, '').split(',')
        if push_hosts and push_hosts[0] is not '':
            for host in push_hosts:
                run(push_cmd.format(host).split(' '))
        else:
            run(push_cmd.split(' '))
Example #13
0
def genimg():
    squashfs_file = get_from_config("images", "trg_img", dist)
    if os.path.isfile(squashfs_file):
        os.rename(squashfs_file, squashfs_file + ".old")
        print("Previous image renamed to {0}.".format(squashfs_file + ".old"))

    print "Creating image at {0}".format(squashfs_file)
    run_chroot(["chroot", work_dir, "apt-get", "clean"])
    run(["mksquashfs", work_dir, squashfs_file, "-no-exports", "-noappend"])
    os.chmod(squashfs_file, 0o755)
Example #14
0
def genimg(image, work_dir, dist):
    if (image is None):
        squashfs_file = get_from_config("images", "trg_img", dist)
        if (squashfs_file == "" or squashfs_file == None):
            image_name = dist + "_image.squashfs"
            squashfs_file = "/var/lib/clara/images/" + image_name
            if os.path.isfile(squashfs_file):
                os.rename(squashfs_file, squashfs_file + ".old")
                logging.info(
                    "Previous image renamed to {0}.".format(squashfs_file +
                                                            ".old"))

    else:
        path_to_image = os.path.dirname(image)
        if not os.path.isdir(path_to_image) and len(path_to_image) != 0:
            os.makedirs(path_to_image)
        squashfs_file = image

    if os.path.isfile(squashfs_file):
        os.rename(squashfs_file, squashfs_file + ".old")
        logging.info("Previous image renamed to {0}.".format(squashfs_file +
                                                             ".old"))

    logging.info("Creating image at {0}".format(squashfs_file))

    if not os.path.exists(os.path.dirname(squashfs_file)):
        logging.info("Creating local directory path",
                     os.path.dirname(squashfs_file))
        os.makedirs(os.path.dirname(squashfs_file))

    if conf.ddebug:
        run([
            "mksquashfs", work_dir, squashfs_file, "-no-exports", "-noappend",
            "-info"
        ])
    else:
        run([
            "mksquashfs", work_dir, squashfs_file, "-no-exports", "-noappend"
        ])

    os.chmod(squashfs_file, 0o755)

    sftp_mode = has_config_value("images", "hosts", dist)
    dargs = docopt.docopt(__doc__)
    if sftp_mode and not dargs['--no-sync']:
        sftp_user = get_from_config("images", "sftp_user", dist)
        sftp_private_key = get_from_config("images", "sftp_private_key", dist)
        sftp_passphrase = get_from_config_or("images", "sftp_passphrase", dist,
                                             None)
        sftp_hosts = get_from_config("images", "hosts", dist).split(',')
        sftp_client = sftp.Sftp(sftp_hosts, sftp_user, sftp_private_key,
                                sftp_passphrase)
        sftp_client.upload([squashfs_file], os.path.dirname(squashfs_file),
                           0o755)
Example #15
0
def do_connect_ipmi(host):

    imm_user = value_from_file(get_from_config("common", "master_passwd_file"), "USER")
    os.environ["IPMI_PASSWORD"] = value_from_file(get_from_config("common", "master_passwd_file"), "IMMPASSWORD")

    pat = re.compile("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}")
    if not pat.match(host):
        host = full_hostname(host)

    ipmitool = ["ipmitool", "-I", "lanplus", "-H", host, "-U", imm_user, "-E", "-e!", "sol", "activate"]
    logging.debug("ipmi/ipmi_do: {0}".format(" ".join(ipmitool)))
    run(ipmitool)
Example #16
0
def do_sync(selected_dist, input_suites=[]):
    info_suites = {}  # Contains all the information
    suite_dist = {}  # Contains the pairs suite - webdir
    all_suites = []  # Contains a list with all the suites names

    for distribution in get_from_config("common", "allowed_distributions").split(","):
        elements = get_from_config("repo", "info_suites", distribution).split(",")
        tmp_info_suites = {}

        for e in elements:
            k, v = e.split(":")
            all_suites.append(k)
            suite_dist[k] = distribution
            tmp_info_suites[k] = v

        info_suites[distribution] = tmp_info_suites

    suites = []
    if selected_dist == 'all':  # We sync everything
        suites = all_suites
    elif len(input_suites) == 0:  # We only sync suites related to the default distribution
        suites = info_suites[selected_dist].keys()
    else:  # If we select one or several suites, we check that are valid
        for s in input_suites:
            if s not in all_suites:
                clara_exit("{0} is not a valid suite. Valid suites are: {1}".format(s, " ".join(all_suites)))
            suites = input_suites

    logging.debug("The suites to sync are: {0}.".format(" ".join(suites)))
    for s in suites:
        mirror_root = get_from_config("repo", "mirror_root", suite_dist[s])
        dm_server = get_from_config("repo", "server", suite_dist[s])
        dm_root = info_suites[suite_dist[s]][s]

        suite_name = s
        if s in ["wheezy-security", "jessie-security"]:
            suite_name = s.split("-")[0] + "/updates"

        archs = get_from_config("repo", "archs", suite_dist[s])
        sections = get_from_config("repo", "sections", suite_dist[s])

        extra = []
        if conf.ddebug:  # if extra debug for 3rd party software
            extra = ['--debug']

        run(['debmirror'] + extra + ["--diff=none", "--method=http",
             "--nosource", "--ignore-release-gpg", "--ignore-missing-release",
             "--arch={0}".format(archs),
             "--host={0}".format(dm_server),
             "--root={0}".format(dm_root),
             "--dist={0}".format(suite_name),
             "--section={0}".format(sections),
              mirror_root + "/" + s])
Example #17
0
def install_cfg():
    passwd_file = get_from_config("common", "master_passwd_file")
    if not os.path.isfile(passwd_file) and os.path.isfile(passwd_file + ".enc"):
        password = value_from_file(get_from_config("common", "master_passwd_file"), "PASSPHRASE")

        if len(password) > 20:
            cmd = ['openssl', 'aes-256-cbc', '-d', '-in', passwd_file + ".enc",
                   '-out', passwd_file, '-k', password]
            run(cmd)
            os.chmod(passwd_file, 0o400)
        else:
            sys.exit('There was some problem reading the PASSPHRASE')
Example #18
0
def do_init():
    import shutil
    repo_dir = get_from_config("repo", "repo_dir", dist)
    reprepro_config = repo_dir + '/conf/distributions'
    mirror_local = get_from_config("repo", "mirror_local", dist)
    if (mirror_local == "" or mirror_local == None):
        mirror_local = repo_dir + '/mirror'

    if os.path.isdir(repo_dir):
        clara_exit("The repository '{0}' already exists !".format(repo_dir))
    else:
        if not os.path.isfile(reprepro_config):

            try:
                if not os.path.isdir(repo_dir + '/conf'):
                    os.makedirs(repo_dir + '/conf')

                freprepro = open(reprepro_config, 'w')
                freprepro.write("""Origin: {0}
        Label: {1}
        Suite: {1}
        Codename: {1}
        Version: {2}
        Architectures: amd64 source
        Components: main contrib non-free
        UDebComponents: main
        SignWith: {3}
        Description: Depot Local {4}
        DebIndices: Packages Release . .gz .bz2
        DscIndices: Sources Release . .gz .bz2
        """.format(get_from_config("common", "origin", dist), dist,
                   get_from_config("repo", "version", dist),
                   get_from_config("repo", "gpg_key", dist),
                   get_from_config("repo", "clustername", dist)))
                freprepro.close()

                os.chdir(repo_dir)

                list_flags = ['--ask-passphrase']
                if conf.ddebug:
                    list_flags.append("-V")

                run(['reprepro'] + list_flags + [
                    '--basedir', repo_dir, '--outdir', mirror_local, 'export',
                    dist
                ])
            except:
                shutil.rmtree(repo_dir)
                clara_exit(
                    "The repository '{0}' has not been initialized properly, it will be deleted  !"
                    .format(repo_dir))
Example #19
0
def mktorrent(image):
    if (image is None):
        squashfs_file = get_from_config("images", "trg_img", dist)
    else:
        squashfs_file = image
    trackers_port = get_from_config("p2p", "trackers_port", dist)
    trackers_schema = get_from_config("p2p", "trackers_schema", dist)
    seeding_service = get_from_config("p2p", "seeding_service", dist)
    init_stop = get_from_config("p2p", "init_stop", dist)
    init_start = get_from_config("p2p", "init_start", dist)

    # trackers is a dictionary with pairs nodeset and torrent file
    trackers = {}
    for e in get_from_config("p2p", "trackers", dist).split(";"):
        k, v = e.split(":")
        trackers[k] = v

    # seeders in the config file is a dictionary with pairs nodeset and torrent file
    seeders_dict = {}
    for e in get_from_config("p2p", "seeders", dist).split(";"):
        k, v = e.split(":")
        seeders_dict[k] = v
    seeders = ",".join(seeders_dict.keys())

    if not os.path.isfile(squashfs_file):
        clara_exit("{0} doesn't exist".format(squashfs_file))

    # Remove old torrent files
    for f in trackers.values():
        if os.path.isfile(f):
            os.remove(f)

    clush(seeders, init_stop.format(seeding_service))

    sftp_mode = has_config_value("p2p", "sftp_user", dist)
    if sftp_mode:
        sftp_user = get_from_config("p2p", "sftp_user", dist)
        sftp_private_key = get_from_config("p2p", "sftp_private_key", dist)
        sftp_passphrase = get_from_config_or("p2p", "sftp_passphrase", dist, None)
        sftp_client = sftp.Sftp(seeders.split(','), sftp_user, sftp_private_key, sftp_passphrase)

    for e in trackers.keys():
        announce = []
        for t in list(ClusterShell.NodeSet.NodeSet(e)):
            announce.append("{0}://{1}:{2}/announce".format(trackers_schema, t, trackers_port))
        run(["/usr/bin/mktorrent", "-a", ",".join(announce), "-o", trackers[e], squashfs_file])
        if sftp_mode:
            sftp_client.upload([trackers[e]], os.path.dirname(trackers[e]))

    clush(seeders, init_start.format(seeding_service))
Example #20
0
def umount_chroot():
    if os.path.ismount(work_dir + "/proc/sys/fs/binfmt_misc"):
        run(["chroot", work_dir, "umount", "/proc/sys/fs/binfmt_misc"])

    if os.path.ismount(work_dir + "/sys"):
        run(["chroot", work_dir, "umount", "/sys"])

    if os.path.ismount(work_dir + "/proc"):
        run(["chroot", work_dir, "umount", "/proc"])
    try:
        extra_bind_mounts = get_from_config("chroot", "extra_bind_mounts",
                                            dist).split(",")
    except:
        extra_bind_mounts = None
    if not extra_bind_mounts:
        logging.warning("extra_bind_mounts is not specified in config.ini")
    else:
        for mounts_params in extra_bind_mounts:
            mountpoint = work_dir + mounts_params.split(" ")[1]
            with open("/proc/mounts", "r") as file_to_read:
                for line in file_to_read:
                    if mountpoint in line:
                        run(["umount", mountpoint])
    time.sleep(1)  # Wait one second so the system has time to unmount
    with open("/proc/mounts", "r") as file_to_read:
        for line in file_to_read:
            if work_dir in line:
                clara_exit("Something went wrong when umounting in the chroot")
Example #21
0
def mount_chroot(work_dir):
    run(["chroot", work_dir, "mount", "-t", "proc", "none", "/proc"])
    run(["chroot", work_dir, "mount", "-t", "sysfs", "none", "/sys"])
    if not os.path.exists(work_dir + "/dev/random"):
        run(["mknod", "-m", "444", work_dir + "/dev/random", "c", "1", "8"])
    if not os.path.exists(work_dir + "/dev/urandom"):
        run(["mknod", "-m", "444", work_dir + "/dev/urandom", "c", "1", "9"])
Example #22
0
def do_sync(selected_dist, input_suites=[]):

    suites = []
    # Sync everything
    if selected_dist == 'all':
        for d in get_from_config("common", "allowed_distributions").split(","):
            suites = suites + get_from_config("repo", "suites", d).split(",")
    else:
        # Only sync suites related to the selected distribution
        if len(input_suites) == 0:
            suites = get_from_config("repo", "suites",
                                     selected_dist).split(",")
        # User selected one or several suites, check that they are valid
        else:
            for s in input_suites:
                if s not in get_from_config("repo", "suites",
                                            selected_dist).split(","):
                    clara_exit(
                        "{0} is not a valid suite from distribution {1}.\n"
                        "Valid suites are: {2}".format(
                            s, selected_dist,
                            get_from_config("repo", "suites", selected_dist)))
            suites = input_suites

    logging.debug("The suites to sync are: {0}.".format(" ".join(suites)))

    # Read /etc/clara/repos.ini
    if not os.path.isfile('/etc/clara/repos.ini'):
        clara_exit("File /etc/clara/repos.ini not found.")
    repos = ConfigParser.ConfigParser()
    repos.read("/etc/clara/repos.ini")

    for s in suites:

        extra = []
        if conf.ddebug:  # if extra debug for 3rd party software
            extra = ['--debug']

        final_dir = get(repos, s, "mirror_root") + "/" + s
        run(['debmirror'] + extra + [
            "--diff=none", "--nosource", "--ignore-release-gpg",
            "--ignore-missing-release", "--method={0}".format(
                get(repos, s, "method")),
            "--arch={0}".format(get(repos, s, "archs")), "--host={0}".format(
                get(repos, s, "server")), "--root={0}".format(
                    get(repos, s, "mirror_dir")), "--dist={0}".format(
                        get(repos, s, "suite_name")), "--section={0}".format(
                            get(repos, s, "sections")), final_dir
        ])
Example #23
0
def do_sync(option=''):
    local = get_from_config("repo", "local_modules", dist).split(',')
    remote = get_from_config("repo", "remote_modules", dist).split(',')

    for elem in range(0, len(local)):
        if os.path.isdir(get_from_config("repo", "mirror_root", dist) +
                         "/" + local[elem]) or (option == 'create'):

            run(['rsync',
                 '-az', '--stats', '--human-readable', '--force', '--delete',
                 '--ignore-errors',
                 get_from_config("repo", "server", dist) + '::' + remote[elem],
                 get_from_config("repo", "mirror_root", dist) + '/' + local[elem]])
        else:
            sys.exit('Local repository not found. '
                     'Please run: \n\tclara repo sync create')
Example #24
0
def base_install():
    # Step 1 - Debootstrap
    src_list = work_dir + "/etc/apt/sources.list"
    apt_pref = work_dir + "/etc/apt/preferences.d/00custompreferences"
    apt_conf = work_dir + "/etc/apt/apt.conf.d/99nocheckvalid"
    dpkg_conf = work_dir + "/etc/dpkg/dpkg.cfg.d/excludes"
    etc_host = work_dir + "/etc/hosts"

    run(["debootstrap", get_from_config("images", "debiandist", dist), work_dir,
         get_from_config("images", "debmirror", dist) + "/debian"])

    # Step 2 - Mirror setup
    list_repos = get_from_config("images", "list_repos", dist).split(",")
    with open(src_list, 'w') as fsources:
        for line in list_repos:
            fsources.write(line + '\n')

    with open(apt_pref, 'w') as fapt:
        fapt.write("""Package: *
Pin: release o={0}
Pin-Priority: 5000

Package: *
Pin: release o={1}
Pin-Priority: 6000
""".format(dist, get_from_config("common", "origin", dist)))

    with open(apt_conf, 'w') as fconf:
        fconf.write('Acquire::Check-Valid-Until "false";\n')

    lists_hosts = get_from_config("images", "etc_hosts", dist).split(",")
    if (len(lists_hosts) % 2 != 0):
        print "WARNING: the option etc_hosts is malformed or missing an argument"
    with open(etc_host, 'w') as fhost:
        for elem in range(0, len(lists_hosts), 2):
            fhost.write("{0} {1}\n".format(lists_hosts[elem], lists_hosts[elem+1]))

    with open(dpkg_conf, 'w') as fdpkg:
        fdpkg.write("""# Drop locales except French
path-exclude=/usr/share/locale/*
path-include=/usr/share/locale/fr/*
path-include=/usr/share/locale/locale.alias

# Drop manual pages
# (We keep manual pages in the image)
## path-exclude=/usr/share/man/*
""")
Example #25
0
def mktorrent(image):
    if (image is None):
        squashfs_file = get_from_config("images", "trg_img", dist)
    else:
        squashfs_file = image
    trackers_port = get_from_config("p2p", "trackers_port", dist)
    trackers_schema = get_from_config("p2p", "trackers_schema", dist)
    seeding_service = get_from_config("p2p", "seeding_service", dist)
    init_stop = get_from_config("p2p", "init_stop", dist)
    init_start = get_from_config("p2p", "init_start", dist)

    # trackers is a dictionary with pairs nodeset and torrent file
    trackers = {}
    for e in get_from_config("p2p", "trackers", dist).split(";"):
        k, v = e.split(":")
        trackers[k] = v

    # seeders in the config file is a dictionary with pairs nodeset and torrent file
    seeders_dict = {}
    for e in get_from_config("p2p", "seeders", dist).split(";"):
        k, v = e.split(":")
        seeders_dict[k] = v
    seeders = ",".join(seeders_dict.keys())

    if not os.path.isfile(squashfs_file):
        clara_exit("The file {0} doesn't exist".format(squashfs_file))

    # Remove old torrent files
    for f in trackers.values():
        if os.path.isfile(f):
            os.remove(f)

    clush(seeders, init_stop.format(seeding_service))

    for e in trackers.keys():
        announce = []
        for t in list(ClusterShell.NodeSet.NodeSet(e)):
            announce.append("{0}://{1}:{2}/announce".format(
                trackers_schema, t, trackers_port))
        run([
            "/usr/bin/mktorrent", "-a", ",".join(announce), "-o", trackers[e],
            squashfs_file
        ])

    clush(seeders, init_start.format(seeding_service))
Example #26
0
def edit(image, work_dir, dist):
    if (image is None):
        squashfs_file = get_from_config("images", "trg_img", dist)
        if (squashfs_file == "" or squashfs_file == None):
            image_name = dist + "_image.squashfs"
            squashfs_file = "/var/lib/clara/images/" + image_name
    else:
        squashfs_file = image

    if not os.path.isfile(squashfs_file):
        clara_exit("{0} doesn't exist.".format(squashfs_file))

    # Extract the image.
    logging.info("Extracting {0} to {1} ...".format(squashfs_file, work_dir))
    if conf.ddebug:
        run(["unsquashfs", "-li", "-f", "-d", work_dir, squashfs_file])
    else:
        run(["unsquashfs", "-f", "-d", work_dir, squashfs_file])

    # Work in the image
    os.chdir(work_dir)
    logging.info(
        "Entering into a bash shell to edit the image. ^d when you have finished."
    )
    os.putenv("PROMPT_COMMAND",
              "echo -ne  '\e[1;31m({0}) clara images> \e[0m'".format(dist))
    pty.spawn(["/bin/bash"])

    save = raw_input('Save changes made in the image? (N/y)')
    logging.debug("Input from the user: '******'".format(save))
    if save not in ('Y', 'y'):
        clara_exit(
            "Changes ignored. The image {0} hasn't been modified.".format(
                squashfs_file))

    # Rename old image and recreate new one
    os.rename(squashfs_file, squashfs_file + ".old")
    if conf.ddebug:
        run([
            "mksquashfs", work_dir, squashfs_file, "-no-exports", "-noappend",
            "-info"
        ])
    else:
        run([
            "mksquashfs", work_dir, squashfs_file, "-no-exports", "-noappend"
        ])

    os.chmod(squashfs_file, 0o755)
    logging.info("\nPrevious image renamed to {0}."
                 "\nThe image has been repacked at {1}".format(
                     squashfs_file + ".old", squashfs_file))
Example #27
0
def do_connect(hosts):
    nodeset = ClusterShell.NodeSet.NodeSet(hosts)
    if (len(nodeset) != 1):
	    sys.exit('Only one host allowed for this command')
    else:
            try:
                cmd = ["service", "conman", "status"]
                retcode = subprocess.call(cmd, stdout=DEVNULL, stderr=STDOUT)
            except OSError, e:
                if (e.errno == errno.ENOENT):
                    sys.exit("Binary not found, check your path and/or retry as root."
                             "You were trying to run:\n {0}".format(" ".join(cmd)))

            if retcode == 0:  # if conman is running
                os.environ["CONMAN_ESCAPE"] = '!'
                conmand = value_from_file(get_from_config("nodes", "conmand"))
                run(["conman", "-d", conmand, hosts])
            elif retcode == 1 or retcode == 3:  # if conman is NOT running
                ipmi_do(hosts, ["sol", "activate"], pty=True)
            else:
                sys.exit('E: ' + ' '.join(cmd))
Example #28
0
def extract_image(image):
    if (image is None):
        squashfs_file = get_from_config("images", "trg_img", dist)
    else:
        squashfs_file = image

    if not os.path.isfile(squashfs_file):
        clara_exit("{0} does not exist!".format(squashfs_file))

    extract_dir = tempfile.mkdtemp(prefix="tmpClara")
    logging.info("Extracting {0} to {1} ...".format(squashfs_file,
                                                    extract_dir))
    if conf.ddebug:
        run(["unsquashfs", "-li", "-f", "-d", extract_dir, squashfs_file])
    else:
        run(["unsquashfs", "-f", "-d", extract_dir, squashfs_file])

    logging.info(
        "Modify the image at {0} and then run:\n"
        "\tclara images repack {0} ( <dist> | --image=<path> )".format(
            extract_dir))
Example #29
0
def copy_jenkins(job, arch, flags=None):
    repo_dir = get_from_config("repo", "repo_dir", dist)
    reprepro_config = repo_dir + '/conf/distributions'

    if not os.path.isfile(reprepro_config):
        clara_exit(
            "There is not configuration for the local repository for {0}. Run first 'clara repo init <dist>'"
            .format(dist))

    list_flags = ['--silent', '--ask-passphrase']
    if conf.ddebug:
        list_flags = ['-V', '--ask-passphrase']

    if flags is not None:
        list_flags.append(flags)

    if not job.endswith("-binaries"):
        job = job + "-binaries"

    jenkins_dir = get_from_config("repo", "jenkins_dir")
    path = os.path.join(jenkins_dir, job,
                        "configurations/builds/lastSuccessfulBuild/archive/")

    if not os.path.isdir(path):
        clara_exit(
            "The job {} doesn't exists or needs to be build.".format(job))

    for f in os.listdir(path):
        if f.endswith("_{0}.changes".format(arch)):
            changesfile = os.path.join(path + f)

    if changesfile is None:
        clara_exit("Not changes files was found in {0}".format(path))

    cmd = ['reprepro'] + list_flags + \
         ['--basedir', get_from_config("repo", "repo_dir", dist),
         '--outdir', get_from_config("repo", "mirror_local", dist),
         "include", dist, changesfile]

    run(cmd)
Example #30
0
def geninitrd():
    trg_dir = get_from_config("images", "trg_dir", dist)
    if not os.path.isdir(trg_dir):
        sys.exit("Directory {0} does not exist!".format(trg_dir))

    mkinitrfs = get_from_config("images", "mkinitramfs", dist)
    if not os.path.isfile(mkinitrfs):
        sys.exit("{0} does not exist!".format(mkinitrfs))

    initramfsc = get_from_config("images", "initramfs-config", dist)
    if not os.path.isdir(initramfsc):
        sys.exit("Directory {0} does not exist!".format(initramfsc))

    kver = get_from_config("images", "kver", dist)
    # Generate the initrd
    run([mkinitrfs, "-d", initramfsc, "-o", trg_dir + "/initrd-" + kver, kver])
    os.chmod(trg_dir + "/initrd-" + kver, 0o644)
    print "Initrd generated in " + trg_dir + "/initrd-" + kver

    # Copy kernel into right directory
    shutil.copy("/boot/vmlinuz-" + kver, trg_dir + "/linux-" + kver)
    print "Kernel copied in " + trg_dir + "/linux-" + kver
Example #31
0
def do_init():
    repo_dir = get_from_config("repo", "repo_dir", dist)
    reprepro_config = repo_dir + '/conf/distributions'

    if not os.path.isfile(reprepro_config):
        if not os.path.isdir(repo_dir + '/conf'):
            os.makedirs(repo_dir + '/conf')

        freprepro = open(reprepro_config, 'w')
        freprepro.write("""Origin: {0}
Label: {1}
Suite: {1}
Codename: {1}
Version: {2}
Architectures: amd64 source
Components: main contrib non-free
UDebComponents: main
SignWith: {3}
Description: Depot Local {4}
DebIndices: Packages Release . .gz .bz2
DscIndices: Sources Release . .gz .bz2
""".format(get_from_config("common", "origin", dist),
            dist,
            get_from_config("repo", "version", dist),
            get_from_config("repo", "gpg_key", dist),
            get_from_config("repo", "clustername", dist)))
        freprepro.close()

        os.chdir(repo_dir)

        list_flags = ['--ask-passphrase']
        if conf.ddebug:
            list_flags.append("-V")

        run(['reprepro'] + list_flags +
            ['--basedir', repo_dir,
             '--outdir', get_from_config("repo", "mirror_local", dist),
             'export', dist])
Example #32
0
def do_reprepro(action, package=None, flags=None, extra=None):
    repo_dir = get_from_config("repo", "repo_dir", dist)
    reprepro_config = repo_dir + '/conf/distributions'
    mirror_local = get_from_config("repo", "mirror_local", dist)
    if (mirror_local == "" or mirror_local == None):
        mirror_local = repo_dir + '/mirror'
    oldMask = os.umask(0022)
    if not os.path.isfile(reprepro_config):
        clara_exit(
            "There is not configuration for the local repository for {0}. Run first 'clara repo init <dist>'"
            .format(dist))

    list_flags = ['--silent', '--ask-passphrase']
    if conf.ddebug:
        list_flags = ['-V', '--ask-passphrase']

    if flags is not None:
        list_flags.append(flags)

    cmd = ['reprepro'] + list_flags + \
         ['--basedir', get_from_config("repo", "repo_dir", dist),
         '--outdir', mirror_local,
         action]

    if extra is not None:
        for e in extra:
            cmd.append(e)
    else:
        if action in [
                'includedeb', 'include', 'includedsc', 'remove', 'removesrc',
                'list'
        ]:
            cmd.append(dist)

        if package is not None:
            cmd.append(package)
    run(cmd)
    os.umask(oldMask)
Example #33
0
def do_reprepro(action, package=None, flags=None):
    repo_dir = get_from_config("repo", "repo_dir", dist)
    reprepro_config = repo_dir + '/conf/distributions'

    if not os.path.isfile(reprepro_config):
        clara_exit("There is not configuration for the local repository for {0}. Run first 'clara repo init <dist>'".format(dist))

    list_flags = ['--silent', '--ask-passphrase']
    if conf.ddebug:
        list_flags = ['-V', '--ask-passphrase']

    if flags is not None:
        list_flags.append(flags)

    cmd = ['reprepro'] + list_flags + \
         ['--basedir', get_from_config("repo", "repo_dir", dist),
         '--outdir', get_from_config("repo", "mirror_local", dist),
         action, dist]

    if package is not None:
        cmd.append(package)

    run(cmd)
Example #34
0
def editimg(image):
    if (image is None):
        squashfs_file = get_from_config("images", "trg_img", dist)
    else:
        squashfs_file = image

    if not os.path.isfile(squashfs_file):
        sys.exit("The image file {0} doesn't exist.".format(squashfs_file))

    # Extract the image.
    print "Extracting {0} to {1} ...".format(squashfs_file, work_dir)
    run(["unsquashfs", "-f", "-d", work_dir, squashfs_file])
    # Work in the image
    os.chdir(work_dir)
    print "Entering into a bash shell to edit the image. ^d when you have finished"
    os.putenv("PROMPT_COMMAND", "echo -ne  '\e[1;31m clara images> \e[0m'")
    pty.spawn(["/bin/bash"])
    # Rename old image and recreate new one
    os.rename(squashfs_file, squashfs_file + ".old")
    print("Previous image renamed to {0}.".format(squashfs_file + ".old"))
    print "Recreating image at {0}".format(squashfs_file)
    run(["mksquashfs", work_dir, squashfs_file, "-no-exports", "-noappend"])
    os.chmod(squashfs_file, 0o755)
Example #35
0
def mktorrent(image):
    ml_path = "/var/lib/mldonkey"
    trg_dir = get_from_config("images", "trg_dir")
    if (image is None):
        squashfs_file = get_from_config("images", "trg_img")
    else:
        squashfs_file = image
    seeders = get_from_config("p2p", "seeders")
    trackers = get_from_config("p2p", "trackers")
    trackers_port = get_from_config("p2p", "trackers_port")
    trackers_schema = get_from_config("p2p", "trackers_schema")

    if not os.path.isfile(squashfs_file):
        sys.exit("The file {0} doesn't exist".format(squashfs_file))

    if os.path.isfile(trg_dir + "/image.torrent"):
        os.remove(trg_dir + "/image.torrent")

    clush(seeders, "service ctorrent stop")
    clush(trackers, "service mldonkey-server stop")

    for files in ["torrents/old", "torrents/seeded", "torrents/tracked"]:
        clush(trackers, "rm -f {0}/{1}/*".format(ml_path, files))

    clush(trackers, "ln -sf {0} {1}/incoming/files/".format(squashfs_file, ml_path))

    clush(trackers, "awk 'BEGIN{verb=1}; / tracked_files = / {verb=0}; /^$/ {verb=1}; {if (verb==1) print}' /var/lib/mldonkey/bittorrent.ini > /var/lib/mldonkey/bittorrent.ini.new")
    clush(trackers, "mv {0}/bittorrent.ini.new {0}/bittorrent.ini".format(ml_path))

    announce = []
    for t in list(ClusterShell.NodeSet.NodeSet(trackers)):
        announce.append("{0}://{1}:{2}/announce".format(trackers_schema, t, trackers_port))
    run(["/usr/bin/mktorrent", "-a", ",".join(announce), "-o", trg_dir + "/image.torrent", squashfs_file])
    clush(trackers, "ln -sf {0}/image.torrent {1}/torrents/seeded/".format(trg_dir, ml_path))

    clush(trackers, "service mldonkey-server start")
    clush(seeders, "service ctorrent start")
Example #36
0
def ipmi_do(hosts, pty=False, *cmd):
    command = []
    if not isinstance(pty, bool):
        command.append(pty)
        pty = False
    command.extend(cmd)
    imm_user = value_from_file(get_from_config("common", "master_passwd_file"), "IMMUSER")
    os.environ["IPMI_PASSWORD"] = value_from_file(get_from_config("common", "master_passwd_file"), "IMMPASSWORD")
    nodeset = ClusterShell.NodeSet.NodeSet(hosts)
    for host in nodeset:

        pat = re.compile("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}")
        if not pat.match(host):
            host = "imm" + host

        ipmitool = ["ipmitool", "-I", "lanplus", "-H", host, "-U", imm_user, "-E", "-e!"]
        ipmitool.extend(command)

        logging.debug("ipmi/ipmi_do: {0}".format(" ".join(ipmitool)))

        if pty:
            run(ipmitool)
        else:
            os.system("echo -n '%s: ' ;" % host + " ".join(ipmitool))
Example #37
0
def do_connect(host, j=False, f=False):
    nodeset = ClusterShell.NodeSet.NodeSet(host)
    if (len(nodeset) != 1):
        clara_exit('Only one host allowed for this command')

    pat = re.compile("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}")
    if pat.match(host):
        logging.debug(
            "The host is an IP adddres: {0}. Using ipmitool without conman.".
            format(host))
        do_connect_ipmi(host)
    else:
        conmand = get_from_config_or("ipmi", "conmand", '')
        port = int(get_from_config("ipmi", "port"))
        if len(conmand) == 0:
            do_connect_ipmi(host)
            return

        cmd = []
        ssh_jhost = get_bool_from_config_or("ipmi", "ssh_jump_host")

        if ssh_jhost:
            cmd += ["ssh", "-t", conmand]
            conmand = "localhost"
        cmd += ["conman"]

        try:
            if j:
                cmd = cmd + ["-j"]
            if f:
                cmd = cmd + ["-f"]
            cmd = cmd + ["-d", conmand, host, "-e!"]
            run(cmd, exit_on_error=False)
        except RuntimeError as e:
            logging.warning("Conman failed, fallback to ipmitool")
            do_connect_ipmi(host)
Example #38
0
def umount_chroot(work_dir):
    if os.path.ismount(work_dir + "/proc/sys/fs/binfmt_misc"):
        run(["chroot", work_dir, "umount", "/proc/sys/fs/binfmt_misc"])

    if os.path.ismount(work_dir + "/sys"):
        run(["chroot", work_dir, "umount", "/sys"])

    if os.path.ismount(work_dir + "/proc"):
        run(["chroot", work_dir, "umount", "-lf", "/proc"])
    time.sleep(1)  # Wait one second so the system has time to unmount
    with open("/proc/mounts", "r") as file_to_read:
        for line in file_to_read:
            if work_dir in line:
                clara_exit("Something went wrong when umounting in the chroot")
Example #39
0
def mount_chroot():
    run(["chroot", work_dir, "mount", "-t", "proc", "none", "/proc"])
    run(["chroot", work_dir, "mount", "-t", "sysfs", "none", "/sys"])
    try:
        extra_bind_mounts = get_from_config("chroot", "extra_bind_mounts",
                                            dist).split(",")
    except:
        extra_bind_mounts = None
    if not extra_bind_mounts:
        logging.warning("extra_bind_mounts is not specified in config.ini")
    else:
        for mounts_params in extra_bind_mounts:
            dirtomount = mounts_params.split(" ")[0]
            mountpoint = work_dir + mounts_params.split(" ")[1]
            if not os.path.isdir(mountpoint):
                os.makedirs(mountpoint)
            run(["mount", "-o", "bind", dirtomount, mountpoint])
Example #40
0
def mount_chroot():
    run(["chroot", work_dir, "mount", "-t", "proc", "none", "/proc"])
    run(["chroot", work_dir, "mount", "-t", "sysfs", "none", "/sys"])
Example #41
0
def geninitrd(path, work_dir, dist):
    ID, VERSION_ID = get_osRelease(dist)
    image = imageInstant(work_dir, ID, VERSION_ID)
    distrib = image.dist

    if (path is None):
        trg_dir = get_from_config("images", "trg_dir", dist)

        if (trg_dir == "" or trg_dir == None):
            trg_dir = "/var/lib/clara/images/"
    else:
        trg_dir = path

    if not os.path.isdir(trg_dir):
        os.makedirs(trg_dir)
    squashfs_file = get_from_config("images", "trg_img", dist)
    if (squashfs_file == "" or squashfs_file == None):
        image_name = dist + "_image.squashfs"
        squashfs_file = "/var/lib/clara/images/" + image_name
    if not os.path.isfile(squashfs_file):
        clara_exit("{0} does not exist!".format(squashfs_file))

    if conf.ddebug:
        run(["unsquashfs", "-li", "-f", "-d", work_dir, squashfs_file])
    else:
        run(["unsquashfs", "-f", "-d", work_dir, squashfs_file])

    mount_chroot(work_dir)

    # Install the kernel in the image
    kver = get_from_config("images", "kver", dist)
    if len(kver) == 0:
        clara_exit("kver hasn't be set in config.ini")
    else:
        run_chroot(["chroot", work_dir, distrib["pkgManager"], "update"],
                   work_dir)
        if ID == "debian":
            run_chroot([
                "chroot", work_dir, distrib["pkgManager"], "install",
                "--no-install-recommends", "--yes", "--force-yes",
                "linux-image-" + kver
            ], work_dir)
        if ID == "centos":
            run_chroot([
                "chroot", work_dir, distrib["pkgManager"], "install", "-y",
                "kernel-" + kver
            ], work_dir)
    # Install packages from 'packages_initrd'
    packages_initrd = get_from_config("images", "packages_initrd", dist)
    if len(packages_initrd) == 0:
        logging.warning("packages_initrd hasn't be set in config.ini")
    else:
        pkgs = packages_initrd.split(',')
        if ID == "debian":
            opts = ["--no-install-recommends", "--yes", "--force-yes"]
            intitrd_opts = ["-o", "/tmp/initrd-" + kver, kver]
        if ID == "centos":
            opts = ["-y"]
            intitrd_opts = [
                "--force", "--add", "livenet", "-v", "/tmp/initrd-" + kver,
                "--kver", kver
            ]
        opts = ["chroot", work_dir, distrib["pkgManager"], "install"
                ] + opts + pkgs
        run_chroot(opts, work_dir)

    # Generate the initrd in the image
    intitrd_opts = ["chroot", work_dir, distrib["initrdGen"]] + intitrd_opts
    run_chroot(intitrd_opts, work_dir)

    umount_chroot(work_dir)

    # Copy the initrd out of the chroot
    initrd_file = trg_dir + "/initrd-" + kver
    shutil.copy(work_dir + "/tmp/initrd-" + kver, initrd_file)
    os.chmod(initrd_file, 0o644)
    logging.info("Initrd available at " + initrd_file)

    # Copy vmlinuz out of the chroot
    vmlinuz_file = trg_dir + "/vmlinuz-" + kver
    shutil.copy(work_dir + "/boot/vmlinuz-" + kver, vmlinuz_file)
    os.chmod(vmlinuz_file, 0o644)
    logging.info("vmlinuz available at " + vmlinuz_file)

    # Send files where they will be used
    sftp_mode = has_config_value("images", "hosts", dist)
    dargs = docopt.docopt(__doc__)
    if sftp_mode and not dargs['--no-sync']:
        sftp_user = get_from_config("images", "sftp_user", dist)
        sftp_private_key = get_from_config("images", "sftp_private_key", dist)
        sftp_passphrase = get_from_config_or("images", "sftp_passphrase", dist,
                                             None)
        sftp_hosts = get_from_config("images", "hosts", dist).split(',')
        sftp_client = sftp.Sftp(sftp_hosts, sftp_user, sftp_private_key,
                                sftp_passphrase)
        sftp_client.upload([initrd_file, vmlinuz_file], trg_dir, 0o644)
Example #42
0
def do_ping(hosts):
    nodes = ClusterShell.NodeSet.NodeSet(hosts)
    cmd = ["fping", "-r1", "-u", "-s"] + list(nodes)
    run(cmd)
Example #43
0
def do_package(action, package):
    run(['reprepro', '--ask-passphrase',
         '--basedir', get_from_config("repo", "repo_dir", dist),
         '--outdir', get_from_config("repo", "mirror_local", dist),
         action, dist, package])
Example #44
0
def main():
    dargs = docopt.docopt(__doc__)

    if dargs['resume']:
        run(["scontrol", "update", "NodeName="+dargs['<nodeset>'],
             "State=RESUME"])
    elif dargs['drain']:
        if dargs['<nodeset>'] is None:
            show_nodes("drain")
        else:
            if len (dargs['<reason>']) == 0:
                sys.exit("You must specify a reason when DRAINING a node")
            else:
                run(["scontrol", "update", "NodeName="+dargs['<nodeset>'],
                     "State=DRAIN", 'Reason="'+" ".join(dargs['<reason>'])+'"'])
    elif dargs['down']:
        if dargs['<nodeset>'] is None:
            show_nodes("down")
        else:
            run(["scontrol", "update", "NodeName="+dargs['<nodeset>'],
                 "State=DOWN"])
    elif dargs['health']:
        clush(dargs['<nodeset>'],
              "/usr/lib/slurm/check_node_health.sh --no-slurm")
    else:
        cmd_list = ['job', 'node', 'steps', 'frontend', 'partition', 'reservation',
                    'block', 'submp']

        # /!\ ∀ x, ∀ y, op_list[x][y] ⇒ op_list[x][y] ∈ cmd_list
        op_list = {
            'show': ['job', 'node', 'partition', 'reservation', 'steps',
                     'frontend', 'block', 'submp'],
            'update': ['job', 'node', 'partition', 'reservation', 'steps',
                       'frontend', 'block', 'submp'],
            'create': ['partition', 'reservation'],
            'delete': ['partition', 'reservation']
        }

        # /!\ ∀ x ∈ cmd_list ⇒ x ∈ keys_list.keys()
        key_list = {
            'job': 'JobId',
            'steps': 'StepId',
            'node': 'NodeName',
            'frontend': 'FrontendName',
            'partition': 'PartitionName',
            'reservation': 'Reservation',
            'block': 'BlockName',
            'submp': 'SubMPName'
            }

        cmd = dargs['<cmd>']
        subject = dargs['<subject>']
        op = dargs['<op>']
        spec = dargs['<spec>']

        if cmd not in cmd_list:
            sys.exit("Known commands are: {0}".format(" ".join(cmd_list)))

        if spec is None:
            if op is not None and "=" in op:
                spec = [op]
            op = 'show'

        if "=" in op:
            spec = [op] + spec
            op = 'show'

        if op not in op_list:
            sys.exit("Known operations are: {0}".format(" ".join(op_list)))

        if cmd not in op_list[op]:
            sys.exit("You can't use {0} with {1}".format(cmd, op))

        if op == 'show':
            # spec should be empty
            run(["scontrol", op, cmd, subject])
        else:
            run(["scontrol",
                 op,
                 "{0}={1}".format(key_list[cmd], subject),
                 " ".join(spec)])
Example #45
0

def do_connect(hosts):

    try:
        cmd = ["service", "conman", "status"]
        retcode = subprocess.call(cmd)
    except OSError, e:
        if (e.errno == errno.ENOENT):
            sys.exit("Binary not found, check your path and/or retry as root."
                     "You were trying to run:\n {0}".format(" ".join(cmd)))

    if retcode == 0:  # if conman is running
        os.environ["CONMAN_ESCAPE"] = '!'
        conmand = value_from_file(get_from_config("nodes", "conmand"))
        run(["conman", "-d", conmand, hosts])
    elif retcode == 1:  # if conman is NOT running
        ipmi_do(hosts, "-e! sol activate")
    else:
        sys.exit('E: ' + ' '.join(cmd))


def do_ping(hosts):
    nodes = ClusterShell.NodeSet.NodeSet(hosts)
    cmd = ["fping", "-r1", "-u", "-s"] + list(nodes)
    run(cmd)


def main():
    dargs = docopt.docopt(__doc__)
Example #46
0
 def bootstrapper(self, opts):
     opts.insert(0, self.dist["bootstrapper"])
     run(opts)
Example #47
0
 def genInitrd(opts):
     run([self.dist["initrdGen"], opts])
Example #48
0
def mount_chroot():
    run(["chroot", work_dir, "mount", "-t", "proc", "none", "/proc"])
    run(["chroot", work_dir, "mount", "-t", "sysfs", "none", "/sys"])
Example #49
0
def main():
    logging.debug(sys.argv)
    dargs = docopt.docopt(__doc__)

    debug = []
    if conf.ddebug:
        debug = ["--verbose", "--details"]

    if dargs['resume']:
        run(["scontrol"] + debug + ["update", "NodeName=" + dargs['<nodeset>'],
             "State=RESUME"])
    elif dargs['drain']:
        if dargs['<nodeset>'] is None:
            show_nodes("drain")
        else:
            if len(dargs['<reason>']) == 0:
                clara_exit("You must specify a reason when DRAINING a node")
            else:
                run(["scontrol"] + debug + ["update", "NodeName=" + dargs['<nodeset>'],
                     "State=DRAIN", 'Reason="' + " ".join(dargs['<reason>']) + '"'])
    elif dargs['down']:
        if dargs['<nodeset>'] is None:
            show_nodes("down")
        else:
            run(["scontrol"] + debug + ["update", "NodeName=" + dargs['<nodeset>'],
                 "State=DOWN"])
    elif dargs['health']:
        script_slurm_health = get_from_config("slurm", "script_slurm_health")
        if (len(script_slurm_health) > 0):
            clush(dargs['<nodeset>'], script_slurm_health)
        else:
            clara_exit("You must set a health check program in the configuration file.")
    else:
        cmd_list = ['job', 'node', 'steps', 'frontend', 'partition', 'reservation',
                    'block', 'submp']

        # /!\ ∀ x, ∀ y, op_list[x][y] ⇒ op_list[x][y] ∈ cmd_list
        op_list = {
            'show': ['job', 'node', 'partition', 'reservation', 'steps',
                     'frontend', 'block', 'submp'],
            'update': ['job', 'node', 'partition', 'reservation', 'steps',
                       'frontend', 'block', 'submp'],
            'create': ['partition', 'reservation'],
            'delete': ['partition', 'reservation']
        }

        # /!\ ∀ x ∈ cmd_list ⇒ x ∈ keys_list.keys()
        key_list = {
            'job': 'JobId',
            'steps': 'StepId',
            'node': 'NodeName',
            'frontend': 'FrontendName',
            'partition': 'PartitionName',
            'reservation': 'Reservation',
            'block': 'BlockName',
            'submp': 'SubMPName'
            }

        cmd = dargs['<cmd>']
        subject = dargs['<subject>']
        op = dargs['<op>']
        spec = dargs['<spec>']

        if cmd not in cmd_list:
            clara_exit("Known commands are: {0}".format(" ".join(cmd_list)))

        if spec is None:
            if op is not None and "=" in op:
                spec = [op]
            op = 'show'

        if "=" in op:
            spec = [op] + spec
            op = 'show'

        if op not in op_list:
            clara_exit("Known operations are: {0}".format(" ".join(op_list)))

        if cmd not in op_list[op]:
            clara_exit("You can't use {0} with {1}".format(cmd, op))

        if op == 'show':
            # spec should be empty
            run(["scontrol"] + debug + [op, cmd, subject])
        else:
            run(["scontrol"] + debug + \
                [op,
                 "{0}={1}".format(key_list[cmd], subject),
                 " ".join(spec)])
Example #50
0
def base_install():
    # Debootstrap
    src_list = work_dir + "/etc/apt/sources.list"
    apt_pref = work_dir + "/etc/apt/preferences.d/00custompreferences"
    apt_conf = work_dir + "/etc/apt/apt.conf.d/99nocheckvalid"
    dpkg_conf = work_dir + "/etc/dpkg/dpkg.cfg.d/excludes"
    etc_host = work_dir + "/etc/hosts"

    debiandist = get_from_config("images", "debiandist", dist)
    debmirror = get_from_config("images", "debmirror", dist)

    # Get GPG options
    gpg_check = get_bool_from_config_or("images", "gpg_check", dist, True)
    gpg_keyring = get_from_config_or("images", "gpg_keyring", dist, None)

    cmd = ["debootstrap", debiandist, work_dir, debmirror]

    if gpg_check:
        if gpg_keyring is not None:
            cmd.insert(1, "--keyring=%s" % gpg_keyring)
    else:
        cmd.insert(1, "--no-check-gpg")

    if conf.ddebug:
        cmd.insert(1, "--verbose")

    run(cmd)

    # Prevent services from starting automatically
    policy_rc = work_dir + "/usr/sbin/policy-rc.d"
    with open(policy_rc, 'w') as p_rcd:
        p_rcd.write("exit 101")
    p_rcd.close()
    os.chmod(policy_rc, 0o755)

    # Mirror setup
    list_repos_nonsplitted = get_from_config("images", "list_repos", dist)
    if ';' in list_repos_nonsplitted:
        separator = ';'
    else:
        separator = ','
    list_repos = list_repos_nonsplitted.split(separator)

    with open(src_list, 'w') as fsources:
        for line in list_repos:
            fsources.write(line + '\n')
    os.chmod(src_list, 0o644)

    with open(apt_pref, 'w') as fapt:
        fapt.write("""Package: *
Pin: release o={0}
Pin-Priority: 5000

Package: *
Pin: release o={1}
Pin-Priority: 6000
""".format(dist, get_from_config("common", "origin", dist)))
    os.chmod(apt_pref, 0o644)

    # Misc config
    with open(apt_conf, 'w') as fconf:
        fconf.write('Acquire::Check-Valid-Until "false";\n')
    os.chmod(apt_conf, 0o644)

    lists_hosts = get_from_config("images", "etc_hosts", dist).split(",")
    with open(etc_host, 'w') as fhost:
        for elem in lists_hosts:
            if ":" in elem:
                ip, host = elem.split(":")
                fhost.write("{0} {1}\n".format(ip, host))
            else:
                logging.warning(
                    "The option etc_hosts is malformed or missing an argument")
    os.chmod(etc_host, 0o644)

    with open(dpkg_conf, 'w') as fdpkg:
        fdpkg.write("""# Drop locales except French
path-exclude=/usr/share/locale/*
path-include=/usr/share/locale/fr/*
path-include=/usr/share/locale/locale.alias

# Drop manual pages
# (We keep manual pages in the image)
## path-exclude=/usr/share/man/*
""")
    os.chmod(dpkg_conf, 0o644)

    # Set root password to 'clara'
    part1 = subprocess.Popen(["echo", "root:clara"], stdout=subprocess.PIPE)
    part2 = subprocess.Popen(["chroot", work_dir, "/usr/sbin/chpasswd"],
                             stdin=part1.stdout)
    part1.stdout.close()  # Allow part1 to receive a SIGPIPE if part2 exits.
Example #51
0
def main():
    logging.debug(sys.argv)
    dargs = docopt.docopt(__doc__)

    debug = []
    if conf.ddebug:
        debug = ["--verbose", "--details"]

    if dargs['resume']:
        run(["scontrol"] + debug +
            ["update", "NodeName=" + dargs['<nodeset>'], "State=RESUME"])
    elif dargs['drain']:
        if dargs['<nodeset>'] is None:
            show_nodes("drain")
        else:
            if len(dargs['<reason>']) == 0:
                clara_exit("You must specify a reason when DRAINING a node")
            else:
                run(["scontrol"] + debug + [
                    "update", "NodeName=" +
                    dargs['<nodeset>'], "State=DRAIN", 'Reason="' +
                    " ".join(dargs['<reason>']) + '"'
                ])
    elif dargs['undrain']:
        run(["scontrol"] + debug +
            ["update", "NodeName=" + dargs['<nodeset>'], "State=UNDRAIN"])
    elif dargs['fail']:
        if dargs['<nodeset>'] is None:
            show_nodes("fail")
        else:
            if len(dargs['<reason>']) == 0:
                clara_exit("You must specify a reason when FAILING a node")
            else:
                run(["scontrol"] + debug + [
                    "update", "NodeName=" +
                    dargs['<nodeset>'], "State=FAIL", 'Reason="' +
                    " ".join(dargs['<reason>']) + '"'
                ])
    elif dargs['down']:
        if dargs['<nodeset>'] is None:
            show_nodes("down")
        else:
            run(["scontrol"] + debug +
                ["update", "NodeName=" + dargs['<nodeset>'], "State=DOWN"])
    elif dargs['power']:
        if dargs['<state>'] == 'up':
            run(["scontrol"] + debug +
                ["update", "NodeName=" + dargs['<nodeset>'], "State=POWER_UP"])
        elif dargs['<state>'] == 'down':
            run(["scontrol"] + debug + [
                "update", "NodeName=" + dargs['<nodeset>'], "State=POWER_DOWN"
            ])
        else:
            clara_exit("Only 'up' and 'down' are valid states")
    elif dargs['health']:
        script_slurm_health = get_from_config("slurm", "script_slurm_health")
        if (len(script_slurm_health) > 0):
            clush(dargs['<nodeset>'], script_slurm_health)
        else:
            clara_exit(
                "You must set a health check program in the configuration file."
            )
    else:
        cmd_list = [
            'job', 'node', 'steps', 'frontend', 'partition', 'reservation',
            'block', 'submp'
        ]

        # /!\ ∀ x, ∀ y, op_list[x][y] ⇒ op_list[x][y] ∈ cmd_list
        op_list = {
            'show': [
                'job', 'node', 'partition', 'reservation', 'steps', 'frontend',
                'block', 'submp'
            ],
            'update': [
                'job', 'node', 'partition', 'reservation', 'steps', 'frontend',
                'block', 'submp'
            ],
            'create': ['partition', 'reservation'],
            'delete': ['partition', 'reservation']
        }

        # /!\ ∀ x ∈ cmd_list ⇒ x ∈ keys_list.keys()
        key_list = {
            'job': 'JobId',
            'steps': 'StepId',
            'node': 'NodeName',
            'frontend': 'FrontendName',
            'partition': 'PartitionName',
            'reservation': 'Reservation',
            'block': 'BlockName',
            'submp': 'SubMPName'
        }

        cmd = dargs['<cmd>']
        subject = dargs['<subject>']
        op = dargs['<op>']
        spec = dargs['<spec>']

        if cmd not in cmd_list:
            clara_exit("Known commands are: {0}".format(" ".join(cmd_list)))

        if spec is None:
            if op is not None and "=" in op:
                spec = [op]
            op = 'show'

        if "=" in op:
            spec = [op] + spec
            op = 'show'

        if op not in op_list:
            clara_exit("Known operations are: {0}".format(" ".join(op_list)))

        if cmd not in op_list[op]:
            clara_exit("You can't use {0} with {1}".format(cmd, op))

        if op == 'show':
            # spec should be empty
            run(["scontrol"] + debug + [op, cmd, subject])
        else:
            run(["scontrol"] + debug + \
                [op,
                 "{0}={1}".format(key_list[cmd], subject),
                 " ".join(spec)])
Example #52
0
def base_install(work_dir, dist):
    ID, VERSION_ID = get_osRelease(dist)
    image = imageInstant(work_dir, ID, VERSION_ID)
    distrib = image.dist
    opts = []

    # bootstrap
    src_list = work_dir + distrib["src_list"]
    if ID == "debian":
        apt_pref = work_dir + distrib["apt_pref"]
        apt_conf = work_dir + distrib["apt_conf"]
        dpkg_conf = work_dir + distrib["dpkg_conf"]
        etc_host = work_dir + "/etc/hosts"

        debiandist = get_from_config("images", "debiandist", dist)
        debmirror = get_from_config("images", "debmirror", dist)

        opts = [debiandist, work_dir, debmirror]

    if ID == "centos":
        rpm_lib = work_dir + distrib["rpm_lib"]
        baseurl = get_from_config("images", "baseurl", dist)
        os.makedirs(rpm_lib)
        run(["rpm", "--root", work_dir, "-initdb"])
        opts = ["install", "-y", "--installroot=" + work_dir, "yum"]

    if conf.ddebug:
        opts = ["--verbose"] + opts

    # Get GPG options
    gpg_check = get_bool_from_config_or("images", "gpg_check", dist, True)
    gpg_keyring = get_from_config_or("images", "gpg_keyring", dist, None)

    if ID == "debian":
        if gpg_check:
            if gpg_keyring is not None:
                opts.insert(0, "--keyring=%s" % gpg_keyring)
        else:
            opts.insert(1, "--no-check-gpg")

    if conf.ddebug:
        opts.insert(1, "--verbose")

    image.bootstrapper(opts)
    if ID == "centos":
        set_yum_src_file(src_list, baseurl, gpg_check)

    if ID == "debian":
        # Prevent services from starting automatically
        policy_rc = work_dir + "/usr/sbin/policy-rc.d"
        with open(policy_rc, 'w') as p_rcd:
            p_rcd.write("exit 101")
        p_rcd.close()

        os.chmod(policy_rc, 0o755)
        # Mirror setup
        list_repos_nonsplitted = get_from_config("images", "list_repos", dist)
        if ';' in list_repos_nonsplitted:
            separator = ';'
        else:
            separator = ','
        list_repos = list_repos_nonsplitted.split(separator)

        with open(src_list, 'w') as fsources:
            for line in list_repos:
                fsources.write(line + '\n')
        os.chmod(src_list, 0o644)

        with open(apt_pref, 'w') as fapt:
            fapt.write("""Package: *
    Pin: release o={0}
    Pin-Priority: 5000
    
    Package: *
    Pin: release o={1}
    Pin-Priority: 6000
    """.format(dist, get_from_config("common", "origin", dist)))
        os.chmod(apt_pref, 0o644)

        # Misc config
        with open(apt_conf, 'w') as fconf:
            fconf.write('Acquire::Check-Valid-Until "false";\n')
        os.chmod(apt_conf, 0o644)

        lists_hosts = get_from_config("images", "etc_hosts", dist).split(",")
        with open(etc_host, 'w') as fhost:
            for elem in lists_hosts:
                if ":" in elem:
                    ip, host = elem.split(":")
                    fhost.write("{0} {1}\n".format(ip, host))
                else:
                    logging.warning(
                        "The option etc_hosts is malformed or missing an argument"
                    )
        os.chmod(etc_host, 0o644)

        with open(dpkg_conf, 'w') as fdpkg:
            fdpkg.write("""# Drop locales except French
path-exclude=/usr/share/locale/*
path-include=/usr/share/locale/fr/*
path-include=/usr/share/locale/locale.alias
# Drop manual pages
# (We keep manual pages in the image)
## path-exclude=/usr/share/man/*
""")
        os.chmod(dpkg_conf, 0o644)

    # Set root password to 'clara'
    part1 = subprocess.Popen(["echo", "root:clara"],
                             stdout=subprocess.PIPE,
                             universal_newlines=True)
    part2 = subprocess.Popen(["chroot", work_dir, "/usr/sbin/chpasswd"],
                             stdin=part1.stdout,
                             universal_newlines=True)
    part1.stdout.close()  # Allow part1 to receive a SIGPIPE if part2 exits.
Example #53
0
def do_ping(hosts):
    nodes = ClusterShell.NodeSet.NodeSet(hosts)
    cmd = ["fping", "-r1", "-u", "-s"] + list(nodes)
    run(cmd)
Example #54
0
def geninitrd(path):
    if (path is None):
        trg_dir = get_from_config("images", "trg_dir", dist)
    else:
        trg_dir = path

    if not os.path.isdir(trg_dir):
        os.makedirs(trg_dir)

    squashfs_file = get_from_config("images", "trg_img", dist)
    if not os.path.isfile(squashfs_file):
        clara_exit("{0} does not exist!".format(squashfs_file))

    if conf.ddebug:
        run(["unsquashfs", "-li", "-f", "-d", work_dir, squashfs_file])
    else:
        run(["unsquashfs", "-f", "-d", work_dir, squashfs_file])

    mount_chroot()

    # Install the kernel in the image
    kver = get_from_config("images", "kver", dist)
    if len(kver) == 0:
        clara_exit("kver hasn't be set in config.ini")
    else:
        run_chroot(["chroot", work_dir, "apt-get", "update"])
        run_chroot([
            "chroot", work_dir, "apt-get", "install",
            "--no-install-recommends", "--yes", "--force-yes",
            "linux-image-" + kver
        ])

    # Install packages from 'packages_initrd'
    packages_initrd = get_from_config("images", "packages_initrd", dist)
    if len(packages_initrd) == 0:
        logging.warning("packages_initrd hasn't be set in config.ini")
    else:
        pkgs = packages_initrd.split(',')
        run_chroot([
            "chroot", work_dir, "apt-get", "install",
            "--no-install-recommends", "--yes", "--force-yes"
        ] + pkgs)

    # Generate the initrd in the image
    run_chroot(
        ["chroot", work_dir, "mkinitramfs", "-o", "/tmp/initrd-" + kver, kver])
    umount_chroot()

    # Copy the initrd out of the chroot
    initrd_file = trg_dir + "/initrd-" + kver
    shutil.copy(work_dir + "/tmp/initrd-" + kver, initrd_file)
    os.chmod(initrd_file, 0o644)
    logging.info("Initrd available at " + initrd_file)

    # Copy vmlinuz out of the chroot
    vmlinuz_file = trg_dir + "/vmlinuz-" + kver
    shutil.copy(work_dir + "/boot/vmlinuz-" + kver, vmlinuz_file)
    os.chmod(vmlinuz_file, 0o644)
    logging.info("vmlinuz available at " + vmlinuz_file)

    # Send files where they will be used
    sftp_mode = has_config_value("images", "hosts", dist)
    dargs = docopt.docopt(__doc__)
    if sftp_mode and not dargs['--no-sync']:
        sftp_user = get_from_config("images", "sftp_user", dist)
        sftp_private_key = get_from_config("images", "sftp_private_key", dist)
        sftp_passphrase = get_from_config_or("images", "sftp_passphrase", dist,
                                             None)
        sftp_hosts = get_from_config("images", "hosts", dist).split(',')
        sftp_client = sftp.Sftp(sftp_hosts, sftp_user, sftp_private_key,
                                sftp_passphrase)
        sftp_client.upload([initrd_file, vmlinuz_file], trg_dir, 0o644)