def do_push(dist=''): push_cmd = get_from_config_or("repo", "push", dist, None) if push_cmd: push_hosts = get_from_config_or("repo", "hosts", dist, '').split(',') if push_hosts and push_hosts[0] is not '': for host in push_hosts: run(push_cmd.format(host).split(' ')) else: run(push_cmd.split(' '))
def push(image, dist): if (image is None): squashfs_file = get_from_config("images", "trg_img", dist) if (squashfs_file == "" or squashfs_file == None): image_name = dist + "_image.squashfs" squashfs_file = "/var/lib/clara/images/" + image_name else: squashfs_file = image if not os.path.isfile(squashfs_file): clara_exit("{0} doesn't exist.".format(squashfs_file)) # Send files where they will be used sftp_mode = has_config_value("images", "hosts", dist) dargs = docopt.docopt(__doc__) if sftp_mode: sftp_user = get_from_config("images", "sftp_user", dist) sftp_private_key = get_from_config("images", "sftp_private_key", dist) sftp_passphrase = get_from_config_or("images", "sftp_passphrase", dist, None) sftp_hosts = get_from_config("images", "hosts", dist).split(',') sftp_client = sftp.Sftp(sftp_hosts, sftp_user, sftp_private_key, sftp_passphrase) sftp_client.upload([squashfs_file], os.path.dirname(squashfs_file), 0o755) else: clara_exit("Hosts not found for the image {0}".format(squashfs_file))
def genimg(image, work_dir, dist): if (image is None): squashfs_file = get_from_config("images", "trg_img", dist) if (squashfs_file == "" or squashfs_file == None): image_name = dist + "_image.squashfs" squashfs_file = "/var/lib/clara/images/" + image_name if os.path.isfile(squashfs_file): os.rename(squashfs_file, squashfs_file + ".old") logging.info( "Previous image renamed to {0}.".format(squashfs_file + ".old")) else: path_to_image = os.path.dirname(image) if not os.path.isdir(path_to_image) and len(path_to_image) != 0: os.makedirs(path_to_image) squashfs_file = image if os.path.isfile(squashfs_file): os.rename(squashfs_file, squashfs_file + ".old") logging.info("Previous image renamed to {0}.".format(squashfs_file + ".old")) logging.info("Creating image at {0}".format(squashfs_file)) if not os.path.exists(os.path.dirname(squashfs_file)): logging.info("Creating local directory path", os.path.dirname(squashfs_file)) os.makedirs(os.path.dirname(squashfs_file)) if conf.ddebug: run([ "mksquashfs", work_dir, squashfs_file, "-no-exports", "-noappend", "-info" ]) else: run([ "mksquashfs", work_dir, squashfs_file, "-no-exports", "-noappend" ]) os.chmod(squashfs_file, 0o755) sftp_mode = has_config_value("images", "hosts", dist) dargs = docopt.docopt(__doc__) if sftp_mode and not dargs['--no-sync']: sftp_user = get_from_config("images", "sftp_user", dist) sftp_private_key = get_from_config("images", "sftp_private_key", dist) sftp_passphrase = get_from_config_or("images", "sftp_passphrase", dist, None) sftp_hosts = get_from_config("images", "hosts", dist).split(',') sftp_client = sftp.Sftp(sftp_hosts, sftp_user, sftp_private_key, sftp_passphrase) sftp_client.upload([squashfs_file], os.path.dirname(squashfs_file), 0o755)
def mktorrent(image): if (image is None): squashfs_file = get_from_config("images", "trg_img", dist) else: squashfs_file = image trackers_port = get_from_config("p2p", "trackers_port", dist) trackers_schema = get_from_config("p2p", "trackers_schema", dist) seeding_service = get_from_config("p2p", "seeding_service", dist) init_stop = get_from_config("p2p", "init_stop", dist) init_start = get_from_config("p2p", "init_start", dist) # trackers is a dictionary with pairs nodeset and torrent file trackers = {} for e in get_from_config("p2p", "trackers", dist).split(";"): k, v = e.split(":") trackers[k] = v # seeders in the config file is a dictionary with pairs nodeset and torrent file seeders_dict = {} for e in get_from_config("p2p", "seeders", dist).split(";"): k, v = e.split(":") seeders_dict[k] = v seeders = ",".join(seeders_dict.keys()) if not os.path.isfile(squashfs_file): clara_exit("{0} doesn't exist".format(squashfs_file)) # Remove old torrent files for f in trackers.values(): if os.path.isfile(f): os.remove(f) clush(seeders, init_stop.format(seeding_service)) sftp_mode = has_config_value("p2p", "sftp_user", dist) if sftp_mode: sftp_user = get_from_config("p2p", "sftp_user", dist) sftp_private_key = get_from_config("p2p", "sftp_private_key", dist) sftp_passphrase = get_from_config_or("p2p", "sftp_passphrase", dist, None) sftp_client = sftp.Sftp(seeders.split(','), sftp_user, sftp_private_key, sftp_passphrase) for e in trackers.keys(): announce = [] for t in list(ClusterShell.NodeSet.NodeSet(e)): announce.append("{0}://{1}:{2}/announce".format(trackers_schema, t, trackers_port)) run(["/usr/bin/mktorrent", "-a", ",".join(announce), "-o", trackers[e], squashfs_file]) if sftp_mode: sftp_client.upload([trackers[e]], os.path.dirname(trackers[e])) clush(seeders, init_start.format(seeding_service))
def get_digest_type(): digest = get_from_config_or("common", "digest_type") if digest == "": logging.warning("Digest type not defined") logging.info("Using default digest type: sha256") digest = "sha256" elif digest not in [ 'md2', 'md5', 'mdc2', 'rmd160', 'sha', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512' ]: logging.warning("Invalid digest type : {0}".format(digest)) logging.info("Using default digest type: sha256 ") digest = "sha256" return digest
def main(): logging.debug(sys.argv) dargs = docopt.docopt(__doc__) global keep_chroot_dir keep_chroot_dir = False # Not executed in the following cases # - the program dies because of a signal # - os._exit() is invoked directly # - a Python fatal error is detected (in the interpreter) atexit.register(clean_and_exit) global dist dist = get_from_config("common", "default_distribution") if dargs['<dist>'] is not None: dist = dargs["<dist>"] if dist not in get_from_config("common", "allowed_distributions"): clara_exit("{0} is not a know distribution".format(dist)) global work_dir if dargs['repack']: work_dir = dargs['<directory>'] else: tmpdir = get_from_config_or("images", "tmp_dir", dist, "/tmp") work_dir = tempfile.mkdtemp(prefix="tmpClara", dir=tmpdir) if dargs['create']: if dargs["--keep-chroot-dir"]: keep_chroot_dir = True base_install() install_files() system_install() remove_files() run_script_post_creation() genimg(dargs['<image>']) elif dargs['repack']: genimg(dargs['--image']) elif dargs['unpack']: extract_image(dargs['--image']) elif dargs['initrd']: geninitrd(dargs['--output']) elif dargs['edit']: edit(dargs['<image>']) elif dargs['push']: push(dargs['<image>'])
def do_connect(host, j=False, f=False): nodeset = ClusterShell.NodeSet.NodeSet(host) if (len(nodeset) != 1): clara_exit('Only one host allowed for this command') pat = re.compile("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}") if pat.match(host): logging.debug( "The host is an IP adddres: {0}. Using ipmitool without conman.". format(host)) do_connect_ipmi(host) else: conmand = get_from_config_or("ipmi", "conmand", '') port = int(get_from_config("ipmi", "port")) if (len(conmand) == 0): do_connect_ipmi(host) return s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: s.connect((conmand, port)) os.environ["CONMAN_ESCAPE"] = '!' cmd = ["conman"] if j: cmd = cmd + ["-j"] if f: cmd = cmd + ["-f"] cmd = cmd + ["-d", conmand, host] run(cmd, exit_on_error=False) except socket.error as e: logging.debug( "Conman not running. Message on connect: Errno {0} - {1}". format(e.errno, e.strerror)) do_connect_ipmi(host) except RuntimeError as e: logging.warning("Conman failed, fallback to ipmitool") do_connect_ipmi(host) s.close()
def do_connect(host, j=False, f=False): nodeset = ClusterShell.NodeSet.NodeSet(host) if (len(nodeset) != 1): clara_exit('Only one host allowed for this command') pat = re.compile("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}") if pat.match(host): logging.debug( "The host is an IP adddres: {0}. Using ipmitool without conman.". format(host)) do_connect_ipmi(host) else: conmand = get_from_config_or("ipmi", "conmand", '') port = int(get_from_config("ipmi", "port")) if len(conmand) == 0: do_connect_ipmi(host) return cmd = [] ssh_jhost = get_bool_from_config_or("ipmi", "ssh_jump_host") if ssh_jhost: cmd += ["ssh", "-t", conmand] conmand = "localhost" cmd += ["conman"] try: if j: cmd = cmd + ["-j"] if f: cmd = cmd + ["-f"] cmd = cmd + ["-d", conmand, host, "-e!"] run(cmd, exit_on_error=False) except RuntimeError as e: logging.warning("Conman failed, fallback to ipmitool") do_connect_ipmi(host)
def geninitrd(path, work_dir, dist): ID, VERSION_ID = get_osRelease(dist) image = imageInstant(work_dir, ID, VERSION_ID) distrib = image.dist if (path is None): trg_dir = get_from_config("images", "trg_dir", dist) if (trg_dir == "" or trg_dir == None): trg_dir = "/var/lib/clara/images/" else: trg_dir = path if not os.path.isdir(trg_dir): os.makedirs(trg_dir) squashfs_file = get_from_config("images", "trg_img", dist) if (squashfs_file == "" or squashfs_file == None): image_name = dist + "_image.squashfs" squashfs_file = "/var/lib/clara/images/" + image_name if not os.path.isfile(squashfs_file): clara_exit("{0} does not exist!".format(squashfs_file)) if conf.ddebug: run(["unsquashfs", "-li", "-f", "-d", work_dir, squashfs_file]) else: run(["unsquashfs", "-f", "-d", work_dir, squashfs_file]) mount_chroot(work_dir) # Install the kernel in the image kver = get_from_config("images", "kver", dist) if len(kver) == 0: clara_exit("kver hasn't be set in config.ini") else: run_chroot(["chroot", work_dir, distrib["pkgManager"], "update"], work_dir) if ID == "debian": run_chroot([ "chroot", work_dir, distrib["pkgManager"], "install", "--no-install-recommends", "--yes", "--force-yes", "linux-image-" + kver ], work_dir) if ID == "centos": run_chroot([ "chroot", work_dir, distrib["pkgManager"], "install", "-y", "kernel-" + kver ], work_dir) # Install packages from 'packages_initrd' packages_initrd = get_from_config("images", "packages_initrd", dist) if len(packages_initrd) == 0: logging.warning("packages_initrd hasn't be set in config.ini") else: pkgs = packages_initrd.split(',') if ID == "debian": opts = ["--no-install-recommends", "--yes", "--force-yes"] intitrd_opts = ["-o", "/tmp/initrd-" + kver, kver] if ID == "centos": opts = ["-y"] intitrd_opts = [ "--force", "--add", "livenet", "-v", "/tmp/initrd-" + kver, "--kver", kver ] opts = ["chroot", work_dir, distrib["pkgManager"], "install" ] + opts + pkgs run_chroot(opts, work_dir) # Generate the initrd in the image intitrd_opts = ["chroot", work_dir, distrib["initrdGen"]] + intitrd_opts run_chroot(intitrd_opts, work_dir) umount_chroot(work_dir) # Copy the initrd out of the chroot initrd_file = trg_dir + "/initrd-" + kver shutil.copy(work_dir + "/tmp/initrd-" + kver, initrd_file) os.chmod(initrd_file, 0o644) logging.info("Initrd available at " + initrd_file) # Copy vmlinuz out of the chroot vmlinuz_file = trg_dir + "/vmlinuz-" + kver shutil.copy(work_dir + "/boot/vmlinuz-" + kver, vmlinuz_file) os.chmod(vmlinuz_file, 0o644) logging.info("vmlinuz available at " + vmlinuz_file) # Send files where they will be used sftp_mode = has_config_value("images", "hosts", dist) dargs = docopt.docopt(__doc__) if sftp_mode and not dargs['--no-sync']: sftp_user = get_from_config("images", "sftp_user", dist) sftp_private_key = get_from_config("images", "sftp_private_key", dist) sftp_passphrase = get_from_config_or("images", "sftp_passphrase", dist, None) sftp_hosts = get_from_config("images", "hosts", dist).split(',') sftp_client = sftp.Sftp(sftp_hosts, sftp_user, sftp_private_key, sftp_passphrase) sftp_client.upload([initrd_file, vmlinuz_file], trg_dir, 0o644)
def base_install(work_dir, dist): ID, VERSION_ID = get_osRelease(dist) image = imageInstant(work_dir, ID, VERSION_ID) distrib = image.dist opts = [] # bootstrap src_list = work_dir + distrib["src_list"] if ID == "debian": apt_pref = work_dir + distrib["apt_pref"] apt_conf = work_dir + distrib["apt_conf"] dpkg_conf = work_dir + distrib["dpkg_conf"] etc_host = work_dir + "/etc/hosts" debiandist = get_from_config("images", "debiandist", dist) debmirror = get_from_config("images", "debmirror", dist) opts = [debiandist, work_dir, debmirror] if ID == "centos": rpm_lib = work_dir + distrib["rpm_lib"] baseurl = get_from_config("images", "baseurl", dist) os.makedirs(rpm_lib) run(["rpm", "--root", work_dir, "-initdb"]) opts = ["install", "-y", "--installroot=" + work_dir, "yum"] if conf.ddebug: opts = ["--verbose"] + opts # Get GPG options gpg_check = get_bool_from_config_or("images", "gpg_check", dist, True) gpg_keyring = get_from_config_or("images", "gpg_keyring", dist, None) if ID == "debian": if gpg_check: if gpg_keyring is not None: opts.insert(0, "--keyring=%s" % gpg_keyring) else: opts.insert(1, "--no-check-gpg") if conf.ddebug: opts.insert(1, "--verbose") image.bootstrapper(opts) if ID == "centos": set_yum_src_file(src_list, baseurl, gpg_check) if ID == "debian": # Prevent services from starting automatically policy_rc = work_dir + "/usr/sbin/policy-rc.d" with open(policy_rc, 'w') as p_rcd: p_rcd.write("exit 101") p_rcd.close() os.chmod(policy_rc, 0o755) # Mirror setup list_repos_nonsplitted = get_from_config("images", "list_repos", dist) if ';' in list_repos_nonsplitted: separator = ';' else: separator = ',' list_repos = list_repos_nonsplitted.split(separator) with open(src_list, 'w') as fsources: for line in list_repos: fsources.write(line + '\n') os.chmod(src_list, 0o644) with open(apt_pref, 'w') as fapt: fapt.write("""Package: * Pin: release o={0} Pin-Priority: 5000 Package: * Pin: release o={1} Pin-Priority: 6000 """.format(dist, get_from_config("common", "origin", dist))) os.chmod(apt_pref, 0o644) # Misc config with open(apt_conf, 'w') as fconf: fconf.write('Acquire::Check-Valid-Until "false";\n') os.chmod(apt_conf, 0o644) lists_hosts = get_from_config("images", "etc_hosts", dist).split(",") with open(etc_host, 'w') as fhost: for elem in lists_hosts: if ":" in elem: ip, host = elem.split(":") fhost.write("{0} {1}\n".format(ip, host)) else: logging.warning( "The option etc_hosts is malformed or missing an argument" ) os.chmod(etc_host, 0o644) with open(dpkg_conf, 'w') as fdpkg: fdpkg.write("""# Drop locales except French path-exclude=/usr/share/locale/* path-include=/usr/share/locale/fr/* path-include=/usr/share/locale/locale.alias # Drop manual pages # (We keep manual pages in the image) ## path-exclude=/usr/share/man/* """) os.chmod(dpkg_conf, 0o644) # Set root password to 'clara' part1 = subprocess.Popen(["echo", "root:clara"], stdout=subprocess.PIPE, universal_newlines=True) part2 = subprocess.Popen(["chroot", work_dir, "/usr/sbin/chpasswd"], stdin=part1.stdout, universal_newlines=True) part1.stdout.close() # Allow part1 to receive a SIGPIPE if part2 exits.
def base_install(): # Debootstrap src_list = work_dir + "/etc/apt/sources.list" apt_pref = work_dir + "/etc/apt/preferences.d/00custompreferences" apt_conf = work_dir + "/etc/apt/apt.conf.d/99nocheckvalid" dpkg_conf = work_dir + "/etc/dpkg/dpkg.cfg.d/excludes" etc_host = work_dir + "/etc/hosts" debiandist = get_from_config("images", "debiandist", dist) debmirror = get_from_config("images", "debmirror", dist) # Get GPG options gpg_check = get_bool_from_config_or("images", "gpg_check", dist, True) gpg_keyring = get_from_config_or("images", "gpg_keyring", dist, None) cmd = ["debootstrap", debiandist, work_dir, debmirror] if gpg_check: if gpg_keyring is not None: cmd.insert(1, "--keyring=%s" % gpg_keyring) else: cmd.insert(1, "--no-check-gpg") if conf.ddebug: cmd.insert(1, "--verbose") run(cmd) # Prevent services from starting automatically policy_rc = work_dir + "/usr/sbin/policy-rc.d" with open(policy_rc, 'w') as p_rcd: p_rcd.write("exit 101") p_rcd.close() os.chmod(policy_rc, 0o755) # Mirror setup list_repos_nonsplitted = get_from_config("images", "list_repos", dist) if ';' in list_repos_nonsplitted: separator = ';' else: separator = ',' list_repos = list_repos_nonsplitted.split(separator) with open(src_list, 'w') as fsources: for line in list_repos: fsources.write(line + '\n') os.chmod(src_list, 0o644) with open(apt_pref, 'w') as fapt: fapt.write("""Package: * Pin: release o={0} Pin-Priority: 5000 Package: * Pin: release o={1} Pin-Priority: 6000 """.format(dist, get_from_config("common", "origin", dist))) os.chmod(apt_pref, 0o644) # Misc config with open(apt_conf, 'w') as fconf: fconf.write('Acquire::Check-Valid-Until "false";\n') os.chmod(apt_conf, 0o644) lists_hosts = get_from_config("images", "etc_hosts", dist).split(",") with open(etc_host, 'w') as fhost: for elem in lists_hosts: if ":" in elem: ip, host = elem.split(":") fhost.write("{0} {1}\n".format(ip, host)) else: logging.warning( "The option etc_hosts is malformed or missing an argument") os.chmod(etc_host, 0o644) with open(dpkg_conf, 'w') as fdpkg: fdpkg.write("""# Drop locales except French path-exclude=/usr/share/locale/* path-include=/usr/share/locale/fr/* path-include=/usr/share/locale/locale.alias # Drop manual pages # (We keep manual pages in the image) ## path-exclude=/usr/share/man/* """) os.chmod(dpkg_conf, 0o644) # Set root password to 'clara' part1 = subprocess.Popen(["echo", "root:clara"], stdout=subprocess.PIPE) part2 = subprocess.Popen(["chroot", work_dir, "/usr/sbin/chpasswd"], stdin=part1.stdout) part1.stdout.close() # Allow part1 to receive a SIGPIPE if part2 exits.
def full_hostname(host): prefix = get_from_config("ipmi", "prefix") suffix = get_from_config_or("ipmi", "suffix", "") return (prefix + host + suffix)
def geninitrd(path): if (path is None): trg_dir = get_from_config("images", "trg_dir", dist) else: trg_dir = path if not os.path.isdir(trg_dir): os.makedirs(trg_dir) squashfs_file = get_from_config("images", "trg_img", dist) if not os.path.isfile(squashfs_file): clara_exit("{0} does not exist!".format(squashfs_file)) if conf.ddebug: run(["unsquashfs", "-li", "-f", "-d", work_dir, squashfs_file]) else: run(["unsquashfs", "-f", "-d", work_dir, squashfs_file]) mount_chroot() # Install the kernel in the image kver = get_from_config("images", "kver", dist) if len(kver) == 0: clara_exit("kver hasn't be set in config.ini") else: run_chroot(["chroot", work_dir, "apt-get", "update"]) run_chroot([ "chroot", work_dir, "apt-get", "install", "--no-install-recommends", "--yes", "--force-yes", "linux-image-" + kver ]) # Install packages from 'packages_initrd' packages_initrd = get_from_config("images", "packages_initrd", dist) if len(packages_initrd) == 0: logging.warning("packages_initrd hasn't be set in config.ini") else: pkgs = packages_initrd.split(',') run_chroot([ "chroot", work_dir, "apt-get", "install", "--no-install-recommends", "--yes", "--force-yes" ] + pkgs) # Generate the initrd in the image run_chroot( ["chroot", work_dir, "mkinitramfs", "-o", "/tmp/initrd-" + kver, kver]) umount_chroot() # Copy the initrd out of the chroot initrd_file = trg_dir + "/initrd-" + kver shutil.copy(work_dir + "/tmp/initrd-" + kver, initrd_file) os.chmod(initrd_file, 0o644) logging.info("Initrd available at " + initrd_file) # Copy vmlinuz out of the chroot vmlinuz_file = trg_dir + "/vmlinuz-" + kver shutil.copy(work_dir + "/boot/vmlinuz-" + kver, vmlinuz_file) os.chmod(vmlinuz_file, 0o644) logging.info("vmlinuz available at " + vmlinuz_file) # Send files where they will be used sftp_mode = has_config_value("images", "hosts", dist) dargs = docopt.docopt(__doc__) if sftp_mode and not dargs['--no-sync']: sftp_user = get_from_config("images", "sftp_user", dist) sftp_private_key = get_from_config("images", "sftp_private_key", dist) sftp_passphrase = get_from_config_or("images", "sftp_passphrase", dist, None) sftp_hosts = get_from_config("images", "hosts", dist).split(',') sftp_client = sftp.Sftp(sftp_hosts, sftp_user, sftp_private_key, sftp_passphrase) sftp_client.upload([initrd_file, vmlinuz_file], trg_dir, 0o644)
def do_key(): key = get_from_config("repo", "gpg_key") fnull = open(os.devnull, 'w') cmd = ['gpg', '--list-secret-keys', key] logging.debug("repo/do_key: {0}".format(" ".join(cmd))) retcode = subprocess.call(cmd, stdout=fnull, stderr=fnull) fnull.close() # We import the key if it hasn't been imported before if retcode != 0: file_stored_key = get_from_config("repo", "stored_enc_key") if os.path.isfile(file_stored_key): password = value_from_file( get_from_config("common", "master_passwd_file"), "ASUPASSWD") digest = get_from_config_or("common", "digest_type", default="sha256") if digest not in [ 'md2', 'md5', 'mdc2', 'rmd160', 'sha', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512' ]: logging.warning( "Invalid digest type : {0}, using default digest type: sha256" .format(digest)) digest = "sha256" if len(password) > 20: fdesc, temp_path = tempfile.mkstemp(prefix="tmpClara") cmd = [ 'openssl', 'aes-256-cbc', '-md', digest, '-d', '-in', file_stored_key, '-out', temp_path, '-k', password ] logging.debug("repo/do_key: {0}".format(" ".join(cmd))) retcode = subprocess.call(cmd) if retcode != 0: os.close(fdesc) os.remove(temp_path) clara_exit('Command failed {0}'.format(" ".join(cmd))) else: logging.info("Trying to import key {0}".format(key)) fnull = open(os.devnull, 'w') cmd = [ 'gpg', '--allow-secret-key-import', '--import', temp_path ] logging.debug("repo/do_key: {0}".format(" ".join(cmd))) retcode = subprocess.call(cmd) fnull.close() os.close(fdesc) os.remove(temp_path) if retcode != 0: logging.info( "\nThere was a problem with the import, make sure the key you imported " "from {0} is the same you have in your configuration: {1}" .format(file_stored_key, key)) else: clara_exit( "There was some problem while reading ASUPASSWD's value") else: clara_exit('Unable to read: {0}'.format(file_stored_key)) else: logging.info("GPG key was already imported.")