def main(): logging.debug(sys.argv) dargs = docopt.docopt(__doc__) if dargs['show'] or dargs['edit'] or dargs['decode']: if not dargs['<file>'].endswith(".enc"): clara_exit("The filename {0} doesn't end with '.enc'.\n" "All encrypted files must have the suffix '.enc'".format(dargs['<file>'])) if dargs['encode']: if dargs['<file>'].endswith(".enc"): clara_exit("The filename {0} ends with '.enc'.\n" "This file is probably already encrypted.".format(dargs['<file>'])) if dargs['show']: f = do("decrypt", dargs['<file>']) subprocess.call(['sensible-pager', f.name]) f.close() elif dargs['edit']: do_edit(dargs['<file>']) elif dargs['encode']: f = do("encrypt", dargs['<file>']) shutil.copy(f.name, dargs['<file>'] + ".enc") f.close() elif dargs['decode']: f = do("decrypt", dargs['<file>']) if dargs['<file>'].endswith(".enc"): shutil.copy(f.name, dargs['<file>'][0:-4]) f.close()
def main(): logging.debug(sys.argv) dargs = docopt.docopt(__doc__) global work_dir, src_list, dist dist = get_from_config("common", "default_distribution") if dargs['<dist>'] is not None: dist = dargs["<dist>"] if dist not in get_from_config("common", "allowed_distributions"): clara_exit("{0} is not a know distribution".format(dist)) if dargs['<chroot_dir>'] is not None: work_dir = dargs['<chroot_dir>'] else: work_dir = get_from_config("chroot", "trg_dir", dist) src_list = work_dir + "/etc/apt/sources.list" # Not executed in the following cases # - the program dies because of a signal # - os._exit() is invoked directly # - a Python fatal error is detected (in the interpreter) atexit.register(clean_and_exit) if dargs['create']: base_install() system_install() install_files() install_https_apt() remove_files() run_script_post_creation() elif dargs['edit']: edit(dargs['<chroot_dir>']) elif dargs['install']: install_packages(dargs['<packages>']) elif dargs['remove']: remove_packages(dargs['<packages>'])
def umount_chroot(): if os.path.ismount(work_dir + "/proc/sys/fs/binfmt_misc"): run(["chroot", work_dir, "umount", "/proc/sys/fs/binfmt_misc"]) if os.path.ismount(work_dir + "/sys"): run(["chroot", work_dir, "umount", "/sys"]) if os.path.ismount(work_dir + "/proc"): run(["chroot", work_dir, "umount", "/proc"]) try: extra_bind_mounts = get_from_config("chroot", "extra_bind_mounts", dist).split(",") except: extra_bind_mounts = None if not extra_bind_mounts: logging.warning("extra_bind_mounts is not specified in config.ini") else: for mounts_params in extra_bind_mounts: mountpoint = work_dir + mounts_params.split(" ")[1] with open("/proc/mounts", "r") as file_to_read: for line in file_to_read: if mountpoint in line: run(["umount", mountpoint]) time.sleep(1) # Wait one second so the system has time to unmount with open("/proc/mounts", "r") as file_to_read: for line in file_to_read: if work_dir in line: clara_exit("Something went wrong when umounting in the chroot")
def mktorrent(image): if (image is None): squashfs_file = get_from_config("images", "trg_img", dist) else: squashfs_file = image seeders = get_from_config("p2p", "seeders", dist) trackers_port = get_from_config("p2p", "trackers_port", dist) trackers_schema = get_from_config("p2p", "trackers_schema", dist) seeding_service = get_from_config("p2p", "seeding_service", dist) init_stop = get_from_config("p2p", "init_stop", dist) init_start = get_from_config("p2p", "init_start", dist) # trackers is a dictionary with pairs nodeset and torrent file trackers = {} for e in get_from_config("p2p", "trackers", dist).split(";"): k, v = e.split(":") trackers[k] = v if not os.path.isfile(squashfs_file): clara_exit("The file {0} doesn't exist".format(squashfs_file)) # Remove old torrent files for f in trackers.values(): if os.path.isfile(f): os.remove(f) clush(seeders, init_stop.format(seeding_service)) for e in trackers.keys(): announce = [] for t in list(ClusterShell.NodeSet.NodeSet(e)): announce.append("{0}://{1}:{2}/announce".format(trackers_schema, t, trackers_port)) run(["/usr/bin/mktorrent", "-a", ",".join(announce), "-o", trackers[e], squashfs_file]) clush(seeders, init_start.format(seeding_service))
def do(op, origfile): password = get_encryption_key() digest = get_digest_type() f = tempfile.NamedTemporaryFile(prefix="tmpClara") if op == "decrypt": cmd = [ 'openssl', 'aes-256-cbc', '-md', digest, '-d', '-in', origfile, '-out', f.name, '-k', password ] elif op == "encrypt": cmd = [ 'openssl', 'aes-256-cbc', '-md', digest, '-in', origfile, '-out', f.name, '-k', password ] cmd_log = cmd[:-1] + ["Password"] logging.debug("enc/do: {0}".format(" ".join(cmd_log))) retcode = subprocess.call(cmd) if retcode != 0: f.close() clara_exit('Command failed {0}'.format(" ".join(cmd_log))) else: return f
def do_connect(host, j=False, f=False): nodeset = ClusterShell.NodeSet.NodeSet(host) if (len(nodeset) != 1): clara_exit('Only one host allowed for this command') pat = re.compile("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}") if pat.match(host): logging.debug( "The host is an IP adddres: {0}. Using ipmitool without conman.". format(host)) do_connect_ipmi(host) else: conmand = get_from_config("ipmi", "conmand") port = int(get_from_config("ipmi", "port")) if (len(conmand) == 0): clara_exit("You must set the configuration paramenter 'conmand'") s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: s.connect((conmand, port)) os.environ["CONMAN_ESCAPE"] = '!' cmd = ["conman"] if j: cmd = cmd + ["-j"] if f: cmd = cmd + ["-f"] cmd = cmd + ["-d", conmand, host] run(cmd) except socket.error as e: logging.debug( "Conman not running. Message on connect: Errno {0} - {1}". format(e.errno, e.strerror)) do_connect_ipmi(host) s.close()
def push(image, dist): if (image is None): squashfs_file = get_from_config("images", "trg_img", dist) if (squashfs_file == "" or squashfs_file == None): image_name = dist + "_image.squashfs" squashfs_file = "/var/lib/clara/images/" + image_name else: squashfs_file = image if not os.path.isfile(squashfs_file): clara_exit("{0} doesn't exist.".format(squashfs_file)) # Send files where they will be used sftp_mode = has_config_value("images", "hosts", dist) dargs = docopt.docopt(__doc__) if sftp_mode: sftp_user = get_from_config("images", "sftp_user", dist) sftp_private_key = get_from_config("images", "sftp_private_key", dist) sftp_passphrase = get_from_config_or("images", "sftp_passphrase", dist, None) sftp_hosts = get_from_config("images", "hosts", dist).split(',') sftp_client = sftp.Sftp(sftp_hosts, sftp_user, sftp_private_key, sftp_passphrase) sftp_client.upload([squashfs_file], os.path.dirname(squashfs_file), 0o755) else: clara_exit("Hosts not found for the image {0}".format(squashfs_file))
def main(): logging.debug(sys.argv) dargs = docopt.docopt(__doc__) global dist dist = get_from_config("common", "default_distribution") if dargs["<dist>"] is not None: dist = dargs["<dist>"] if dist not in get_from_config("common", "allowed_distributions"): clara_exit("{0} is not a know distribution".format(dist)) trackers = get_from_config("p2p", "trackers", dist) seeders = get_from_config("p2p", "seeders", dist) tracking_service = get_from_config("p2p", "tracking_service", dist) seeding_service = get_from_config("p2p", "seeding_service", dist) if dargs['status']: init_status = get_from_config("p2p", "init_status") clush(trackers, init_status.format(tracking_service)) clush(seeders, init_status.format(seeding_service)) elif dargs['restart']: init_stop = get_from_config("p2p", "init_stop") clush(seeders, init_stop.format(seeding_service)) clush(trackers, init_stop.format(tracking_service)) time.sleep(1) init_start = get_from_config("p2p", "init_start") clush(trackers, init_start.format(tracking_service)) clush(seeders, init_start.format(seeding_service)) elif dargs['mktorrent']: mktorrent(dargs['--image'])
def do_connect(host, j=False, f=False): nodeset = ClusterShell.NodeSet.NodeSet(host) if (len(nodeset) != 1): clara_exit('Only one host allowed for this command') pat = re.compile("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}") if pat.match(host): logging.debug("The host is an IP adddres: {0}. Using ipmitool without conman.".format(host)) ipmi_do(host, True, "sol", "activate") else: conmand = get_from_config("ipmi", "conmand") port = int(get_from_config("ipmi", "port")) if (len(conmand) == 0): clara_exit("You must set the paramenter 'conmand' in the configuration file") s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: s.connect((conmand, port)) os.environ["CONMAN_ESCAPE"] = '!' cmd = ["conman"] if j: cmd = cmd + ["-j"] if f: cmd = cmd + ["-f"] cmd = cmd + ["-d", conmand, host] run(cmd) except socket.error as e: logging.debug("Conman not running. Message on connect: {0}" % e) ipmi_do(host, True, "sol", "activate") s.close()
def do_reprepro(action, package=None, flags=None): repo_dir = get_from_config("repo", "repo_dir", dist) reprepro_config = repo_dir + '/conf/distributions' if not os.path.isfile(reprepro_config): clara_exit( "There is not configuration for the local repository for {0}. Run first 'clara repo init <dist>'" .format(dist)) list_flags = ['--silent', '--ask-passphrase'] if conf.ddebug: list_flags = ['-V', '--ask-passphrase'] if flags is not None: list_flags.append(flags) cmd = ['reprepro'] + list_flags + \ ['--basedir', get_from_config("repo", "repo_dir", dist), '--outdir', get_from_config("repo", "mirror_local", dist), action, dist] if package is not None: cmd.append(package) run(cmd)
def geninitrd(path): if (path is None): trg_dir = get_from_config("images", "trg_dir", dist) else: trg_dir = path if not os.path.isdir(trg_dir): os.makedirs(trg_dir) squashfs_file = get_from_config("images", "trg_img", dist) if not os.path.isfile(squashfs_file): clara_exit("The image {0} does not exist!".format(squashfs_file)) if conf.ddebug: run(["unsquashfs", "-li", "-f", "-d", work_dir, squashfs_file]) else: run(["unsquashfs", "-f", "-d", work_dir, squashfs_file]) mount_chroot() # Install the kernel in the image kver = get_from_config("images", "kver", dist) if len(kver) == 0: clara_exit("kver hasn't be set in config.ini") else: run_chroot(["chroot", work_dir, "apt-get", "update"]) run_chroot([ "chroot", work_dir, "apt-get", "install", "--no-install-recommends", "--yes", "--force-yes", "linux-image-" + kver ]) # Install packages from 'packages_initrd' packages_initrd = get_from_config("images", "packages_initrd", dist) if len(packages_initrd) == 0: logging.warning("packages_initrd hasn't be set in config.ini") else: pkgs = packages_initrd.split(',') run_chroot([ "chroot", work_dir, "apt-get", "install", "--no-install-recommends", "--yes", "--force-yes" ] + pkgs) # Generate the initrd in the image run_chroot( ["chroot", work_dir, "mkinitramfs", "-o", "/tmp/initrd-" + kver, kver]) umount_chroot() # Copy the initrd out of the chroot shutil.copy(work_dir + "/tmp/initrd-" + kver, trg_dir + "/initrd-" + kver) os.chmod(trg_dir + "/initrd-" + kver, 0o644) logging.info("Initrd available at " + trg_dir + "/initrd-" + kver) # Copy vmlinuz out of the chroot shutil.copy(work_dir + "/boot/vmlinuz-" + kver, trg_dir + "/vmlinuz-" + kver) logging.info("vmlinuz available at " + trg_dir + "/vmlinuz-" + kver)
def run_chroot(cmd): logging.debug("images/run_chroot: {0}".format(" ".join(cmd))) try: retcode = subprocess.call(cmd) except OSError, e: if (e.errno == errno.ENOENT): clara_exit( "Binary not found, check your path and/or retry as root. \ You were trying to run:\n {0}".format(" ".join(cmd)))
def get(config, section, value): # If the value is not in the override section, look in "repos" from the config.ini if config.has_option(section, value): return config.get(section, value).strip() else: try: return get_from_config("repo", value) except: clara_exit("Value '{0}' not found in section '{1}'".format(value, section))
def main(): logging.debug(sys.argv) dargs = docopt.docopt(__doc__) virt_conf = VirtConf(dargs['--virt-config']) config_dir = os.path.dirname(dargs['--virt-config']) template_dir = os.path.join(config_dir, 'templates') try: virt_conf.read() except VirtConfigurationException, err: utils.clara_exit("Configuration Error: %s" % err)
def get_encryption_key(): master_passwd_file = get_from_config("common", "master_passwd_file") if os.path.isfile(master_passwd_file): password = value_from_file(master_passwd_file, "ASUPASSWD") if len(password) > 20: return password else: clara_exit("There was some problem reading the value of ASUPASSWD") else: clara_exit("Unable to read: {0}".format(master_passwd_file))
def main(): logging.debug(sys.argv) dargs = docopt.docopt(__doc__) virt_conf = VirtConf(dargs['--virt-config']) config_dir = os.path.dirname(dargs['--virt-config']) template_dir = os.path.join(config_dir, 'templates') try: virt_conf.read() except VirtConfigurationException as err: utils.clara_exit("Configuration Error: %s" % err) params = { 'config_dir': config_dir, 'template_dir': template_dir, } if dargs['list']: show_hosts = dargs['--details'] show_volumes = dargs['--details'] if '--host' in dargs.keys(): host_name = dargs['--host'] else: host_name = None do_list(virt_conf, show_hosts, show_volumes, host_name) else: params['vm_names'] = ClusterShell.NodeSet.NodeSet(dargs['<vm_names>']) if '--host' in dargs.keys(): params['host'] = dargs['--host'] else: params['host'] = None if dargs['define']: params['template'] = dargs['--template'] do_define(virt_conf, params) elif dargs['undefine']: do_action(virt_conf, params, 'undefine') elif dargs['start']: params['wipe'] = dargs['--wipe'] do_action(virt_conf, params, 'start') elif dargs['stop']: params['hard'] = dargs['--hard'] do_action(virt_conf, params, 'stop') elif dargs['migrate']: params['dest_host'] = dargs['--dest-host'] do_action(virt_conf, params, 'migrate') elif dargs['getmacs']: params['template'] = dargs['--template'] do_getmacs(virt_conf, params)
def do_sync(selected_dist, input_suites=[]): info_suites = {} # Contains all the information suite_dist = {} # Contains the pairs suite - webdir all_suites = [] # Contains a list with all the suites names for distribution in get_from_config("common", "allowed_distributions").split(","): elements = get_from_config("repo", "info_suites", distribution).split(",") tmp_info_suites = {} for e in elements: k, v = e.split(":") all_suites.append(k) suite_dist[k] = distribution tmp_info_suites[k] = v info_suites[distribution] = tmp_info_suites suites = [] if selected_dist == 'all': # We sync everything suites = all_suites elif len(input_suites) == 0: # We only sync suites related to the default distribution suites = info_suites[selected_dist].keys() else: # If we select one or several suites, we check that are valid for s in input_suites: if s not in all_suites: clara_exit("{0} is not a valid suite. Valid suites are: {1}".format(s, " ".join(all_suites))) suites = input_suites logging.debug("The suites to sync are: {0}.".format(" ".join(suites))) for s in suites: mirror_root = get_from_config("repo", "mirror_root", suite_dist[s]) dm_server = get_from_config("repo", "server", suite_dist[s]) dm_root = info_suites[suite_dist[s]][s] suite_name = s if s in ["wheezy-security", "jessie-security"]: suite_name = s.split("-")[0] + "/updates" archs = get_from_config("repo", "archs", suite_dist[s]) sections = get_from_config("repo", "sections", suite_dist[s]) extra = [] if conf.ddebug: # if extra debug for 3rd party software extra = ['--debug'] run(['debmirror'] + extra + ["--diff=none", "--method=http", "--nosource", "--ignore-release-gpg", "--ignore-missing-release", "--arch={0}".format(archs), "--host={0}".format(dm_server), "--root={0}".format(dm_root), "--dist={0}".format(suite_name), "--section={0}".format(sections), mirror_root + "/" + s])
def edit(chroot_dir, dist): if not os.path.isdir(chroot_dir): clara_exit("The directory {0} doesn't exist.".format(chroot_dir)) # Work in the chroot mount_chroot(chroot_dir, dist) logging.info( "Entering into the chroot to edit. ^d when you have finished.") os.putenv("debian_chroot", "clara-chroot-{0}".format(dist)) run_chroot(["chroot", work_dir]) clara_exit("Exiting the chroot {0}.".format(chroot_dir))
def do_init(): import shutil repo_dir = get_from_config("repo", "repo_dir", dist) reprepro_config = repo_dir + '/conf/distributions' mirror_local = get_from_config("repo", "mirror_local", dist) if (mirror_local == "" or mirror_local == None): mirror_local = repo_dir + '/mirror' if os.path.isdir(repo_dir): clara_exit("The repository '{0}' already exists !".format(repo_dir)) else: if not os.path.isfile(reprepro_config): try: if not os.path.isdir(repo_dir + '/conf'): os.makedirs(repo_dir + '/conf') freprepro = open(reprepro_config, 'w') freprepro.write("""Origin: {0} Label: {1} Suite: {1} Codename: {1} Version: {2} Architectures: amd64 source Components: main contrib non-free UDebComponents: main SignWith: {3} Description: Depot Local {4} DebIndices: Packages Release . .gz .bz2 DscIndices: Sources Release . .gz .bz2 """.format(get_from_config("common", "origin", dist), dist, get_from_config("repo", "version", dist), get_from_config("repo", "gpg_key", dist), get_from_config("repo", "clustername", dist))) freprepro.close() os.chdir(repo_dir) list_flags = ['--ask-passphrase'] if conf.ddebug: list_flags.append("-V") run(['reprepro'] + list_flags + [ '--basedir', repo_dir, '--outdir', mirror_local, 'export', dist ]) except: shutil.rmtree(repo_dir) clara_exit( "The repository '{0}' has not been initialized properly, it will be deleted !" .format(repo_dir))
def edit(image, work_dir, dist): if (image is None): squashfs_file = get_from_config("images", "trg_img", dist) if (squashfs_file == "" or squashfs_file == None): image_name = dist + "_image.squashfs" squashfs_file = "/var/lib/clara/images/" + image_name else: squashfs_file = image if not os.path.isfile(squashfs_file): clara_exit("{0} doesn't exist.".format(squashfs_file)) # Extract the image. logging.info("Extracting {0} to {1} ...".format(squashfs_file, work_dir)) if conf.ddebug: run(["unsquashfs", "-li", "-f", "-d", work_dir, squashfs_file]) else: run(["unsquashfs", "-f", "-d", work_dir, squashfs_file]) # Work in the image os.chdir(work_dir) logging.info( "Entering into a bash shell to edit the image. ^d when you have finished." ) os.putenv("PROMPT_COMMAND", "echo -ne '\e[1;31m({0}) clara images> \e[0m'".format(dist)) pty.spawn(["/bin/bash"]) save = raw_input('Save changes made in the image? (N/y)') logging.debug("Input from the user: '******'".format(save)) if save not in ('Y', 'y'): clara_exit( "Changes ignored. The image {0} hasn't been modified.".format( squashfs_file)) # Rename old image and recreate new one os.rename(squashfs_file, squashfs_file + ".old") if conf.ddebug: run([ "mksquashfs", work_dir, squashfs_file, "-no-exports", "-noappend", "-info" ]) else: run([ "mksquashfs", work_dir, squashfs_file, "-no-exports", "-noappend" ]) os.chmod(squashfs_file, 0o755) logging.info("\nPrevious image renamed to {0}." "\nThe image has been repacked at {1}".format( squashfs_file + ".old", squashfs_file))
def umount_chroot(work_dir): if os.path.ismount(work_dir + "/proc/sys/fs/binfmt_misc"): run(["chroot", work_dir, "umount", "/proc/sys/fs/binfmt_misc"]) if os.path.ismount(work_dir + "/sys"): run(["chroot", work_dir, "umount", "/sys"]) if os.path.ismount(work_dir + "/proc"): run(["chroot", work_dir, "umount", "-lf", "/proc"]) time.sleep(1) # Wait one second so the system has time to unmount with open("/proc/mounts", "r") as file_to_read: for line in file_to_read: if work_dir in line: clara_exit("Something went wrong when umounting in the chroot")
def mktorrent(image): if (image is None): squashfs_file = get_from_config("images", "trg_img", dist) else: squashfs_file = image trackers_port = get_from_config("p2p", "trackers_port", dist) trackers_schema = get_from_config("p2p", "trackers_schema", dist) seeding_service = get_from_config("p2p", "seeding_service", dist) init_stop = get_from_config("p2p", "init_stop", dist) init_start = get_from_config("p2p", "init_start", dist) # trackers is a dictionary with pairs nodeset and torrent file trackers = {} for e in get_from_config("p2p", "trackers", dist).split(";"): k, v = e.split(":") trackers[k] = v # seeders in the config file is a dictionary with pairs nodeset and torrent file seeders_dict = {} for e in get_from_config("p2p", "seeders", dist).split(";"): k, v = e.split(":") seeders_dict[k] = v seeders = ",".join(seeders_dict.keys()) if not os.path.isfile(squashfs_file): clara_exit("{0} doesn't exist".format(squashfs_file)) # Remove old torrent files for f in trackers.values(): if os.path.isfile(f): os.remove(f) clush(seeders, init_stop.format(seeding_service)) sftp_mode = has_config_value("p2p", "sftp_user", dist) if sftp_mode: sftp_user = get_from_config("p2p", "sftp_user", dist) sftp_private_key = get_from_config("p2p", "sftp_private_key", dist) sftp_passphrase = get_from_config_or("p2p", "sftp_passphrase", dist, None) sftp_client = sftp.Sftp(seeders.split(','), sftp_user, sftp_private_key, sftp_passphrase) for e in trackers.keys(): announce = [] for t in list(ClusterShell.NodeSet.NodeSet(e)): announce.append("{0}://{1}:{2}/announce".format(trackers_schema, t, trackers_port)) run(["/usr/bin/mktorrent", "-a", ",".join(announce), "-o", trackers[e], squashfs_file]) if sftp_mode: sftp_client.upload([trackers[e]], os.path.dirname(trackers[e])) clush(seeders, init_start.format(seeding_service))
def do_sync(selected_dist, input_suites=[]): suites = [] # Sync everything if selected_dist == 'all': for d in get_from_config("common", "allowed_distributions").split(","): suites = suites + get_from_config("repo", "suites", d).split(",") else: # Only sync suites related to the selected distribution if len(input_suites) == 0: suites = get_from_config("repo", "suites", selected_dist).split(",") # User selected one or several suites, check that they are valid else: for s in input_suites: if s not in get_from_config("repo", "suites", selected_dist).split(","): clara_exit( "{0} is not a valid suite from distribution {1}.\n" "Valid suites are: {2}".format( s, selected_dist, get_from_config("repo", "suites", selected_dist))) suites = input_suites logging.debug("The suites to sync are: {0}.".format(" ".join(suites))) # Read /etc/clara/repos.ini if not os.path.isfile('/etc/clara/repos.ini'): clara_exit("File /etc/clara/repos.ini not found.") repos = ConfigParser.ConfigParser() repos.read("/etc/clara/repos.ini") for s in suites: extra = [] if conf.ddebug: # if extra debug for 3rd party software extra = ['--debug'] final_dir = get(repos, s, "mirror_root") + "/" + s run(['debmirror'] + extra + [ "--diff=none", "--nosource", "--ignore-release-gpg", "--ignore-missing-release", "--method={0}".format( get(repos, s, "method")), "--arch={0}".format(get(repos, s, "archs")), "--host={0}".format( get(repos, s, "server")), "--root={0}".format( get(repos, s, "mirror_dir")), "--dist={0}".format( get(repos, s, "suite_name")), "--section={0}".format( get(repos, s, "sections")), final_dir ])
def edit(chroot_dir, dist): if not os.path.isdir(chroot_dir): clara_exit("The directory {0} doesn't exist.".format(chroot_dir)) # Work in the chroot mount_chroot(chroot_dir, dist) os.chdir(chroot_dir) logging.info( "Entering into the chroot to edit. ^d when you have finished.") os.putenv("PROMPT_COMMAND", "echo -ne '\e[1;31m({0}) clara chroot> \e[0m'".format(dist)) pty.spawn(["chroot", "."]) clara_exit("Exiting the chroot {0}.".format(chroot_dir))
def run_chroot(cmd, work_dir): logging.debug("images/run_chroot: {0}".format(" ".join(cmd))) try: retcode = subprocess.call(cmd) except OSError as e: if (e.errno == errno.ENOENT): clara_exit( "Binary not found, check your path and/or retry as root. \ You were trying to run:\n {0}".format(" ".join(cmd))) if retcode != 0: umount_chroot(work_dir) if not _opts['keep_chroot_dir']: shutil.rmtree(work_dir) clara_exit(' '.join(cmd))
def mktorrent(image): if (image is None): squashfs_file = get_from_config("images", "trg_img", dist) else: squashfs_file = image trackers_port = get_from_config("p2p", "trackers_port", dist) trackers_schema = get_from_config("p2p", "trackers_schema", dist) seeding_service = get_from_config("p2p", "seeding_service", dist) init_stop = get_from_config("p2p", "init_stop", dist) init_start = get_from_config("p2p", "init_start", dist) # trackers is a dictionary with pairs nodeset and torrent file trackers = {} for e in get_from_config("p2p", "trackers", dist).split(";"): k, v = e.split(":") trackers[k] = v # seeders in the config file is a dictionary with pairs nodeset and torrent file seeders_dict = {} for e in get_from_config("p2p", "seeders", dist).split(";"): k, v = e.split(":") seeders_dict[k] = v seeders = ",".join(seeders_dict.keys()) if not os.path.isfile(squashfs_file): clara_exit("The file {0} doesn't exist".format(squashfs_file)) # Remove old torrent files for f in trackers.values(): if os.path.isfile(f): os.remove(f) clush(seeders, init_stop.format(seeding_service)) for e in trackers.keys(): announce = [] for t in list(ClusterShell.NodeSet.NodeSet(e)): announce.append("{0}://{1}:{2}/announce".format( trackers_schema, t, trackers_port)) run([ "/usr/bin/mktorrent", "-a", ",".join(announce), "-o", trackers[e], squashfs_file ]) clush(seeders, init_start.format(seeding_service))
def main(): logging.debug(sys.argv) dargs = docopt.docopt(__doc__) global keep_chroot_dir keep_chroot_dir = False # Not executed in the following cases # - the program dies because of a signal # - os._exit() is invoked directly # - a Python fatal error is detected (in the interpreter) atexit.register(clean_and_exit) global dist dist = get_from_config("common", "default_distribution") if dargs['<dist>'] is not None: dist = dargs["<dist>"] if dist not in get_from_config("common", "allowed_distributions"): clara_exit("{0} is not a know distribution".format(dist)) global work_dir if dargs['repack']: work_dir = dargs['<directory>'] else: tmpdir = get_from_config_or("images", "tmp_dir", dist, "/tmp") work_dir = tempfile.mkdtemp(prefix="tmpClara", dir=tmpdir) if dargs['create']: if dargs["--keep-chroot-dir"]: keep_chroot_dir = True base_install() install_files() system_install() remove_files() run_script_post_creation() genimg(dargs['<image>']) elif dargs['repack']: genimg(dargs['--image']) elif dargs['unpack']: extract_image(dargs['--image']) elif dargs['initrd']: geninitrd(dargs['--output']) elif dargs['edit']: edit(dargs['<image>']) elif dargs['push']: push(dargs['<image>'])
def main(): logging.debug(sys.argv) dargs = docopt.docopt(__doc__) _opts['dist'] = get_from_config("common", "default_distribution") if dargs["<dist>"] is not None: _opts['dist'] = dargs["<dist>"] if _opts['dist'] not in get_from_config("common", "allowed_distributions"): clara_exit("{0} is not a known distribution".format(_opts['dist'])) trackers_dict = {} for e in get_from_config("p2p", "trackers", _opts['dist']).split(";"): k, v = e.split(":") trackers_dict[k] = v trackers = ",".join(trackers_dict.keys()) seeders_dict = {} for e in get_from_config( "p2p", "seeders", ).split(";"): k, v = e.split(":") seeders_dict[k] = v seeders = ",".join(seeders_dict.keys()) tracking_service = get_from_config("p2p", "tracking_service", _opts['dist']) seeding_service = get_from_config("p2p", "seeding_service", _opts['dist']) if dargs['status']: init_status = get_from_config("p2p", "init_status") clush(trackers, init_status.format(tracking_service)) clush(seeders, init_status.format(seeding_service)) elif dargs['restart']: init_stop = get_from_config("p2p", "init_stop") clush(seeders, init_stop.format(seeding_service)) clush(trackers, init_stop.format(tracking_service)) time.sleep(1) init_start = get_from_config("p2p", "init_start") clush(trackers, init_start.format(tracking_service)) clush(seeders, init_start.format(seeding_service)) elif dargs['mktorrent']: mktorrent(dargs['--image'])
def copy_jenkins(job, arch, flags=None): if not job.endswith("-binaries"): job = job + "-binaries" jenkins_dir = get_from_config("repo", "jenkins_dir") path = os.path.join(jenkins_dir, job, "builds/lastSuccessfulBuild/archive/") if not os.path.isdir(path): clara_exit("The job {} doesn't exist or needs to be built.".format(job)) for f in os.listdir(path): if f.endswith("_{0}.changes".format(arch)): changesfile = os.path.join(path+f) if changesfile is None: clara_exit("Not changes file was found in {0}".format(path)) do_reprepro('include', package=changesfile, flags=flags)
def show(section='all', dist=None): if section == 'all': for sec in getconfig().sections(): print_section(getconfig().items(sec)) elif section in getconfig().sections(): if dist is None: print_section(getconfig().items(section)) elif dist in getconfig().get("common", "allowed_distributions"): s_section = "{}-{}".format(section, dist) print("Section - {}".format(section)) print_section(getconfig().items(section)) print("Section - {}".format(s_section)) if s_section in getconfig().sections(): print_section(getconfig().items(section)) else: clara_exit( "'{0}' is not section in config.ini".format(s_section)) else: clara_exit("'{0}' is not a section in config.ini".format(section))
def extract_image(image): if (image is None): squashfs_file = get_from_config("images", "trg_img", dist) else: squashfs_file = image if not os.path.isfile(squashfs_file): clara_exit("{0} does not exist!".format(squashfs_file)) extract_dir = tempfile.mkdtemp(prefix="tmpClara") logging.info("Extracting {0} to {1} ...".format(squashfs_file, extract_dir)) if conf.ddebug: run(["unsquashfs", "-li", "-f", "-d", extract_dir, squashfs_file]) else: run(["unsquashfs", "-f", "-d", extract_dir, squashfs_file]) logging.info( "Modify the image at {0} and then run:\n" "\tclara images repack {0} ( <dist> | --image=<path> )".format( extract_dir))
def getmac(hosts): imm_user = value_from_file(get_from_config("common", "master_passwd_file"), "IMMUSER") os.environ["IPMI_PASSWORD"] = value_from_file(get_from_config("common", "master_passwd_file"), "IMMPASSWORD") nodeset = ClusterShell.NodeSet.NodeSet(hosts) for host in nodeset: pat = re.compile("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}") if not pat.match(host): host = "imm" + host logging.info("{0}: ".format(host)) cmd = ["ipmitool", "-I", "lanplus", "-H", host, "-U", imm_user, "-E", "fru", "print", "0"] logging.debug("ipmi/getmac: {0}".format(" ".join(cmd))) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) # The data we want is in line 15 lines = proc.stdout.readlines() if (len(lines) < 14): clara_exit("The host {0} can't be reached".format(host)) full_mac = lines[14].split(":")[1].strip().upper() mac_address1 = "{0}:{1}:{2}:{3}:{4}:{5}".format(full_mac[0:2], full_mac[2:4], full_mac[4:6], full_mac[6:8], full_mac[8:10], full_mac[10:12]) mac_address2 = "{0}:{1}:{2}:{3}:{4}:{5}".format(full_mac[12:14], full_mac[14:16], full_mac[16:18], full_mac[18:20], full_mac[20:22], full_mac[22:24]) logging.info(" eth0's MAC address is {0}\n" " eth1's MAC address is {1}".format(mac_address1, mac_address2))
def getmac(hosts): imm_user = value_from_file(get_from_config("common", "master_passwd_file"), "IMMUSER") os.environ["IPMI_PASSWORD"] = value_from_file( get_from_config("common", "master_passwd_file"), "IMMPASSWORD") nodeset = ClusterShell.NodeSet.NodeSet(hosts) for host in nodeset: pat = re.compile("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}") if not pat.match(host): prefix = get_from_config("ipmi", "prefix") host = prefix + host logging.info("{0}: ".format(host)) cmd = [ "ipmitool", "-I", "lanplus", "-H", host, "-U", imm_user, "-E", "fru", "print", "0" ] logging.debug("ipmi/getmac: {0}".format(" ".join(cmd))) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) # The data we want is in line 15 lines = proc.stdout.readlines() if (len(lines) < 14): clara_exit("The host {0} can't be reached".format(host)) full_mac = lines[14].split(":")[1].strip().upper() mac_address1 = "{0}:{1}:{2}:{3}:{4}:{5}".format( full_mac[0:2], full_mac[2:4], full_mac[4:6], full_mac[6:8], full_mac[8:10], full_mac[10:12]) mac_address2 = "{0}:{1}:{2}:{3}:{4}:{5}".format( full_mac[12:14], full_mac[14:16], full_mac[16:18], full_mac[18:20], full_mac[20:22], full_mac[22:24]) logging.info(" eth0's MAC address is {0}\n" " eth1's MAC address is {1}".format( mac_address1, mac_address2))
def do_reprepro(action, package=None, flags=None, extra=None): repo_dir = get_from_config("repo", "repo_dir", dist) reprepro_config = repo_dir + '/conf/distributions' mirror_local = get_from_config("repo", "mirror_local", dist) if (mirror_local == "" or mirror_local == None): mirror_local = repo_dir + '/mirror' oldMask = os.umask(0022) if not os.path.isfile(reprepro_config): clara_exit( "There is not configuration for the local repository for {0}. Run first 'clara repo init <dist>'" .format(dist)) list_flags = ['--silent', '--ask-passphrase'] if conf.ddebug: list_flags = ['-V', '--ask-passphrase'] if flags is not None: list_flags.append(flags) cmd = ['reprepro'] + list_flags + \ ['--basedir', get_from_config("repo", "repo_dir", dist), '--outdir', mirror_local, action] if extra is not None: for e in extra: cmd.append(e) else: if action in [ 'includedeb', 'include', 'includedsc', 'remove', 'removesrc', 'list' ]: cmd.append(dist) if package is not None: cmd.append(package) run(cmd) os.umask(oldMask)
def show(section=None, dist=None): if section == "all": for sec in getconfig().sections(): for item in list(getconfig().items(sec)): print item[0], ":", item[1] elif section in getconfig().sections(): if (dist == '') or (dist == None): for item in list(getconfig().items(section)): print item[0], ":", item[1] elif dist in getconfig().get("common", "allowed_distributions"): or_section = section + "-" + dist print "\nSection - ", section, "\n" for item in list(getconfig().items(section)): print item[0], ":", item[1] print "\nSection - ", or_section, "\n" if or_section in getconfig().sections(): for item in list(getconfig().items(or_section)): print item[0], ":", item[1] else: clara_exit( "'{0}' is not section in config.ini".format(or_section)) else: clara_exit("'{0}' is not a section in config.ini".format(section))
def do_reprepro(action, package=None, flags=None): repo_dir = get_from_config("repo", "repo_dir", dist) reprepro_config = repo_dir + '/conf/distributions' if not os.path.isfile(reprepro_config): clara_exit("There is not configuration for the local repository for {0}. Run first 'clara repo init <dist>'".format(dist)) list_flags = ['--silent', '--ask-passphrase'] if conf.ddebug: list_flags = ['-V', '--ask-passphrase'] if flags is not None: list_flags.append(flags) cmd = ['reprepro'] + list_flags + \ ['--basedir', get_from_config("repo", "repo_dir", dist), '--outdir', get_from_config("repo", "mirror_local", dist), action, dist] if package is not None: cmd.append(package) run(cmd)
def main(): logging.debug(sys.argv) dargs = docopt.docopt(__doc__) global dist dist = get_from_config("common", "default_distribution") if dargs["<dist>"] is not None: dist = dargs["<dist>"] if dist not in get_from_config("common", "allowed_distributions"): clara_exit("{0} is not a know distribution".format(dist)) if dargs['key']: do_key() if dargs['init']: do_init() elif dargs['sync']: if dargs['all']: do_sync('all') else: do_sync(dargs['<dist>'], dargs['<suites>']) elif dargs['add']: for elem in dargs['<file>']: if elem.endswith(".deb"): do_reprepro('includedeb', elem, dargs['--reprepro-flags']) elif elem.endswith(".changes"): do_reprepro('include', elem, dargs['--reprepro-flags']) elif elem.endswith(".dsc"): do_reprepro('includedsc', elem, dargs['--reprepro-flags']) else: clara_exit("File is not a *.deb *.dsc or *.changes") elif dargs['del']: for elem in dargs['<name>']: do_reprepro('remove', elem) do_reprepro('removesrc', elem) elif dargs['list']: do_reprepro('list')
def do_key(): key = get_from_config("repo", "gpg_key") fnull = open(os.devnull, 'w') cmd = ['gpg', '--list-secret-keys', key] logging.debug("repo/do_key: {0}".format(" ".join(cmd))) retcode = subprocess.call(cmd, stdout=fnull, stderr=fnull) fnull.close() # We import the key if it hasn't been imported before if retcode != 0: file_stored_key = get_from_config("repo", "stored_enc_key") if os.path.isfile(file_stored_key): password = value_from_file(get_from_config("common", "master_passwd_file"), "ASUPASSWD") if len(password) > 20: fdesc, temp_path = tempfile.mkstemp(prefix="tmpClara") cmd = ['openssl', 'aes-256-cbc', '-d', '-in', file_stored_key, '-out', temp_path, '-k', password] logging.debug("repo/do_key: {0}".format(" ".join(cmd))) retcode = subprocess.call(cmd) if retcode != 0: os.close(fdesc) os.remove(temp_path) clara_exit('Command failed {0}'.format(" ".join(cmd))) else: logging.info("Trying to import key {0}".format(key)) fnull = open(os.devnull, 'w') cmd = ['gpg', '--allow-secret-key-import', '--import', temp_path] logging.debug("repo/do_key: {0}".format(" ".join(cmd))) retcode = subprocess.call(cmd) fnull.close() os.close(fdesc) os.remove(temp_path) if retcode != 0: logging.info("\nThere was a problem with the import, make sure the key you imported " "from {0} is the same you have in your configuration: {1}".format(file_stored_key, key)) else: clara_exit('There was some problem reading the value of ASUPASSWD') else: clara_exit('Unable to read: {0}'.format(file_stored_key)) else: logging.info("GPG key was already imported.")
def main(): logging.debug(sys.argv) dargs = docopt.docopt(__doc__) debug = [] if conf.ddebug: debug = ["--verbose", "--details"] if dargs['resume']: run(["scontrol"] + debug + ["update", "NodeName=" + dargs['<nodeset>'], "State=RESUME"]) elif dargs['drain']: if dargs['<nodeset>'] is None: show_nodes("drain") else: if len(dargs['<reason>']) == 0: clara_exit("You must specify a reason when DRAINING a node") else: run(["scontrol"] + debug + ["update", "NodeName=" + dargs['<nodeset>'], "State=DRAIN", 'Reason="' + " ".join(dargs['<reason>']) + '"']) elif dargs['down']: if dargs['<nodeset>'] is None: show_nodes("down") else: run(["scontrol"] + debug + ["update", "NodeName=" + dargs['<nodeset>'], "State=DOWN"]) elif dargs['health']: script_slurm_health = get_from_config("slurm", "script_slurm_health") if (len(script_slurm_health) > 0): clush(dargs['<nodeset>'], script_slurm_health) else: clara_exit("You must set a health check program in the configuration file.") else: cmd_list = ['job', 'node', 'steps', 'frontend', 'partition', 'reservation', 'block', 'submp'] # /!\ ∀ x, ∀ y, op_list[x][y] ⇒ op_list[x][y] ∈ cmd_list op_list = { 'show': ['job', 'node', 'partition', 'reservation', 'steps', 'frontend', 'block', 'submp'], 'update': ['job', 'node', 'partition', 'reservation', 'steps', 'frontend', 'block', 'submp'], 'create': ['partition', 'reservation'], 'delete': ['partition', 'reservation'] } # /!\ ∀ x ∈ cmd_list ⇒ x ∈ keys_list.keys() key_list = { 'job': 'JobId', 'steps': 'StepId', 'node': 'NodeName', 'frontend': 'FrontendName', 'partition': 'PartitionName', 'reservation': 'Reservation', 'block': 'BlockName', 'submp': 'SubMPName' } cmd = dargs['<cmd>'] subject = dargs['<subject>'] op = dargs['<op>'] spec = dargs['<spec>'] if cmd not in cmd_list: clara_exit("Known commands are: {0}".format(" ".join(cmd_list))) if spec is None: if op is not None and "=" in op: spec = [op] op = 'show' if "=" in op: spec = [op] + spec op = 'show' if op not in op_list: clara_exit("Known operations are: {0}".format(" ".join(op_list))) if cmd not in op_list[op]: clara_exit("You can't use {0} with {1}".format(cmd, op)) if op == 'show': # spec should be empty run(["scontrol"] + debug + [op, cmd, subject]) else: run(["scontrol"] + debug + \ [op, "{0}={1}".format(key_list[cmd], subject), " ".join(spec)])
def main(): logging.debug(sys.argv) dargs = docopt.docopt(__doc__, options_first=True) target_dist = dargs["<dist>"] if target_dist not in target_dists.keys(): sys.exit("Unknown target dist {0}".format(target_dist)) # Create a temp directory work_dir = tempfile.mkdtemp(prefix="tmpRebuild") if dargs["source"]: dsc_file = dargs["<dsc_file>"] # Check dsc file exists if not os.path.isfile(dsc_file): sys.exit("The file {0} doesn't exist".format(dsc_file)) # Find out versioning name, full_version = os.path.basename(dsc_file)[:-4].split("_") if "-" in full_version: upstream_version, debian_version = full_version.rsplit("-", 1) else: # Binary package upstream_version, debian_version = full_version, "" print_info(name, full_version, upstream_version, debian_version) dirname = os.path.dirname(dsc_file) copy_files_to_workdir(os.path.join(dirname, name + "_" + upstream_version + "*"), work_dir) elif dargs["repo"]: origin_dist = dargs["<origin_dist>"] package_name = dargs["<package_name>"] if origin_dist not in target_dists.keys(): sys.exit("Unknown origin dist {0}".format(origin_dist)) if not isinstance(package_name, str): clara_exit("Package name {0} is not a string".format(package_name)) # Check if the source package exists in the reprepro of origin_dist cmd = ["clara", "repo", "list", origin_dist] logging.debug(" ".join(cmd)) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) lines_proc = proc.stdout.readlines() found_package = False for line in lines_proc: if "|source: {} ".format(package_name) in line: found_package = True break # If we fail to find the package, we list what's available and exit if not found_package: logging.info("Package {0} not found. The available packages in {1} are:".format(package_name, origin_dist)) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) lines_proc = proc.stdout.readlines() for l in lines_proc: if "|source: " in l: logging.info(l.split(":")[1]) clara_exit("Select an available package and re-run clara.") # Look for the files and copy them to the temp directory area = "main" if "non-free" in line: area = "non-free" if package_name.startswith("lib"): package_dir = package_name[0:4] + "/" + package_name else: package_dir = package_name[0] + "/" + package_name # Note this is the path from the *origin* dist repo_path_pool = get_from_config("build", "repo_path_pool", origin_dist) repo_path_pool = repo_path_pool + "{0}/{1}".format(area, package_dir) full_version = line.split(" ")[-1].strip() upstream_version, debian_version = full_version.split("-") name = package_name print_info(name, full_version, upstream_version, debian_version) copy_files_to_workdir(os.path.join(repo_path_pool, name + "_" + upstream_version + "*"), work_dir) # Common code: append the calibre tag and build the package os.chdir(work_dir) cmd = ["dpkg-source", "-x", "{0}_{1}.dsc".format(name, full_version)] logging.debug(" ".join(cmd)) p = subprocess.call(cmd) path = "{0}-{1}".format(name, upstream_version) os.chdir(path) # Check if the debian version already appends the calibre tag tag = target_dists[target_dist] debian_version_re = r"(.*)\+{0}([\.\+])(\d)".format(tag) debian_version_p = re.compile(debian_version_re) debian_version_m = debian_version_p.match(debian_version) if debian_version_m: calibre_dump_s = debian_version_m.group(3) try: calibre_dump = int(calibre_dump_s) + 1 except ValueError: # calibre_dump_s was probably not an integer so start with 1 logging.info("Unable to handle calibre dump '{0}', restarting with 1".format(calibre_dump_s)) calibre_dump = 1 debian_version = debian_version_m.group(1) sep = debian_version_m.group(2) new_full_version = upstream_version + "-" + debian_version + "+" + tag + sep + str(calibre_dump) else: logging.info("Tag {0} not found in Debian version, appending new one".format(tag)) new_full_version = full_version + "+" + tag + "+1" # Bump the version, adding "+target_dist+1" cmd = ["dch", "--force-distribution", "-D", target_dist, "-v", new_full_version, "Rebuild for " + target_dist + "."] logging.debug(" ".join(cmd)) p = subprocess.call(cmd) # Recreate the source package os.chdir(work_dir) if "-" in new_full_version: cmd = ["dpkg-source", "-b", path] else: # Native cmd = ["dpkg-source", "-b", "{0}-{1}".format(name, new_full_version)] logging.debug(" ".join(cmd)) p = subprocess.call(cmd) # And build with cowbuilder cowbuilder_bin = get_from_config("build", "cowbuilder_bin", target_dist) if not os.path.isfile(cowbuilder_bin): sys.exit("The file {0} doesn't exist".format(cowbuilder_bin)) cmd = [cowbuilder_bin, "--build", "{0}_{1}.dsc".format(name, new_full_version)] logging.debug(" ".join(cmd)) p = subprocess.call(cmd) # Remove work directory shutil.rmtree(work_dir) repo_bin_dir = get_from_config("build", "repo_bin_dir", target_dist) repo_src_dir = get_from_config("build", "repo_src_dir", target_dist) dsc_file = "{0}_{1}.dsc".format(name, new_full_version) dsc_path = os.path.join(repo_bin_dir, dsc_file) changes_file = "{0}_{1}_amd64.changes".format(name, new_full_version) changes_path = os.path.join(repo_bin_dir, changes_file) orig_file = "{0}_{1}.orig.tar.gz".format(name, upstream_version) debian_file = "{0}_{1}.debian.tar.gz".format(name, new_full_version) logging.info( """ Commands you are supposed to run then: -------------8<----------------------- unset DISPLAY eval $(gpg-agent --daemon) clara repo del {0} {1} clara repo add {0} {2} clara repo add {0} {3} cp -v {4}/{{{5},{6},{7}}} {8}/ #-------------8<----------------------- """.format( target_dist, name, dsc_path, changes_path, repo_bin_dir, dsc_file, orig_file, debian_file, repo_src_dir ) )