def createrepo_walker(self, repo, dirname, fnames): """ Used to run createrepo on a copied Yum mirror. :param repo: The repository object to run for. :param dirname: The directory to run in. :param fnames: Not known what this is for. """ if os.path.exists(dirname) or repo['breed'] == 'rsync': utils.remove_yum_olddata(dirname) # add any repo metadata we can use mdoptions = [] if os.path.isfile("%s/.origin/repodata/repomd.xml" % (dirname)): if HAS_LIBREPO: rd = self.librepo_getinfo("%s/.origin" % (dirname)) elif HAS_YUM: rmd = yum.repoMDObject.RepoMD( '', "%s/.origin/repodata/repomd.xml" % (dirname)) rd = rmd.repoData else: utils.die(self.logger, "yum/librepo is required to use this feature") if "group" in rd: if HAS_LIBREPO: groupmdfile = rd['group']['location_href'] else: groupmdfile = rmd.getData("group").location[1] mdoptions.append("-g %s" % groupmdfile) if "prestodelta" in rd: # need createrepo >= 0.9.7 to add deltas if utils.get_family() in ("redhat", "suse"): cmd = "/usr/bin/rpmquery --queryformat=%{VERSION} createrepo" createrepo_ver = utils.subprocess_get(self.logger, cmd) if not createrepo_ver[0:1].isdigit(): cmd = "/usr/bin/rpmquery --queryformat=%{VERSION} createrepo_c" createrepo_ver = utils.subprocess_get( self.logger, cmd) if utils.compare_versions_gt(createrepo_ver, "0.9.7"): mdoptions.append("--deltas") else: self.logger.error( "this repo has presto metadata; you must upgrade createrepo to >= 0.9.7 first and then need to resync the repo through Cobbler." ) blended = utils.blender(self.api, False, repo) flags = blended.get("createrepo_flags", "(ERROR: FLAGS)") try: cmd = "createrepo %s %s %s" % (" ".join(mdoptions), flags, pipes.quote(dirname)) utils.subprocess_call(self.logger, cmd) except: utils.log_exc(self.logger) self.logger.error("createrepo failed.") del fnames[:] # we're in the right place
def createrepo_walker(self, repo, dirname: str, fnames): """ Used to run createrepo on a copied Yum mirror. :param repo: The repository object to run for. :param dirname: The directory to run in. :param fnames: Not known what this is for. """ if os.path.exists(dirname) or repo.breed == RepoBreeds.RSYNC: utils.remove_yum_olddata(dirname) # add any repo metadata we can use mdoptions = [] origin_path = os.path.join(dirname, ".origin") repodata_path = os.path.join(origin_path, "repodata") if os.path.isfile(os.path.join(repodata_path, "repomd.xml")): rd = self.librepo_getinfo(origin_path) if "group" in rd: groupmdfile = rd['group']['location_href'] mdoptions.append("-g %s" % os.path.join(origin_path, groupmdfile)) if "prestodelta" in rd: # need createrepo >= 0.9.7 to add deltas if utils.get_family() in ("redhat", "suse"): cmd = "/usr/bin/rpmquery --queryformat=%{VERSION} createrepo" createrepo_ver = utils.subprocess_get(cmd) if not createrepo_ver[0:1].isdigit(): cmd = "/usr/bin/rpmquery --queryformat=%{VERSION} createrepo_c" createrepo_ver = utils.subprocess_get(cmd) if utils.compare_versions_gt(createrepo_ver, "0.9.7"): mdoptions.append("--deltas") else: self.logger.error( "this repo has presto metadata; you must upgrade createrepo to >= 0.9.7 " "first and then need to resync the repo through Cobbler." ) blended = utils.blender(self.api, False, repo) flags = blended.get("createrepo_flags", "(ERROR: FLAGS)") try: cmd = "createrepo %s %s %s" % (" ".join(mdoptions), flags, pipes.quote(dirname)) utils.subprocess_call(cmd) except: utils.log_exc() self.logger.error("createrepo failed.") del fnames[:] # we're in the right place
def check_dnsmasq_bin(self, status): """ Check if dnsmasq is installed """ rc = utils.subprocess_get(self.logger, "dnsmasq --help") if rc.find("Valid options") == -1: status.append("dnsmasq is not installed and/or in path")
def get_file_lines(self, filename: str): """ Get lines from a file, which may or may not be compressed. If compressed then it will be uncompressed using ``gzip`` as the algorithm. :param filename: The name of the file to be read. :return: An array with all the lines. """ ftype = magic.detect_from_filename(filename) if ftype.mime_type == "application/gzip": try: with gzip.open(filename, 'r') as f: return f.readlines() except: pass if ftype.mime_type == "application/x-ms-wim": cmd = "/usr/bin/wiminfo" if os.path.exists(cmd): cmd = "%s %s" % (cmd, filename) return utils.subprocess_get(cmd).splitlines() self.logger.info("no %s found, please install wimlib-utils", cmd) elif ftype.mime_type == "text/plain": with open(filename, 'r') as f: return f.readlines() else: self.logger.info( 'Could not detect the filetype and read the content of file "%s". Returning nothing.', filename) return []
def check_bind_bin(self, status): """ Check if bind is installed. """ rc = utils.subprocess_get(self.logger, "named -v") # it should return something like "BIND 9.6.1-P1-RedHat-9.6.1-6.P1.fc11" if rc.find("BIND") == -1: status.append("named is not installed and/or in path")
def check_dnsmasq_bin(self, status): """ Check if dnsmasq is installed. :param status: The status list with possible problems. """ rc = utils.subprocess_get(self.logger, "dnsmasq --help") if rc.find("Valid options") == -1: status.append("dnsmasq is not installed and/or in path")
def test_subprocess_get(): # Arrange # Act result = utils.subprocess_get("echo Test") # Assert # The newline makes sense in my (@SchoolGuy) eyes since we want to have multiline output also in a single string. assert result == "Test\n"
def get_file_lines(self, filename): """ Get lines from a file, which may or may not be compressed """ lines = [] ftype = utils.subprocess_get(self.logger, "/usr/bin/file %s" % filename) if ftype.find("gzip") != -1: try: import gzip f = gzip.open(filename, 'r') lines = f.readlines() f.close() except: pass elif ftype.find("text") != -1: f = open(filename, 'r') lines = f.readlines() f.close() return lines
def get_file_lines(self, filename): """ Get lines from a file, which may or may not be compressed """ lines = [] ftype = utils.subprocess_get(self.logger, "/usr/bin/file %s" % filename) if ftype.find("gzip") != -1: try: import gzip f = gzip.open(filename, 'r') lines = f.readlines() f.close() except: pass elif ftype.find("text") != -1: f = open(filename, 'r') lines = f.readlines() f.close() return lines
def get_file_lines(self, filename): """ Get lines from a file, which may or may not be compressed. If compressed then it will be uncompressed using ``gzip`` as the algorithm. :param filename: The name of the file to be read. :return: An array with all the lines. """ lines = [] ftype = utils.subprocess_get(self.logger, "/usr/bin/file %s" % filename) if ftype.find("gzip") != -1: try: import gzip f = gzip.open(filename, 'r') lines = f.readlines() f.close() except: pass elif ftype.find("text") != -1: f = open(filename, 'r') lines = f.readlines() f.close() return lines
def check_push(fn, tftpbootdir, settings, lcache='/var/lib/cobbler', logger=None): """ Returns the sha1sum of the file """ db = {} try: dbfile = os.path.join(lcache, 'pxe_cache.json') if os.path.exists(dbfile): db = simplejson.load(open(dbfile, 'r')) except: pass count = 0 while not os.path.exists(fn) and count < 10: count += 1 if _DEBUG: logger.debug("%s does not exist yet - retrying (try %s)" % (fn, count)) time.sleep(1) mtime = os.stat(fn).st_mtime key = None needpush = True if _DEBUG: logger.debug("check_push(%s)" % fn) if db.has_key(fn): if db[fn][0] < mtime: if _DEBUG: logger.debug("mtime differ - old: %s new: %s" % (db[fn][0], mtime)) if os.path.exists(fn): cmd = '/usr/bin/sha1sum %s' % fn key = utils.subprocess_get(None, cmd).split(' ')[0] if _DEBUG: logger.debug("checking checksum - old: %s new: %s" % (db[fn][1], key)) if key == db[fn][1]: needpush = False else: needpush = False if key is None: if os.path.exists(fn): cmd = '/usr/bin/sha1sum %s' % fn key = utils.subprocess_get(None, cmd).split(' ')[0] if _DEBUG: logger.debug("push(%s) ? %s" % (fn, needpush)) if needpush: # reset the Result var ProxySync.ResultLock.acquire() ProxySync.Result = True ProxySync.ResultLock.release() format = 'other' if "pxelinux.cfg" in fn: format = 'pxe' elif "grub" in fn: format = 'grub' if sync_to_proxies(fn, tftpbootdir, format, settings, logger): db[fn] = (mtime, key) simplejson.dump(db, open(dbfile, 'w')) logger.info("Push successfull") else: logger.info("Push failed")
def gen_win_files(distro, meta): (kernel_path, kernel_name) = os.path.split(distro.kernel) distro_path = utils.find_distro_path(settings, distro) distro_dir = wim_file_name = os.path.join(settings.tftpboot_location, "images", distro.name) web_dir = os.path.join(settings.webdir, "images", distro.name) is_winpe = "winpe" in meta and meta['winpe'] != "" is_bcd = "bcd" in meta and meta['bcd'] != "" if "kernel" in meta: kernel_name = meta["kernel"] kernel_name = os.path.basename(kernel_name) is_wimboot = "wimboot" in kernel_name if is_wimboot: distro_path = os.path.join(settings.webdir, "distro_mirror", distro.name) kernel_path = os.path.join(distro_path, "boot") if "kernel" in meta and "wimboot" not in distro.kernel: tgen.copy_single_distro_file( os.path.join(settings.tftpboot_location, kernel_name), distro_dir, False) tgen.copy_single_distro_file( os.path.join(distro_dir, kernel_name), web_dir, True) if "post_install_script" in meta: post_install_dir = distro_path if distro.os_version not in ("XP", "2003"): post_install_dir = os.path.join(post_install_dir, "sources") post_install_dir = os.path.join(post_install_dir, "$OEM$", "$1") if not os.path.exists(post_install_dir): utils.mkdir(post_install_dir) data = templ.render(post_tmpl_data, meta, None) post_install_script = os.path.join(post_install_dir, meta["post_install_script"]) logger.info('Build post install script: ' + post_install_script) with open(post_install_script, "w+") as pi_file: pi_file.write(data) if "answerfile" in meta: data = templ.render(tmpl_data, meta, None) answerfile_name = os.path.join(distro_dir, meta["answerfile"]) logger.info('Build answer file: ' + answerfile_name) with open(answerfile_name, "w+") as answerfile: answerfile.write(data) tgen.copy_single_distro_file(answerfile_name, distro_path, False) tgen.copy_single_distro_file(answerfile_name, web_dir, True) if "kernel" in meta and "bootmgr" in meta: wk_file_name = os.path.join(distro_dir, kernel_name) wl_file_name = os.path.join(distro_dir, meta["bootmgr"]) tl_file_name = os.path.join(kernel_path, "bootmgr.exe") if distro.os_version in ("XP", "2003") and not is_winpe: tl_file_name = os.path.join(kernel_path, "setupldr.exe") if len(meta["bootmgr"]) != 5: logger.error( "The loader name should be EXACTLY 5 character") return 1 pat1 = re.compile(br'NTLDR', re.IGNORECASE) pat2 = re.compile(br'winnt\.sif', re.IGNORECASE) with open(tl_file_name, 'rb') as file: out = data = file.read() if "answerfile" in meta: if len(meta["answerfile"]) != 9: logger.error( "The response file name should be EXACTLY 9 character" ) return 1 out = pat2.sub(bytes(meta["answerfile"], 'utf-8'), data) else: if len(meta["bootmgr"]) != 11: logger.error( "The Boot manager file name should be EXACTLY 11 character" ) return 1 bcd_name = "bcd" if is_bcd: bcd_name = meta["bcd"] if len(bcd_name) != 3: logger.error( "The BCD file name should be EXACTLY 3 character") return 1 if not os.path.isfile(tl_file_name): logger.error("File not found: %s" % tl_file_name) return 1 pat1 = re.compile(br'bootmgr\.exe', re.IGNORECASE) pat2 = re.compile(br'(\\.B.o.o.t.\\.)(B)(.)(C)(.)(D)', re.IGNORECASE) bcd_name = bytes( "\\g<1>" + bcd_name[0] + "\\g<3>" + bcd_name[1] + "\\g<5>" + bcd_name[2], 'utf-8') with open(tl_file_name, 'rb') as file: out = file.read() if not is_wimboot: logger.info('Patching build Loader: %s' % wl_file_name) out = pat2.sub(bcd_name, out) if tl_file_name != wl_file_name: logger.info('Build Loader: %s from %s' % (wl_file_name, tl_file_name)) with open(wl_file_name, 'wb+') as file: file.write(out) tgen.copy_single_distro_file(wl_file_name, web_dir, True) if not is_wimboot: if distro.os_version not in ("XP", "2003") or is_winpe: pe = pefile.PE(wl_file_name, fast_load=True) pe.OPTIONAL_HEADER.CheckSum = pe.generate_checksum() pe.write(filename=wl_file_name) with open(distro.kernel, 'rb') as file: data = file.read() out = pat1.sub(bytes(meta["bootmgr"], 'utf-8'), data) if wk_file_name != distro.kernel: logger.info("Build PXEBoot: %s from %s" % (wk_file_name, distro.kernel)) with open(wk_file_name, 'wb+') as file: file.write(out) tgen.copy_single_distro_file(wk_file_name, web_dir, True) if is_bcd: obcd_file_name = os.path.join(kernel_path, "bcd") bcd_file_name = os.path.join(distro_dir, meta["bcd"]) wim_file_name = 'winpe.wim' if not os.path.isfile(obcd_file_name): logger.error("File not found: %s" % obcd_file_name) return 1 if is_winpe: wim_file_name = meta["winpe"] if is_wimboot: wim_file_name = '\\Boot\\' + wim_file_name sdi_file_name = '\\Boot\\' + 'boot.sdi' else: wim_file_name = os.path.join("/images", distro.name, wim_file_name) sdi_file_name = os.path.join("/images", distro.name, os.path.basename(distro.initrd)) logger.info('Build BCD: %s from %s for %s' % (bcd_file_name, obcd_file_name, wim_file_name)) bcdedit(obcd_file_name, bcd_file_name, wim_file_name, sdi_file_name) tgen.copy_single_distro_file(bcd_file_name, web_dir, True) if is_winpe: ps_file_name = os.path.join(distro_dir, meta["winpe"]) wim_pl_name = os.path.join(kernel_path, "winpe.wim") cmd = ["/usr/bin/cp", "--reflink=auto", wim_pl_name, ps_file_name] utils.subprocess_call(cmd, shell=False) tgen.copy_single_distro_file(ps_file_name, web_dir, True) if os.path.exists(wimupdate): data = templ.render(tmplstart_data, meta, None) pi_file = tempfile.NamedTemporaryFile() pi_file.write(bytes(data, 'utf-8')) pi_file.flush() cmd = [ "/usr/bin/wimdir %s 1 | /usr/bin/grep -i '^/Windows/System32/startnet.cmd$'" % ps_file_name ] startnet_path = utils.subprocess_get(cmd, shell=True)[0:-1] cmd = [ wimupdate, ps_file_name, "--command=add %s %s" % (pi_file.name, startnet_path) ] utils.subprocess_call(cmd, shell=False) pi_file.close()
def run(self, path: str, name: str, network_root=None, autoinstall_file=None, arch: Optional[str] = None, breed=None, os_version=None): """ This is the main entry point in a manager. It is a required function for import modules. :param path: the directory we are scanning for files :param name: the base name of the distro :param network_root: the remote path (nfs/http/ftp) for the distro files :param autoinstall_file: user-specified response file, which will override the default :param arch: user-specified architecture :param breed: user-specified breed :param os_version: user-specified OS version :raises CX """ self.name = name self.network_root = network_root self.autoinstall_file = autoinstall_file self.arch = arch self.breed = breed self.os_version = os_version self.path = path self.rootdir = path self.pkgdir = path # some fixups for the XMLRPC interface, which does not use "None" if self.arch == "": self.arch = None if self.name == "": self.name = None if self.autoinstall_file == "": self.autoinstall_file = None if self.os_version == "": self.os_version = None if self.network_root == "": self.network_root = None if self.os_version and not self.breed: utils.die( "OS version can only be specified when a specific breed is selected" ) self.signature = self.scan_signatures() if not self.signature: error_msg = "No signature matched in %s" % path self.logger.error(error_msg) raise CX(error_msg) # now walk the filesystem looking for distributions that match certain patterns self.logger.info("Adding distros from path %s:" % self.path) distros_added = [] import_walker(self.path, self.distro_adder, distros_added) if len(distros_added) == 0: if self.breed == "windows": cmd_path = "/usr/bin/wimexport" bootwim_path = os.path.join(self.path, "sources", "boot.wim") dest_path = os.path.join(self.path, "boot") if os.path.exists(cmd_path) and os.path.exists(bootwim_path): winpe_path = os.path.join(dest_path, "winpe.wim") if not os.path.exists(dest_path): utils.mkdir(dest_path) rc = utils.subprocess_call( [cmd_path, bootwim_path, "1", winpe_path, "--boot"], shell=False) if rc == 0: cmd = [ "/usr/bin/wimdir %s 1 | /usr/bin/grep -i '^/Windows/Boot/PXE$'" % winpe_path ] pxe_path = utils.subprocess_get(cmd, shell=True)[0:-1] cmd = [ "/usr/bin/wimdir %s 1 | /usr/bin/grep -i '^/Windows/System32/config/SOFTWARE$'" % winpe_path ] config_path = utils.subprocess_get(cmd, shell=True)[0:-1] cmd_path = "/usr/bin/wimextract" rc = utils.subprocess_call([ cmd_path, bootwim_path, "1", "%s/pxeboot.n12" % pxe_path, "%s/bootmgr.exe" % pxe_path, config_path, "--dest-dir=%s" % dest_path, "--no-acls", "--no-attributes" ], shell=False) if rc == 0: if HAS_HIVEX: software = os.path.join( dest_path, os.path.basename(config_path)) h = hivex.Hivex(software, write=True) root = h.root() node = h.node_get_child(root, "Microsoft") node = h.node_get_child(node, "Windows NT") node = h.node_get_child(node, "CurrentVersion") h.node_set_value( node, { "key": "SystemRoot", "t": REG_SZ, "value": "x:\\Windows\0".encode( encoding="utf-16le") }) node = h.node_get_child(node, "WinPE") # remove the key InstRoot from the registry values = h.node_values(node) new_values = [] for value in values: keyname = h.value_key(value) if keyname == "InstRoot": continue val = h.node_get_value(node, keyname) valtype = h.value_type(val)[0] value2 = h.value_value(val)[1] valobject = { "key": keyname, "t": int(valtype), "value": value2 } new_values.append(valobject) h.node_set_values(node, new_values) h.commit(software) cmd_path = "/usr/bin/wimupdate" rc = utils.subprocess_call([ cmd_path, winpe_path, "--command=add %s %s" % (software, config_path) ], shell=False) os.remove(software) else: self.logger.info( "python3-hivex not found. If you need Automatic Windows " "Installation support, please install.") import_walker(self.path, self.distro_adder, distros_added) if len(distros_added) == 0: self.logger.warning("No distros imported, bailing out") return # find out if we can auto-create any repository records from the install tree if self.network_root is None: self.logger.info("associating repos") # FIXME: this automagic is not possible (yet) without mirroring self.repo_finder(distros_added)