def createrepo_walker(self, repo, dirname, fnames):
"""
Used to run createrepo on a copied Yum mirror.
:param repo: The repository object to run for.
:param dirname: The directory to run in.
:param fnames: Not known what this is for.
"""
if os.path.exists(dirname) or repo['breed'] == 'rsync':
utils.remove_yum_olddata(dirname)
# add any repo metadata we can use
mdoptions = []
if os.path.isfile("%s/.origin/repodata/repomd.xml" % (dirname)):
if HAS_LIBREPO:
rd = self.librepo_getinfo("%s/.origin" % (dirname))
elif HAS_YUM:
rmd = yum.repoMDObject.RepoMD(
'', "%s/.origin/repodata/repomd.xml" % (dirname))
rd = rmd.repoData
else:
utils.die(self.logger,
"yum/librepo is required to use this feature")
if "group" in rd:
if HAS_LIBREPO:
groupmdfile = rd['group']['location_href']
else:
groupmdfile = rmd.getData("group").location[1]
mdoptions.append("-g %s" % groupmdfile)
if "prestodelta" in rd:
# need createrepo >= 0.9.7 to add deltas
if utils.get_family() in ("redhat", "suse"):
cmd = "/usr/bin/rpmquery --queryformat=%{VERSION} createrepo"
createrepo_ver = utils.subprocess_get(self.logger, cmd)
if not createrepo_ver[0:1].isdigit():
cmd = "/usr/bin/rpmquery --queryformat=%{VERSION} createrepo_c"
createrepo_ver = utils.subprocess_get(
self.logger, cmd)
if utils.compare_versions_gt(createrepo_ver, "0.9.7"):
mdoptions.append("--deltas")
else:
self.logger.error(
"this repo has presto metadata; you must upgrade createrepo to >= 0.9.7 first and then need to resync the repo through Cobbler."
)
blended = utils.blender(self.api, False, repo)
flags = blended.get("createrepo_flags", "(ERROR: FLAGS)")
try:
cmd = "createrepo %s %s %s" % (" ".join(mdoptions), flags,
pipes.quote(dirname))
utils.subprocess_call(self.logger, cmd)
except:
utils.log_exc(self.logger)
self.logger.error("createrepo failed.")
del fnames[:] # we're in the right place
def createrepo_walker(self, repo, dirname: str, fnames):
"""
Used to run createrepo on a copied Yum mirror.
:param repo: The repository object to run for.
:param dirname: The directory to run in.
:param fnames: Not known what this is for.
"""
if os.path.exists(dirname) or repo.breed == RepoBreeds.RSYNC:
utils.remove_yum_olddata(dirname)
# add any repo metadata we can use
mdoptions = []
origin_path = os.path.join(dirname, ".origin")
repodata_path = os.path.join(origin_path, "repodata")
if os.path.isfile(os.path.join(repodata_path, "repomd.xml")):
rd = self.librepo_getinfo(origin_path)
if "group" in rd:
groupmdfile = rd['group']['location_href']
mdoptions.append("-g %s" %
os.path.join(origin_path, groupmdfile))
if "prestodelta" in rd:
# need createrepo >= 0.9.7 to add deltas
if utils.get_family() in ("redhat", "suse"):
cmd = "/usr/bin/rpmquery --queryformat=%{VERSION} createrepo"
createrepo_ver = utils.subprocess_get(cmd)
if not createrepo_ver[0:1].isdigit():
cmd = "/usr/bin/rpmquery --queryformat=%{VERSION} createrepo_c"
createrepo_ver = utils.subprocess_get(cmd)
if utils.compare_versions_gt(createrepo_ver, "0.9.7"):
mdoptions.append("--deltas")
else:
self.logger.error(
"this repo has presto metadata; you must upgrade createrepo to >= 0.9.7 "
"first and then need to resync the repo through Cobbler."
)
blended = utils.blender(self.api, False, repo)
flags = blended.get("createrepo_flags", "(ERROR: FLAGS)")
try:
cmd = "createrepo %s %s %s" % (" ".join(mdoptions), flags,
pipes.quote(dirname))
utils.subprocess_call(cmd)
except:
utils.log_exc()
self.logger.error("createrepo failed.")
del fnames[:] # we're in the right place
def check_dnsmasq_bin(self, status):
"""
Check if dnsmasq is installed
"""
rc = utils.subprocess_get(self.logger, "dnsmasq --help")
if rc.find("Valid options") == -1:
status.append("dnsmasq is not installed and/or in path")
def get_file_lines(self, filename: str):
"""
Get lines from a file, which may or may not be compressed. If compressed then it will be uncompressed using
``gzip`` as the algorithm.
:param filename: The name of the file to be read.
:return: An array with all the lines.
"""
ftype = magic.detect_from_filename(filename)
if ftype.mime_type == "application/gzip":
try:
with gzip.open(filename, 'r') as f:
return f.readlines()
except:
pass
if ftype.mime_type == "application/x-ms-wim":
cmd = "/usr/bin/wiminfo"
if os.path.exists(cmd):
cmd = "%s %s" % (cmd, filename)
return utils.subprocess_get(cmd).splitlines()
self.logger.info("no %s found, please install wimlib-utils", cmd)
elif ftype.mime_type == "text/plain":
with open(filename, 'r') as f:
return f.readlines()
else:
self.logger.info(
'Could not detect the filetype and read the content of file "%s". Returning nothing.',
filename)
return []
def check_bind_bin(self, status):
"""
Check if bind is installed.
"""
rc = utils.subprocess_get(self.logger, "named -v")
# it should return something like "BIND 9.6.1-P1-RedHat-9.6.1-6.P1.fc11"
if rc.find("BIND") == -1:
status.append("named is not installed and/or in path")
def check_dnsmasq_bin(self, status):
"""
Check if dnsmasq is installed.
:param status: The status list with possible problems.
"""
rc = utils.subprocess_get(self.logger, "dnsmasq --help")
if rc.find("Valid options") == -1:
status.append("dnsmasq is not installed and/or in path")
def test_subprocess_get():
# Arrange
# Act
result = utils.subprocess_get("echo Test")
# Assert
# The newline makes sense in my (@SchoolGuy) eyes since we want to have multiline output also in a single string.
assert result == "Test\n"
def get_file_lines(self, filename):
"""
Get lines from a file, which may or may not be compressed
"""
lines = []
ftype = utils.subprocess_get(self.logger, "/usr/bin/file %s" % filename)
if ftype.find("gzip") != -1:
try:
import gzip
f = gzip.open(filename, 'r')
lines = f.readlines()
f.close()
except:
pass
elif ftype.find("text") != -1:
f = open(filename, 'r')
lines = f.readlines()
f.close()
return lines
def get_file_lines(self, filename):
"""
Get lines from a file, which may or may not be compressed
"""
lines = []
ftype = utils.subprocess_get(self.logger, "/usr/bin/file %s" % filename)
if ftype.find("gzip") != -1:
try:
import gzip
f = gzip.open(filename, 'r')
lines = f.readlines()
f.close()
except:
pass
elif ftype.find("text") != -1:
f = open(filename, 'r')
lines = f.readlines()
f.close()
return lines
def get_file_lines(self, filename):
"""
Get lines from a file, which may or may not be compressed. If compressed then it will be uncompressed using
``gzip`` as the algorithm.
:param filename: The name of the file to be read.
:return: An array with all the lines.
"""
lines = []
ftype = utils.subprocess_get(self.logger, "/usr/bin/file %s" % filename)
if ftype.find("gzip") != -1:
try:
import gzip
f = gzip.open(filename, 'r')
lines = f.readlines()
f.close()
except:
pass
elif ftype.find("text") != -1:
f = open(filename, 'r')
lines = f.readlines()
f.close()
return lines
def check_push(fn,
tftpbootdir,
settings,
lcache='/var/lib/cobbler',
logger=None):
"""
Returns the sha1sum of the file
"""
db = {}
try:
dbfile = os.path.join(lcache, 'pxe_cache.json')
if os.path.exists(dbfile):
db = simplejson.load(open(dbfile, 'r'))
except:
pass
count = 0
while not os.path.exists(fn) and count < 10:
count += 1
if _DEBUG:
logger.debug("%s does not exist yet - retrying (try %s)" %
(fn, count))
time.sleep(1)
mtime = os.stat(fn).st_mtime
key = None
needpush = True
if _DEBUG:
logger.debug("check_push(%s)" % fn)
if db.has_key(fn):
if db[fn][0] < mtime:
if _DEBUG:
logger.debug("mtime differ - old: %s new: %s" %
(db[fn][0], mtime))
if os.path.exists(fn):
cmd = '/usr/bin/sha1sum %s' % fn
key = utils.subprocess_get(None, cmd).split(' ')[0]
if _DEBUG:
logger.debug("checking checksum - old: %s new: %s" %
(db[fn][1], key))
if key == db[fn][1]:
needpush = False
else:
needpush = False
if key is None:
if os.path.exists(fn):
cmd = '/usr/bin/sha1sum %s' % fn
key = utils.subprocess_get(None, cmd).split(' ')[0]
if _DEBUG:
logger.debug("push(%s) ? %s" % (fn, needpush))
if needpush:
# reset the Result var
ProxySync.ResultLock.acquire()
ProxySync.Result = True
ProxySync.ResultLock.release()
format = 'other'
if "pxelinux.cfg" in fn:
format = 'pxe'
elif "grub" in fn:
format = 'grub'
if sync_to_proxies(fn, tftpbootdir, format, settings, logger):
db[fn] = (mtime, key)
simplejson.dump(db, open(dbfile, 'w'))
logger.info("Push successfull")
else:
logger.info("Push failed")
def gen_win_files(distro, meta):
(kernel_path, kernel_name) = os.path.split(distro.kernel)
distro_path = utils.find_distro_path(settings, distro)
distro_dir = wim_file_name = os.path.join(settings.tftpboot_location,
"images", distro.name)
web_dir = os.path.join(settings.webdir, "images", distro.name)
is_winpe = "winpe" in meta and meta['winpe'] != ""
is_bcd = "bcd" in meta and meta['bcd'] != ""
if "kernel" in meta:
kernel_name = meta["kernel"]
kernel_name = os.path.basename(kernel_name)
is_wimboot = "wimboot" in kernel_name
if is_wimboot:
distro_path = os.path.join(settings.webdir, "distro_mirror",
distro.name)
kernel_path = os.path.join(distro_path, "boot")
if "kernel" in meta and "wimboot" not in distro.kernel:
tgen.copy_single_distro_file(
os.path.join(settings.tftpboot_location, kernel_name),
distro_dir, False)
tgen.copy_single_distro_file(
os.path.join(distro_dir, kernel_name), web_dir, True)
if "post_install_script" in meta:
post_install_dir = distro_path
if distro.os_version not in ("XP", "2003"):
post_install_dir = os.path.join(post_install_dir, "sources")
post_install_dir = os.path.join(post_install_dir, "$OEM$", "$1")
if not os.path.exists(post_install_dir):
utils.mkdir(post_install_dir)
data = templ.render(post_tmpl_data, meta, None)
post_install_script = os.path.join(post_install_dir,
meta["post_install_script"])
logger.info('Build post install script: ' + post_install_script)
with open(post_install_script, "w+") as pi_file:
pi_file.write(data)
if "answerfile" in meta:
data = templ.render(tmpl_data, meta, None)
answerfile_name = os.path.join(distro_dir, meta["answerfile"])
logger.info('Build answer file: ' + answerfile_name)
with open(answerfile_name, "w+") as answerfile:
answerfile.write(data)
tgen.copy_single_distro_file(answerfile_name, distro_path, False)
tgen.copy_single_distro_file(answerfile_name, web_dir, True)
if "kernel" in meta and "bootmgr" in meta:
wk_file_name = os.path.join(distro_dir, kernel_name)
wl_file_name = os.path.join(distro_dir, meta["bootmgr"])
tl_file_name = os.path.join(kernel_path, "bootmgr.exe")
if distro.os_version in ("XP", "2003") and not is_winpe:
tl_file_name = os.path.join(kernel_path, "setupldr.exe")
if len(meta["bootmgr"]) != 5:
logger.error(
"The loader name should be EXACTLY 5 character")
return 1
pat1 = re.compile(br'NTLDR', re.IGNORECASE)
pat2 = re.compile(br'winnt\.sif', re.IGNORECASE)
with open(tl_file_name, 'rb') as file:
out = data = file.read()
if "answerfile" in meta:
if len(meta["answerfile"]) != 9:
logger.error(
"The response file name should be EXACTLY 9 character"
)
return 1
out = pat2.sub(bytes(meta["answerfile"], 'utf-8'), data)
else:
if len(meta["bootmgr"]) != 11:
logger.error(
"The Boot manager file name should be EXACTLY 11 character"
)
return 1
bcd_name = "bcd"
if is_bcd:
bcd_name = meta["bcd"]
if len(bcd_name) != 3:
logger.error(
"The BCD file name should be EXACTLY 3 character")
return 1
if not os.path.isfile(tl_file_name):
logger.error("File not found: %s" % tl_file_name)
return 1
pat1 = re.compile(br'bootmgr\.exe', re.IGNORECASE)
pat2 = re.compile(br'(\\.B.o.o.t.\\.)(B)(.)(C)(.)(D)',
re.IGNORECASE)
bcd_name = bytes(
"\\g<1>" + bcd_name[0] + "\\g<3>" + bcd_name[1] +
"\\g<5>" + bcd_name[2], 'utf-8')
with open(tl_file_name, 'rb') as file:
out = file.read()
if not is_wimboot:
logger.info('Patching build Loader: %s' % wl_file_name)
out = pat2.sub(bcd_name, out)
if tl_file_name != wl_file_name:
logger.info('Build Loader: %s from %s' %
(wl_file_name, tl_file_name))
with open(wl_file_name, 'wb+') as file:
file.write(out)
tgen.copy_single_distro_file(wl_file_name, web_dir, True)
if not is_wimboot:
if distro.os_version not in ("XP", "2003") or is_winpe:
pe = pefile.PE(wl_file_name, fast_load=True)
pe.OPTIONAL_HEADER.CheckSum = pe.generate_checksum()
pe.write(filename=wl_file_name)
with open(distro.kernel, 'rb') as file:
data = file.read()
out = pat1.sub(bytes(meta["bootmgr"], 'utf-8'), data)
if wk_file_name != distro.kernel:
logger.info("Build PXEBoot: %s from %s" %
(wk_file_name, distro.kernel))
with open(wk_file_name, 'wb+') as file:
file.write(out)
tgen.copy_single_distro_file(wk_file_name, web_dir, True)
if is_bcd:
obcd_file_name = os.path.join(kernel_path, "bcd")
bcd_file_name = os.path.join(distro_dir, meta["bcd"])
wim_file_name = 'winpe.wim'
if not os.path.isfile(obcd_file_name):
logger.error("File not found: %s" % obcd_file_name)
return 1
if is_winpe:
wim_file_name = meta["winpe"]
if is_wimboot:
wim_file_name = '\\Boot\\' + wim_file_name
sdi_file_name = '\\Boot\\' + 'boot.sdi'
else:
wim_file_name = os.path.join("/images", distro.name,
wim_file_name)
sdi_file_name = os.path.join("/images", distro.name,
os.path.basename(distro.initrd))
logger.info('Build BCD: %s from %s for %s' %
(bcd_file_name, obcd_file_name, wim_file_name))
bcdedit(obcd_file_name, bcd_file_name, wim_file_name,
sdi_file_name)
tgen.copy_single_distro_file(bcd_file_name, web_dir, True)
if is_winpe:
ps_file_name = os.path.join(distro_dir, meta["winpe"])
wim_pl_name = os.path.join(kernel_path, "winpe.wim")
cmd = ["/usr/bin/cp", "--reflink=auto", wim_pl_name, ps_file_name]
utils.subprocess_call(cmd, shell=False)
tgen.copy_single_distro_file(ps_file_name, web_dir, True)
if os.path.exists(wimupdate):
data = templ.render(tmplstart_data, meta, None)
pi_file = tempfile.NamedTemporaryFile()
pi_file.write(bytes(data, 'utf-8'))
pi_file.flush()
cmd = [
"/usr/bin/wimdir %s 1 | /usr/bin/grep -i '^/Windows/System32/startnet.cmd$'"
% ps_file_name
]
startnet_path = utils.subprocess_get(cmd, shell=True)[0:-1]
cmd = [
wimupdate, ps_file_name,
"--command=add %s %s" % (pi_file.name, startnet_path)
]
utils.subprocess_call(cmd, shell=False)
pi_file.close()
def run(self,
path: str,
name: str,
network_root=None,
autoinstall_file=None,
arch: Optional[str] = None,
breed=None,
os_version=None):
"""
This is the main entry point in a manager. It is a required function for import modules.
:param path: the directory we are scanning for files
:param name: the base name of the distro
:param network_root: the remote path (nfs/http/ftp) for the distro files
:param autoinstall_file: user-specified response file, which will override the default
:param arch: user-specified architecture
:param breed: user-specified breed
:param os_version: user-specified OS version
:raises CX
"""
self.name = name
self.network_root = network_root
self.autoinstall_file = autoinstall_file
self.arch = arch
self.breed = breed
self.os_version = os_version
self.path = path
self.rootdir = path
self.pkgdir = path
# some fixups for the XMLRPC interface, which does not use "None"
if self.arch == "":
self.arch = None
if self.name == "":
self.name = None
if self.autoinstall_file == "":
self.autoinstall_file = None
if self.os_version == "":
self.os_version = None
if self.network_root == "":
self.network_root = None
if self.os_version and not self.breed:
utils.die(
"OS version can only be specified when a specific breed is selected"
)
self.signature = self.scan_signatures()
if not self.signature:
error_msg = "No signature matched in %s" % path
self.logger.error(error_msg)
raise CX(error_msg)
# now walk the filesystem looking for distributions that match certain patterns
self.logger.info("Adding distros from path %s:" % self.path)
distros_added = []
import_walker(self.path, self.distro_adder, distros_added)
if len(distros_added) == 0:
if self.breed == "windows":
cmd_path = "/usr/bin/wimexport"
bootwim_path = os.path.join(self.path, "sources", "boot.wim")
dest_path = os.path.join(self.path, "boot")
if os.path.exists(cmd_path) and os.path.exists(bootwim_path):
winpe_path = os.path.join(dest_path, "winpe.wim")
if not os.path.exists(dest_path):
utils.mkdir(dest_path)
rc = utils.subprocess_call(
[cmd_path, bootwim_path, "1", winpe_path, "--boot"],
shell=False)
if rc == 0:
cmd = [
"/usr/bin/wimdir %s 1 | /usr/bin/grep -i '^/Windows/Boot/PXE$'"
% winpe_path
]
pxe_path = utils.subprocess_get(cmd, shell=True)[0:-1]
cmd = [
"/usr/bin/wimdir %s 1 | /usr/bin/grep -i '^/Windows/System32/config/SOFTWARE$'"
% winpe_path
]
config_path = utils.subprocess_get(cmd,
shell=True)[0:-1]
cmd_path = "/usr/bin/wimextract"
rc = utils.subprocess_call([
cmd_path, bootwim_path, "1",
"%s/pxeboot.n12" % pxe_path,
"%s/bootmgr.exe" % pxe_path, config_path,
"--dest-dir=%s" % dest_path, "--no-acls",
"--no-attributes"
],
shell=False)
if rc == 0:
if HAS_HIVEX:
software = os.path.join(
dest_path, os.path.basename(config_path))
h = hivex.Hivex(software, write=True)
root = h.root()
node = h.node_get_child(root, "Microsoft")
node = h.node_get_child(node, "Windows NT")
node = h.node_get_child(node, "CurrentVersion")
h.node_set_value(
node, {
"key":
"SystemRoot",
"t":
REG_SZ,
"value":
"x:\\Windows\0".encode(
encoding="utf-16le")
})
node = h.node_get_child(node, "WinPE")
# remove the key InstRoot from the registry
values = h.node_values(node)
new_values = []
for value in values:
keyname = h.value_key(value)
if keyname == "InstRoot":
continue
val = h.node_get_value(node, keyname)
valtype = h.value_type(val)[0]
value2 = h.value_value(val)[1]
valobject = {
"key": keyname,
"t": int(valtype),
"value": value2
}
new_values.append(valobject)
h.node_set_values(node, new_values)
h.commit(software)
cmd_path = "/usr/bin/wimupdate"
rc = utils.subprocess_call([
cmd_path, winpe_path,
"--command=add %s %s" %
(software, config_path)
],
shell=False)
os.remove(software)
else:
self.logger.info(
"python3-hivex not found. If you need Automatic Windows "
"Installation support, please install.")
import_walker(self.path, self.distro_adder,
distros_added)
if len(distros_added) == 0:
self.logger.warning("No distros imported, bailing out")
return
# find out if we can auto-create any repository records from the install tree
if self.network_root is None:
self.logger.info("associating repos")
# FIXME: this automagic is not possible (yet) without mirroring
self.repo_finder(distros_added)
