def sync_dhcp(self):
restart_dhcp = str(self.settings.restart_dhcp).lower()
which_dhcp_module = module_loader.get_module_name("dhcp", "module").strip()
if self.settings.manage_dhcp:
self.write_dhcp()
if which_dhcp_module == "manage_isc":
service_name = utils.dhcp_service_name(self.api)
if restart_dhcp != "0":
rc = utils.subprocess_call(self.logger, "dhcpd -t -q", shell=True)
if rc != 0:
self.logger.error("dhcpd -t failed")
return False
service_restart = "service %s restart" % service_name
rc = utils.subprocess_call(self.logger, service_restart, shell=True)
if rc != 0:
self.logger.error("%s failed" % service_name)
return False
elif which_dhcp_module == "manage_dnsmasq":
if restart_dhcp != "0":
rc = utils.subprocess_call(self.logger, "service dnsmasq restart")
if rc != 0:
self.logger.error("service dnsmasq restart failed")
return False
return True
def check_service(self, status, which, notes=""):
if notes != "":
notes = " (NOTE: %s)" % notes
rc = 0
if self.checked_dist == "redhat" or self.checked_dist == "suse":
if os.path.exists("/etc/rc.d/init.d/%s" % which):
rc = utils.subprocess_call(self.logger,"/sbin/service %s status > /dev/null 2>/dev/null" % which, shell=True)
if rc != 0:
status.append(_("service %s is not running%s") % (which,notes))
return False
elif self.checked_dist in ["debian", "ubuntu"]:
# we still use /etc/init.d
if os.path.exists("/etc/init.d/%s" % which):
rc = utils.subprocess_call(self.logger,"/etc/init.d/%s status /dev/null 2>/dev/null" % which, shell=True)
if rc != 0:
status.append(_("service %s is not running%s") % which,notes)
return False
elif self.checked_dist == "ubuntu":
if os.path.exists("/etc/init/%s.conf" % which):
rc = utils.subprocess_call(self.logger,"status %s > /dev/null 2>&1" % which, shell=True)
if rc != 0:
status.append(_("service %s is not running%s") % (which,notes))
else:
status.append(_("Unknown distribution type, cannot check for running service %s" % which))
return False
return True
def check_for_wget_curl(self,status):
"""
Check to make sure wget or curl is installed
"""
rc1 = utils.subprocess_call(self.logger,"which wget")
rc2 = utils.subprocess_call(self.logger,"which curl")
if rc1 != 0 and rc2 != 0:
status.append("Neither wget nor curl are installed and/or available in $PATH. Cobbler requires that one of these utilities be installed.")
def update_permissions(self, repo_path):
"""
Verifies that permissions and contexts after an rsync are as expected.
Sending proper rsync flags should prevent the need for this, though this is largely
a safeguard.
"""
# all_path = os.path.join(repo_path, "*")
cmd1 = "chown -R root:apache %s" % repo_path
utils.subprocess_call(self.logger, cmd1)
cmd2 = "chmod -R 755 %s" % repo_path
utils.subprocess_call(self.logger, cmd2)
def after_download(self, profile_data):
if not os.path.exists("/sbin/grubby"):
raise InfoException, "grubby is not installed"
k_args = self.calc_kernel_args(profile_data,replace_self=True)
kickstart = self.safe_load(profile_data,'kickstart')
self.build_initrd(
self.safe_load(profile_data,'initrd_local'),
kickstart,
profile_data
)
if len(k_args) > 255:
raise InfoException, "Kernel options are too long, 255 chars exceeded: %s" % k_args
cmd = [ "/sbin/grubby",
"--add-kernel", self.safe_load(profile_data,'kernel_local'),
"--initrd", self.safe_load(profile_data,'initrd_local'),
"--args", k_args,
"--copy-default"
]
if self.add_reinstall_entry:
cmd.append("--title=Reinstall")
else:
cmd.append("--make-default")
cmd.append("--title=kick%s" % int(time.time()))
if self.live_cd:
cmd.append("--bad-image-okay")
cmd.append("--boot-filesystem=/dev/sda1")
cmd.append("--config-file=/tmp/boot/boot/grub/grub.conf")
# utils.subprocess_call(["/sbin/grubby","--remove-kernel","/boot/vmlinuz"])
utils.subprocess_call(cmd)
# if grubby --bootloader-probe returns lilo,
# apply lilo changes
cmd = [ "/sbin/grubby", "--bootloader-probe" ]
probe_process = sub_process.Popen(cmd, stdout=sub_process.PIPE)
which_loader = probe_process.communicate()[0]
if probe_process.returncode == 0 and \
which_loader.find("lilo") != -1:
print "- applying lilo changes"
cmd = [ "/sbin/lilo" ]
sub_process.Popen(cmd, stdout=sub_process.PIPE).communicate()[0]
if not self.add_reinstall_entry:
print "- reboot to apply changes"
else:
print "- reinstallation entry added"
def sync_dhcp(self):
restart_dhcp = str(self.settings.restart_dhcp).lower()
service_name = utils.dhcp_service_name(self.api)
if restart_dhcp != "0":
rc = utils.subprocess_call(self.logger, "dhcpd -t -q", shell=True)
if rc != 0:
error_msg = "dhcpd -t failed"
self.logger.error(error_msg)
raise CX(error_msg)
service_restart = "service %s restart" % service_name
rc = utils.subprocess_call(self.logger, service_restart, shell=True)
if rc != 0:
error_msg = "%s failed" % service_name
self.logger.error(error_msg)
raise CX(error_msg)
def run(api,args,logger):
# FIXME: make everything use the logger, no prints, use util.subprocess_call, etc
objtype = args[0] # "system" or "profile"
name = args[1] # name of system or profile
ip = args[2] # ip or "?"
if objtype == "system":
target = api.find_system(name)
else:
target = api.find_profile(name)
# collapse the object down to a rendered datastructure
target = utils.blender(api, False, target)
if target == {}:
logger.info("unable to locate %s " % name)
raise CX("failure looking up target")
if target['ks_meta']['vms']:
for vm in target['ks_meta']['vms'].split(','):
try:
arglist = ["/usr/local/bin/createvm",target['ip_address_vmnic1'],vm,target['server']]
logger.info("creating virtual guest %s" % vm)
rc = utils.subprocess_call(logger, arglist, shell=False)
except Exception, reason:
logger.error("unable to create %s: %s" % (name,reason))
if rc != 0:
raise CX("cobbler trigger failed: %(file)s returns ")
def run(api, args, logger):
objtype = args[0] # "system" or "profile"
name = args[1] # name of system or profile
ip = args[2] # ip or "?"
if objtype != "system":
return 0
settings = api.settings()
if not str(settings.puppet_auto_setup).lower() in [ "1", "yes", "y", "true"]:
return 0
if not str(settings.sign_puppet_certs_automatically).lower() in [ "1", "yes", "y", "true"]:
return 0
system = api.find_system(name)
system = utils.blender(api, False, system)
hostname = system[ "hostname" ]
puppetca_path = settings.puppetca_path
cmd = [puppetca_path, '--sign', hostname]
rc = 0
try:
rc = utils.subprocess_call(logger, cmd, shell=False)
except:
if logger is not None:
logger.warning("failed to execute %s", puppetca_path)
if rc != 0:
if logger is not None:
logger.warning("signing of puppet cert for %s failed", name)
return 0
def rsync_sync(self, repo):
"""
Handle copying of rsync:// and rsync-over-ssh repos.
"""
repo_mirror = repo.mirror
if not repo.mirror_locally:
utils.die(self.logger,"rsync:// urls must be mirrored locally, yum cannot access them directly")
if repo.rpm_list != "" and repo.rpm_list != []:
self.logger.warning("--rpm-list is not supported for rsync'd repositories")
# FIXME: don't hardcode
dest_path = os.path.join(self.settings.webdir+"/repo_mirror", repo.name)
spacer = ""
if not repo.mirror.startswith("rsync://") and not repo.mirror.startswith("/"):
spacer = "-e ssh"
if not repo.mirror.endswith("/"):
repo.mirror = "%s/" % repo.mirror
# FIXME: wrapper for subprocess that logs to logger
cmd = "rsync -rltDv --copy-unsafe-links --delete-after %s --delete --exclude-from=/etc/cobbler/rsync.exclude %s %s" % (spacer, repo.mirror, dest_path)
rc = utils.subprocess_call(self.logger, cmd)
if rc !=0:
utils.die(self.logger,"cobbler reposync failed")
os.path.walk(dest_path, self.createrepo_walker, repo)
self.create_local_file(dest_path, repo)
def power(self, desired_state):
"""
state is either "on" or "off". Rebooting is implemented at the api.py
level.
The user and password need not be supplied. If not supplied they
will be taken from the environment, COBBLER_POWER_USER and COBBLER_POWER_PASS.
If provided, these will override any other data and be used instead. Users
interested in maximum security should take that route.
"""
template = self.get_command_template()
template_file = open(template, "r")
meta = utils.blender(self.api, False, self.system)
meta["power_mode"] = desired_state
# allow command line overrides of the username/password
if self.force_user is not None:
meta["power_user"] = self.force_user
if self.force_pass is not None:
meta["power_pass"] = self.force_pass
tmp = templar.Templar(self.api._config)
cmd = tmp.render(template_file, meta, None, self.system)
template_file.close()
cmd = cmd.strip()
self.logger.info("cobbler power configuration is:")
self.logger.info(" type : %s" % self.system.power_type)
self.logger.info(" address: %s" % self.system.power_address)
self.logger.info(" user : %s" % self.system.power_user)
self.logger.info(" id : %s" % self.system.power_id)
# if no username/password data, check the environment
if meta.get("power_user","") == "":
meta["power_user"] = os.environ.get("COBBLER_POWER_USER","")
if meta.get("power_pass","") == "":
meta["power_pass"] = os.environ.get("COBBLER_POWER_PASS","")
# now reprocess the command so we don't feed it through the shell
cmd = cmd.split(" ")
# Try the power command 5 times before giving up.
# Some power switches are flakey
for x in range(0,5):
rc = utils.subprocess_call(self.logger, cmd, shell=False)
if rc == 0:
break
else:
time.sleep(2)
if not rc == 0:
utils.die(self.logger,"command failed (rc=%s), please validate the physical setup and cobbler config" % rc)
return rc
def sync_dhcp(self):
restart_dhcp = str(self.settings.restart_dhcp).lower()
if restart_dhcp != "0":
rc = utils.subprocess_call(self.logger, "service dnsmasq restart")
if rc != 0:
error_msg = "service dnsmasq restart failed"
self.logger.error(error_msg)
raise CX(error_msg)
def check_iptables(self, status):
if os.path.exists("/etc/rc.d/init.d/iptables"):
rc = utils.subprocess_call(self.logger, "/sbin/service iptables status >/dev/null 2>/dev/null", shell=True)
if rc == 0:
status.append(
_("since iptables may be running, ensure 69, 80, and %(xmlrpc)s are unblocked")
% {"xmlrpc": self.settings.xmlrpc_port}
)
def run(self):
"""
Simply hardlinks directories that are cobbler managed.
This is a /very/ simple command but may grow more complex
and intelligent over time.
"""
# FIXME: if these directories become configurable some
# changes will be required here.
if not os.path.exists("/usr/sbin/hardlink"):
utils.die(self.logger,"please install 'hardlink' (/usr/sbin/hardlink) to use this feature")
self.logger.info("now hardlinking to save space, this may take some time.")
utils.subprocess_call(self.logger,"/usr/sbin/hardlink -c -v /var/www/cobbler/ks_mirror /var/www/cobbler/repo_mirror",shell=True)
return rc
def build_initrd(self,initrd,kickstart,data):
"""
Crack open an initrd and install the kickstart file.
"""
# save kickstart to file
ksdata = utils.urlread(kickstart)
fd = open("/var/spool/koan/ks.cfg","w+")
if ksdata is not None:
fd.write(ksdata)
fd.close()
# handle insertion of kickstart based on type of initrd
fd = open("/var/spool/koan/insert.sh","w+")
fd.write(self.get_insert_script(initrd))
fd.close()
utils.subprocess_call([ "/bin/bash", "/var/spool/koan/insert.sh" ])
shutil.copyfile("/var/spool/koan/initrd_final", initrd)
def run_this(self, cmd, args):
"""
A simple wrapper around subprocess calls.
"""
my_cmd = cmd % args
rc = utils.subprocess_call(self.logger,my_cmd,shell=True)
if rc != 0:
utils.die(self.logger,"Command failed")
def modacl(self,isadd,isuser,who):
webdir = self.settings.webdir
snipdir = self.settings.snippetsdir
tftpboot = utils.tftpboot_location()
PROCESS_DIRS = {
"/var/log/cobbler" : "rwx",
"/var/log/cobbler/tasks" : "rwx",
"/var/lib/cobbler" : "rwx",
"/etc/cobbler" : "rwx",
tftpboot : "rwx",
"/var/lib/cobbler/triggers" : "rwx"
}
if not snipdir.startswith("/var/lib/cobbler/"):
PROCESS_DIRS[snipdir] = "r"
cmd = "-R"
if isadd:
cmd = "%s -m" % cmd
else:
cmd = "%s -x" % cmd
if isuser:
cmd = "%s u:%s" % (cmd,who)
else:
cmd = "%s g:%s" % (cmd,who)
for d in PROCESS_DIRS:
how = PROCESS_DIRS[d]
if isadd:
cmd2 = "%s:%s" % (cmd,how)
else:
cmd2 = cmd
cmd2 = "%s %s" % (cmd2,d)
rc = utils.subprocess_call(self.logger,"setfacl -d %s" % cmd2,shell=True)
if not rc == 0:
utils.die(self.logger,"command failed")
rc = utils.subprocess_call(self.logger,"setfacl %s" % cmd2,shell=True)
if not rc == 0:
utils.die(self.logger,"command failed")
def rsync_it(self, from_path, to_path, type=None):
from_path = "%s::%s" % (self.master, from_path)
if type == 'repo':
cmd = "rsync %s %s %s" % (self.settings.replicate_repo_rsync_options, from_path, to_path)
else:
cmd = "rsync %s %s %s" % (self.settings.replicate_rsync_options, from_path, to_path)
rc = utils.subprocess_call(self.logger, cmd, shell=True)
if rc != 0:
self.logger.info("rsync failed")
def update_permissions(self, repo_path):
"""
Verifies that permissions and contexts after an rsync are as expected.
Sending proper rsync flags should prevent the need for this, though this is largely
a safeguard.
"""
# all_path = os.path.join(repo_path, "*")
owner = "root:apache"
if os.path.exists("/etc/SuSE-release"):
owner = "root:www"
elif os.path.exists("/etc/debian_version"):
owner = "root:www-data"
cmd1 = "chown -R " + owner + " %s" % repo_path
utils.subprocess_call(self.logger, cmd1)
cmd2 = "chmod -R 755 %s" % repo_path
utils.subprocess_call(self.logger, cmd2)
def createrepo_walker(self, repo, dirname, fnames):
"""
Used to run createrepo on a copied Yum mirror.
"""
if os.path.exists(dirname) or repo["breed"] == "rsync":
utils.remove_yum_olddata(dirname)
# add any repo metadata we can use
mdoptions = []
if os.path.isfile("%s/repodata/repomd.xml" % (dirname)):
if not HAS_YUM:
utils.die(self.logger, "yum is required to use this feature")
rmd = yum.repoMDObject.RepoMD("", "%s/repodata/repomd.xml" % (dirname))
if rmd.repoData.has_key("group"):
groupmdfile = rmd.getData("group").location[1]
mdoptions.append("-g %s" % groupmdfile)
if rmd.repoData.has_key("prestodelta"):
# need createrepo >= 0.9.7 to add deltas
if utils.check_dist() == "redhat" or utils.check_dist() == "suse":
cmd = "/usr/bin/rpmquery --queryformat=%{VERSION} createrepo"
createrepo_ver = utils.subprocess_get(self.logger, cmd)
if createrepo_ver >= "0.9.7":
mdoptions.append("--deltas")
else:
utils.die(
self.logger,
"this repo has presto metadata; you must upgrade createrepo to >= 0.9.7 first and then need to resync the repo through cobbler.",
)
blended = utils.blender(self.api, False, repo)
flags = blended.get("createrepo_flags", "(ERROR: FLAGS)")
try:
# BOOKMARK
cmd = "createrepo %s %s %s" % (" ".join(mdoptions), flags, dirname)
utils.subprocess_call(self.logger, cmd)
except:
utils.log_exc(self.logger)
self.logger.error("createrepo failed.")
del fnames[:] # we're in the right place
def run(self):
"""
Simply hardlinks directories that are cobbler managed.
This is a /very/ simple command but may grow more complex
and intelligent over time.
"""
# FIXME: if these directories become configurable some
# changes will be required here.
if not os.path.exists(self.hardlink):
utils.die(self.logger, "please install 'hardlink' (%s) to use this feature" % self.hardlink)
self.logger.info("now hardlinking to save space, this may take some time.")
rc = utils.subprocess_call(self.logger, self.hardlink_cmd, shell=True)
# FIXME: how about settings? (self.settings.webdir)
webdir = "/var/www/cobbler"
if os.path.exists("/srv/www"):
webdir = "/srv/www/cobbler"
rc = utils.subprocess_call(self.logger, "/usr/sbin/hardlink -c -v " + webdir + "/ks_mirror /var/www/cobbler/repo_mirror", shell=True)
return rc
def run(self):
"""
Simply hardlinks directories that are cobbler managed.
This is a /very/ simple command but may grow more complex
and intelligent over time.
"""
# FIXME: if these directories become configurable some
# changes will be required here.
if not os.path.exists(self.hardlink):
utils.die(self.logger,"please install 'hardlink' (%s) to use this feature" % self.hardlink)
self.logger.info("now hardlinking to save space, this may take some time.")
rc = utils.subprocess_call(self.logger,self.hardlink_cmd,shell=True)
return rc
def run(api, args, logger):
objtype = args[0] # "system" or "profile"
name = args[1] # name of system or profile
# ip = args[2] # ip or "?"
if objtype != "system":
return 0
settings = api.settings()
if not str(settings.puppet_auto_setup).lower() in ["1", "yes", "y", "true"]:
return 0
if not str(settings.remove_old_puppet_certs_automatically).lower() in ["1", "yes", "y", "true"]:
return 0
system = api.find_system(name)
system = utils.blender(api, False, system)
hostname = system["hostname"]
if not re.match(r"[\w-]+\..+", hostname):
search_domains = system["name_servers_search"]
if search_domains:
hostname += "." + search_domains[0]
if not re.match(r"[\w-]+\..+", hostname):
default_search_domains = system["default_name_servers_search"]
if default_search_domains:
hostname += "." + default_search_domains[0]
puppetca_path = settings.puppetca_path
cmd = [puppetca_path, "cert", "clean", hostname]
rc = 0
try:
rc = utils.subprocess_call(logger, cmd, shell=False)
except:
if logger is not None:
logger.warning("failed to execute %s" % puppetca_path)
if rc != 0:
if logger is not None:
logger.warning("puppet cert removal for %s failed" % name)
return 0
def update_permissions(self, repo_path):
"""
Verifies that permissions and contexts after an rsync are as expected.
Sending proper rsync flags should prevent the need for this, though this is largely
a safeguard.
"""
# all_path = os.path.join(repo_path, "*")
cmd1 = "chown -R root:apache %s" % repo_path
utils.subprocess_call(self.logger, cmd1)
cmd2 = "chmod -R 755 %s" % repo_path
utils.subprocess_call(self.logger, cmd2)
if self.config.api.is_selinux_enabled():
cmd3 = "chcon --reference /var/www %s >/dev/null 2>/dev/null" % repo_path
utils.subprocess_call(self.logger, cmd3)
def wget_sync(self, repo):
"""
Handle mirroring of directories using wget
"""
repo_mirror = repo.mirror.strip()
if repo.rpm_list != "" and repo.rpm_list != []:
self.logger.warning("--rpm-list is not supported for wget'd repositories")
# FIXME: don't hardcode
dest_path = os.path.join(self.settings.webdir + "/repo_mirror", repo.name)
# FIXME: wrapper for subprocess that logs to logger
cmd = "wget -N -np -r -l inf -nd -P %s %s" % (dest_path, repo_mirror)
rc = utils.subprocess_call(self.logger, cmd)
if rc != 0:
utils.die(self.logger, "cobbler reposync failed")
os.path.walk(dest_path, self.createrepo_walker, repo)
self.create_local_file(dest_path, repo)
def checkfile(self, obj, is_profile):
last_errors = []
blended = utils.blender(self.config.api, False, obj)
os_version = blended["os_version"]
self.logger.info("----------------------------")
self.logger.debug("osversion: %s" % os_version)
ks = blended["kickstart"]
if ks is None or ks == "":
self.logger.info("%s has no kickstart, skipping" % obj.name)
return [True, last_errors]
breed = blended["breed"]
if breed != "redhat":
self.logger.info("%s has a breed of %s, skipping" % (obj.name, breed))
return [True, last_errors]
server = blended["server"]
if not ks.startswith("/"):
url = ks
else:
if is_profile:
url = "http://%s/cblr/svc/op/ks/profile/%s" % (server, obj.name)
self.kickgen.generate_kickstart_for_profile(obj.name)
else:
url = "http://%s/cblr/svc/op/ks/system/%s" % (server, obj.name)
self.kickgen.generate_kickstart_for_system(obj.name)
last_errors = self.kickgen.get_last_errors()
self.logger.info("checking url: %s" % url)
rc = utils.subprocess_call(self.logger, "/usr/bin/ksvalidator -v \"%s\" \"%s\"" % (os_version, url), shell=True)
if rc != 0:
return [False, last_errors]
return [True, last_errors]
def __test_setup():
# this contains some code from remote.py that has been modified
# slightly to add in some extra parameters for these checks.
# it can probably be combined into something like a test_utils
# module later.
api = cobbler_api.BootAPI()
fake = open("/tmp/cobbler.fake","w+")
fake.write("")
fake.close()
distro = api.new_distro()
distro.set_name("distro0")
distro.set_kernel("/tmp/cobbler.fake")
distro.set_initrd("/tmp/cobbler.fake")
api.add_distro(distro)
repo = api.new_repo()
repo.set_name("repo0")
if not os.path.exists("/tmp/empty"):
os.mkdir("/tmp/empty",770)
repo.set_mirror("/tmp/empty")
files = glob.glob("rpm-build/*.rpm")
if len(files) == 0:
raise Exception("Tests must be run from the cobbler checkout directory.")
rc = utils.subprocess_call(None,"cp rpm-build/*.rpm /tmp/empty",shell=True)
api.add_repo(repo)
fd = open("/tmp/cobbler_t1","w+")
fd.write("$profile_name")
fd.close()
fd = open("/tmp/cobbler_t2","w+")
fd.write("$system_name")
fd.close()
profile = api.new_profile()
profile.set_name("profile0")
profile.set_distro("distro0")
profile.set_kickstart("/var/lib/cobbler/kickstarts/sample.ks")
profile.set_repos(["repo0"])
profile.set_mgmt_classes(["alpha","beta"])
profile.set_ksmeta({"tree":"look_for_this1","gamma":3})
profile.set_template_files("/tmp/cobbler_t1=/tmp/t1-rendered")
api.add_profile(profile)
system = api.new_system()
system.set_name("system0")
system.set_hostname("hostname0")
system.set_gateway("192.168.1.1")
system.set_profile("profile0")
system.set_dns_name("hostname0","eth0")
system.set_ksmeta({"tree":"look_for_this2"})
system.set_template_files({"/tmp/cobbler_t2":"/tmp/t2-rendered"})
api.add_system(system)
image = api.new_image()
image.set_name("image0")
image.set_file("/tmp/cobbler.fake")
api.add_image(image)
# perhaps an artifact of the test process?
# FIXME: get path (at least webdir) from settings?
if os.path.exists("/var/www/cobbler/repo_mirror/"):
utils.os_system("rm -rf /var/www/cobbler/repo_mirror/repo0")
elif os.path.exists("/srv/www/cobbler/repo_mirror/"):
utils.os_system("rm -rf /srv/www/cobbler/repo_mirror/repo0")
api.reposync(name="repo0")
def run(self,
iso=None,
buildisodir=None,
profiles=None,
systems=None,
distro=None,
standalone=None,
source=None,
exclude_dns=None):
# the distro option is for stand-alone builds only
if not standalone and distro is not None:
utils.die(
self.logger,
"The --distro option should only be used when creating a standalone ISO"
)
# if building standalone, we only want --distro,
# profiles/systems are disallowed
if standalone:
if profiles is not None or systems is not None:
utils.die(
self.logger,
"When building a standalone ISO, use --distro only instead of --profiles/--systems"
)
elif distro is None:
utils.die(
self.logger,
"When building a standalone ISO, you must specify a --distro"
)
if source != None and not os.path.exists(source):
utils.die(self.logger,
"The source specified (%s) does not exist" % source)
# if iso is none, create it in . as "kickstart.iso"
if iso is None:
iso = "kickstart.iso"
if buildisodir is None:
buildisodir = self.settings.buildisodir
else:
if not os.path.isdir(buildisodir):
utils.die(self.logger,
"The --tempdir specified is not a directory")
(buildisodir_head,
buildisodir_tail) = os.path.split(os.path.normpath(buildisodir))
if buildisodir_tail != "buildiso":
buildisodir = os.path.join(buildisodir, "buildiso")
self.logger.info("using/creating buildisodir: %s" % buildisodir)
if not os.path.exists(buildisodir):
os.makedirs(buildisodir)
else:
shutil.rmtree(buildisodir)
os.makedirs(buildisodir)
# if base of buildisodir does not exist, fail
# create all profiles unless filtered by "profiles"
imagesdir = os.path.join(buildisodir, "images")
isolinuxdir = os.path.join(buildisodir, "isolinux")
self.logger.info("building tree for isolinux")
if not os.path.exists(imagesdir):
os.makedirs(imagesdir)
if not os.path.exists(isolinuxdir):
os.makedirs(isolinuxdir)
self.logger.info("copying miscellaneous files")
isolinuxbin = "/usr/share/syslinux/isolinux.bin"
if not os.path.exists(isolinuxbin):
isolinuxbin = "/usr/lib/syslinux/isolinux.bin"
menu = "/usr/share/syslinux/menu.c32"
if not os.path.exists(menu):
menu = "/var/lib/cobbler/loaders/menu.c32"
chain = "/usr/share/syslinux/chain.c32"
if not os.path.exists(chain):
chain = "/usr/lib/syslinux/chain.c32"
files = [isolinuxbin, menu, chain]
for f in files:
if not os.path.exists(f):
utils.die(self.logger, "Required file not found: %s" % f)
else:
utils.copyfile(f, os.path.join(isolinuxdir,
os.path.basename(f)), self.api)
if standalone:
self.generate_standalone_iso(imagesdir, isolinuxdir, distro,
source)
else:
self.generate_netboot_iso(imagesdir, isolinuxdir, profiles,
systems, exclude_dns)
# removed --quiet
cmd = "mkisofs -o %s -r -b isolinux/isolinux.bin -c isolinux/boot.cat" % iso
cmd = cmd + " -no-emul-boot -boot-load-size 4"
cmd = cmd + " -boot-info-table -V Cobbler\ Install -R -J -T %s" % buildisodir
rc = utils.subprocess_call(self.logger, cmd, shell=True)
if rc != 0:
utils.die(self.logger, "mkisofs failed")
self.logger.info("ISO build complete")
self.logger.info("You may wish to delete: %s" % buildisodir)
self.logger.info("The output file is: %s" % iso)
return True
def clean_link_cache(self):
for dirtree in [self.bootloc, self.settings.webdir]:
cachedir = "%s/.link_cache" % dirtree
cmd = "find %s -maxdepth 1 -type f -links 1 -exec rm -f '{}' ';'" % cachedir
utils.subprocess_call(self.logger, cmd)
def download_and_trim_contours(bucket:str, key: str, localpath: str, trimstart:float=-1., trimend:float=-1., trimduration:float=-1., \
outdir:str='', refilter:bool=False, pickle_params:str=''):
s3_download_folder(bucket, key, localpath)
if trimend < 0. and trimduration < 0.:
trimend = int(1e9)
trimduration = int(1e9)
else:
assert not (trimend > 0. and trimduration > 0.), 'use one or the other'
if trimduration > 0.:
trimend = trimstart + trimduration
else:
trimduration = trimend - trimstart
assert trimduration > 0., str(trimend)+', '+str(trimstart)
tsfileend = '_timestamps.png'
#----------------------
tmpdir = localpath
while tmpdir.endswith('/'):
tmpdir = tmpdir[:-1]
subprocess_call(['mkdir','-p', tmpdir])
#----------------------
basecf = localpath.split('/')[-1]
backupbaseblockfold = copy(basecf)
if len(outdir) < 1:
ival = None
if '_' in basecf:
try:
ival = int(basecf.split('_')[-1])
except ValueError:
pass
if isinstance(ival,int):
ival += 1
basecf = '_'.join(basecf.split('_')[:-1])+'_'+str(ival)
# TODO: s3 head to check if this folder exists already
print("ayyy: "+str(ival))
else:
basecf += '_0'
outdir = basecf
assert '/' not in outdir, str(outdir)
print("outdir: "+str(outdir))
localoutdir = os.path.join(os.path.dirname(tmpdir), outdir)
subprocess_call(['mkdir','-p', localoutdir])
univ2lect = '/'.join(key.split('/')[:-2])
full_key = f'{univ2lect}/{outdir}'
print("localoutdir: "+str(localoutdir))
print("s3 out key: "+str(full_key))
pngs = [os.path.join(tmpdir,ff) for ff in os.listdir(tmpdir) if ff.endswith(tsfileend)]
assert len(pngs) > 0, str(pngs)+'\n'+str(list(os.listdir(tmpdir)))
subtrme = int(round(float(trimstart)))
maxtime = int(round(float(trimend )))
syncme = []
for pngf in pngs:
img = read_timestamps_image(pngf)
assert isinstance(img,np.ndarray), str(type(img))
assert len(img.shape) == 2, str(img.shape)
describe(os.path.basename(pngf), img)
if np.amin(img) > int(round(trimend)):
continue
# TODO: if we know erase times, remove contours which were erased before "trimstart"
img[img<subtrme] = subtrme
img[img>maxtime] = maxtime # TODO: this is needlessly destructive; but fixes potential frontend bugs (seek past end of video?)
img -= subtrme
writer = png.Writer(width=img.shape[1], height=img.shape[0], greyscale=True, alpha=False, bitdepth=16, compression=5)
outfname = os.path.join(localoutdir,os.path.basename(pngf))
syncme.append(outfname)
with open(outfname, 'wb') as openedfile:
writer.write(openedfile, img)
for fpth in syncme:
oldimgf = os.path.join(os.path.dirname(fpth), os.path.basename(fpth)[:-len(tsfileend)])
oldfs = [os.path.join(tmpdir,ff) for ff in os.listdir(tmpdir) if not ff.endswith(tsfileend) and ff.startswith(os.path.basename(oldimgf))]
assert len(oldfs) == 1, str(oldimgf)+'\n'+str(oldfs)
oldfs = oldfs[0]
subprocess_check_output(['cp', oldfs, localoutdir+'/'])
metafiles = [os.path.join(tmpdir,ff) for ff in os.listdir(tmpdir) if ff.lower().startswith('meta') and ff.lower().endswith('.json')]
for ff in metafiles:
print("in meta.json, replacing \'"+str(backupbaseblockfold)+"\' with \'"+str(outdir)+"\'")
replace_folder_in_metafile(ff, localoutdir+'/'+os.path.basename(ff), backupbaseblockfold, outdir)
# TODO: ADD THIS FUNCTIONALITY BACK
test_it = False
if refilter and test_it is True:
filterfile = '/evt/interactive-writing-segmentation/filter_keyframes.py'
assert os.path.isfile(filterfile), filterfile
fargs = ['python',filterfile,localoutdir,'--were_transparency_on_s3','--overwrite_in_place']
if len(pickle_params) > 1:
fargs += ['--autorun','--pickle_params',pickle_params]
assert 0 == subprocess_call(fargs)
print("syncing resulting folder")
while full_key.endswith('/'):
full_key = full_key[:-1]
s3_upload_folder(bucket, full_key, localoutdir)
return outdir
def clean_link_cache(self):
for dirtree in [os.path.join(self.bootloc, 'images'), self.settings.webdir]:
cachedir = os.path.join(dirtree, '.link_cache')
if os.path.isdir(cachedir):
cmd = "find %s -maxdepth 1 -type f -links 1 -exec rm -f '{}' ';'" % cachedir
utils.subprocess_call(self.logger, cmd)
def clean_link_cache(self):
for dirtree in [os.path.join(self.bootloc, 'images'), self.settings.webdir]:
cachedir = os.path.join(dirtree, '.link_cache')
if os.path.isdir(cachedir):
cmd = "find %s -maxdepth 1 -type f -links 1 -exec rm -f '{}' ';'" % cachedir
utils.subprocess_call(self.logger, cmd)
def check_iptables(self, status):
if os.path.exists("/etc/rc.d/init.d/iptables"):
rc = utils.subprocess_call(self.logger, "/sbin/service iptables status >/dev/null 2>/dev/null", shell=True)
if rc == 0:
status.append(_("since iptables may be running, ensure 69, 80/443, and %(xmlrpc)s are unblocked") % {"xmlrpc": self.settings.xmlrpc_port})
def start_install(*args, **kwargs):
cmd = virtinstall.build_commandline("import", *args, **kwargs)
utils.subprocess_call(cmd)
def generate_standalone_iso(self, imagesdir, isolinuxdir, distname,
filesource, airgapped):
"""
Create bootable CD image to be used for handsoff CD installtions
"""
# Get the distro object for the requested distro
# and then get all of its descendants (profiles/sub-profiles/systems)
distro = self.api.find_distro(distname)
if distro is None:
utils.die(self.logger,
"distro %s was not found, aborting" % distname)
descendants = distro.get_descendants()
if filesource is None:
# Try to determine the source from the distro kernel path
self.logger.debug("trying to locate source for distro")
found_source = False
(source_head, source_tail) = os.path.split(distro.kernel)
while source_tail != '':
if source_head == os.path.join(self.api.settings().webdir,
"distro_mirror"):
filesource = os.path.join(source_head, source_tail)
found_source = True
self.logger.debug("found source in %s" % filesource)
break
(source_head, source_tail) = os.path.split(source_head)
# Can't find the source, raise an error
if not found_source:
utils.die(
self.logger,
"Error, no installation source found. When building a standalone ISO, you must specify a --source if the distro install tree is not hosted locally"
)
self.logger.info("copying kernels and initrds for standalone distro")
self.copy_boot_files(distro, isolinuxdir, None)
self.logger.info("generating a isolinux.cfg")
isolinuxcfg = os.path.join(isolinuxdir, "isolinux.cfg")
cfg = open(isolinuxcfg, "w+")
cfg.write(self.iso_template)
if airgapped:
repo_names_to_copy = {}
for descendant in descendants:
data = utils.blender(self.api, False, descendant)
cfg.write("\n")
cfg.write("LABEL %s\n" % descendant.name)
cfg.write(" MENU LABEL %s\n" % descendant.name)
cfg.write(" kernel %s\n" % os.path.basename(distro.kernel))
append_line = " append initrd=%s" % os.path.basename(
distro.initrd)
if distro.breed == "redhat":
append_line += " ks=cdrom:/isolinux/%s.cfg" % descendant.name
if distro.breed == "suse":
append_line += " autoyast=file:///isolinux/%s.cfg install=cdrom:///" % descendant.name
if "install" in data["kernel_options"]:
del data["kernel_options"]["install"]
if distro.breed in ["ubuntu", "debian"]:
append_line += " auto-install/enable=true preseed/file=/cdrom/isolinux/%s.cfg" % descendant.name
# add remaining kernel_options to append_line
append_line += self.add_remaining_kopts(data["kernel_options"])
cfg.write(append_line)
if descendant.COLLECTION_TYPE == 'profile':
autoinstall_data = self.api.autoinstallgen.generate_autoinstall_for_profile(
descendant.name)
elif descendant.COLLECTION_TYPE == 'system':
autoinstall_data = self.api.autoinstallgen.generate_autoinstall_for_system(
descendant.name)
if distro.breed == "redhat":
cdregex = re.compile("^\s*url .*\n",
re.IGNORECASE | re.MULTILINE)
autoinstall_data = cdregex.sub("cdrom\n",
autoinstall_data,
count=1)
if airgapped:
descendant_repos = data['repos']
for repo_name in descendant_repos:
repo_obj = self.api.find_repo(repo_name)
error_fmt = (descendant.COLLECTION_TYPE + " " +
descendant.name + " refers to repo " +
repo_name +
", which %%s; cannot build airgapped ISO")
if repo_obj is None:
utils.die(self.logger, error_fmt % "does not exist")
if not repo_obj.mirror_locally:
utils.die(
self.logger, error_fmt %
"is not configured for local mirroring")
# FIXME: don't hardcode
mirrordir = os.path.join(self.settings.webdir,
"repo_mirror", repo_obj.name)
if not os.path.exists(mirrordir):
utils.die(
self.logger,
error_fmt % "has a missing local mirror directory")
repo_names_to_copy[repo_obj.name] = mirrordir
# update the baseurl in autoinstall_data to use the cdrom copy of this repo
reporegex = re.compile(
"^(\s*repo --name=" + repo_obj.name + " --baseurl=).*",
re.MULTILINE)
autoinstall_data = reporegex.sub(
r"\1" + "file:///mnt/source/repo_mirror/" +
repo_obj.name, autoinstall_data)
# rewrite any split-tree repos, such as in redhat, to use cdrom
srcreporegex = re.compile(
"^(\s*repo --name=\S+ --baseurl=).*/cobbler/ks_mirror/" +
distro.name + "/?(.*)", re.MULTILINE)
autoinstall_data = srcreporegex.sub(
r"\1" + "file:///mnt/source" + r"\2", autoinstall_data)
autoinstall_name = os.path.join(isolinuxdir,
"%s.cfg" % descendant.name)
autoinstall_file = open(autoinstall_name, "w+")
autoinstall_file.write(autoinstall_data)
autoinstall_file.close()
self.logger.info("done writing config")
cfg.write("\n")
cfg.write("MENU END\n")
cfg.close()
if airgapped:
# copy any repos found in profiles or systems to the iso build
repodir = os.path.abspath(isolinuxdir, "..", "repo_mirror")
if not os.path.exists(repodir):
os.makedirs(repodir)
for repo_name in repo_names_to_copy:
src = repo_names_to_copy[repo_name]
dst = os.path.join(repodir, repo_name)
self.logger.info(" - copying repo '" + repo_name +
"' for airgapped ISO")
ok = utils.rsync_files(
src,
dst,
"--exclude=TRANS.TBL --exclude=cache/ --no-g",
logger=self.logger,
quiet=True)
if not ok:
utils.die(self.logger,
"rsync of repo '" + repo_name + "' failed")
# copy distro files last, since they take the most time
cmd = "rsync -rlptgu --exclude=boot.cat --exclude=TRANS.TBL --exclude=isolinux/ %s/ %s/../" % (
filesource, isolinuxdir)
self.logger.info("- copying distro %s files (%s)" % (distname, cmd))
rc = utils.subprocess_call(self.logger, cmd, shell=True)
if rc:
utils.die(self.logger, "rsync of distro files failed")
def yum_sync(self, repo):
"""
Handle copying of http:// and ftp:// yum repos.
"""
# create the config file the hosts will use to access the repository.
repo_mirror = repo.mirror
dest_path = os.path.join(self.settings.webdir + "/repo_mirror",
repo.name)
self.create_local_file(dest_path, repo)
if not repo.mirror_locally:
return
cmd = self.reposync_cmd() # command to run
has_rpm_list = False # flag indicating not to pull the whole repo
# detect cases that require special handling
if repo.rpm_list != "" and repo.rpm_list != []:
has_rpm_list = True
# create yum config file for use by reposync
temp_path = os.path.join(dest_path, ".origin")
if not os.path.isdir(temp_path):
# FIXME: there's a chance this might break the RHN D/L case
os.makedirs(temp_path)
temp_file = self.create_local_file(temp_path, repo, output=False)
if not has_rpm_list:
# if we have not requested only certain RPMs, use reposync
cmd = "%s %s --config=%s --repoid=%s --download_path=%s" % (
cmd, self.rflags, temp_file, repo.name,
self.settings.webdir + "/repo_mirror")
if repo.arch != "":
if repo.arch == "x86":
repo.arch = "i386" # FIX potential arch errors
if repo.arch == "i386":
# counter-intuitive, but we want the newish kernels too
cmd = "%s -a i686" % (cmd)
else:
cmd = "%s -a %s" % (cmd, repo.arch)
else:
# create the output directory if it doesn't exist
if not os.path.exists(dest_path):
os.makedirs(dest_path)
use_source = ""
if repo.arch == "src":
use_source = "--source"
# older yumdownloader sometimes explodes on --resolvedeps
# if this happens to you, upgrade yum & yum-utils
extra_flags = self.settings.yumdownloader_flags
cmd = ""
if os.path.exists("/usr/bin/dnf"):
cmd = "/usr/bin/dnf download"
else:
cmd = "/usr/bin/yumdownloader"
cmd = "%s %s %s --disablerepo=* --enablerepo=%s -c %s --destdir=%s %s" % (
cmd, extra_flags, use_source, repo.name, temp_file, dest_path,
" ".join(repo.rpm_list))
# now regardless of whether we're doing yumdownloader or reposync
# or whether the repo was http://, ftp://, or rhn://, execute all queued
# commands here. Any failure at any point stops the operation.
rc = utils.subprocess_call(self.logger, cmd)
if rc != 0:
utils.die(self.logger, "cobbler reposync failed")
repodata_path = os.path.join(dest_path, "repodata")
# grab repomd.xml and use it to download any metadata we can use
proxies = None
if repo.proxy == '<<inherit>>':
proxies = {'http': self.settings.proxy_url_ext}
elif repo.proxy != '<<None>>' and repo.proxy != '':
proxies = {'http': repo.proxy, 'https': repo.proxy}
src = repo_mirror + "/repodata/repomd.xml"
dst = temp_path + "/repomd.xml"
try:
urlgrabber.grabber.urlgrab(src, filename=dst, proxies=proxies)
except Exception as e:
utils.die(self.logger, "failed to fetch " + src + " " + e.args)
# create our repodata directory now, as any extra metadata we're
# about to download probably lives there
if not os.path.isdir(repodata_path):
os.makedirs(repodata_path)
rmd = yum.repoMDObject.RepoMD('', "%s/repomd.xml" % (temp_path))
for mdtype in rmd.repoData.keys():
# don't download metadata files that are created by default
if mdtype not in [
"primary", "primary_db", "filelists", "filelists_db",
"other", "other_db"
]:
mdfile = rmd.getData(mdtype).location[1]
src = repo_mirror + "/" + mdfile
dst = dest_path + "/" + mdfile
try:
urlgrabber.grabber.urlgrab(src,
filename=dst,
proxies=proxies)
except Exception as e:
utils.die(self.logger,
"failed to fetch " + src + " " + e.args)
# now run createrepo to rebuild the index
if repo.mirror_locally:
os.path.walk(dest_path, self.createrepo_walker, repo)
def yum_sync(self, repo):
"""
Handle copying of http:// and ftp:// yum repos.
"""
repo_mirror = repo.mirror
# warn about not having yum-utils. We don't want to require it in the package because
# RHEL4 and RHEL5U0 don't have it.
if not os.path.exists("/usr/bin/reposync"):
utils.die(self.logger,"no /usr/bin/reposync found, please install yum-utils")
cmd = "" # command to run
has_rpm_list = False # flag indicating not to pull the whole repo
# detect cases that require special handling
if repo.rpm_list != "" and repo.rpm_list != []:
has_rpm_list = True
# create yum config file for use by reposync
dest_path = os.path.join(self.settings.webdir+"/repo_mirror", repo.name)
temp_path = os.path.join(dest_path, ".origin")
if not os.path.isdir(temp_path) and repo.mirror_locally:
# FIXME: there's a chance this might break the RHN D/L case
os.makedirs(temp_path)
# create the config file that yum will use for the copying
if repo.mirror_locally:
temp_file = self.create_local_file(temp_path, repo, output=False)
if not has_rpm_list and repo.mirror_locally:
# if we have not requested only certain RPMs, use reposync
cmd = "/usr/bin/reposync %s --config=%s --repoid=%s --download_path=%s" % (self.rflags, temp_file, repo.name, self.settings.webdir+"/repo_mirror")
if repo.arch != "":
if repo.arch == "x86":
repo.arch = "i386" # FIX potential arch errors
if repo.arch == "i386":
# counter-intuitive, but we want the newish kernels too
cmd = "%s -a i686" % (cmd)
else:
cmd = "%s -a %s" % (cmd, repo.arch)
elif repo.mirror_locally:
# create the output directory if it doesn't exist
if not os.path.exists(dest_path):
os.makedirs(dest_path)
use_source = ""
if repo.arch == "src":
use_source = "--source"
# older yumdownloader sometimes explodes on --resolvedeps
# if this happens to you, upgrade yum & yum-utils
extra_flags = self.settings.yumdownloader_flags
cmd = "/usr/bin/yumdownloader %s %s --disablerepo=* --enablerepo=%s -c %s --destdir=%s %s" % (extra_flags, use_source, repo.name, temp_file, dest_path, " ".join(repo.rpm_list))
# now regardless of whether we're doing yumdownloader or reposync
# or whether the repo was http://, ftp://, or rhn://, execute all queued
# commands here. Any failure at any point stops the operation.
if repo.mirror_locally:
rc = utils.subprocess_call(self.logger, cmd)
if rc !=0:
utils.die(self.logger,"cobbler reposync failed")
repodata_path = os.path.join(dest_path, "repodata")
if not os.path.exists("/usr/bin/wget"):
utils.die(self.logger,"no /usr/bin/wget found, please install wget")
# grab repomd.xml and use it to download any metadata we can use
cmd2 = "/usr/bin/wget -q %s/repodata/repomd.xml -O %s/repomd.xml" % (repo_mirror, temp_path)
rc = utils.subprocess_call(self.logger,cmd2)
if rc == 0:
# create our repodata directory now, as any extra metadata we're
# about to download probably lives there
if not os.path.isdir(repodata_path):
os.makedirs(repodata_path)
rmd = yum.repoMDObject.RepoMD('', "%s/repomd.xml" % (temp_path))
for mdtype in rmd.repoData.keys():
# don't download metadata files that are created by default
if mdtype not in ["primary", "primary_db", "filelists", "filelists_db", "other", "other_db"]:
mdfile = rmd.getData(mdtype).location[1]
cmd3 = "/usr/bin/wget -q %s/%s -O %s/%s" % (repo_mirror, mdfile, dest_path, mdfile)
utils.subprocess_call(self.logger,cmd3)
if rc !=0:
utils.die(self.logger,"wget failed")
# now run createrepo to rebuild the index
if repo.mirror_locally:
os.path.walk(dest_path, self.createrepo_walker, repo)
# create the config file the hosts will use to access the repository.
self.create_local_file(dest_path, repo)
def apt_sync(self, repo):
"""
Handle copying of http:// and ftp:// debian repos.
"""
repo_mirror = repo.mirror
# warn about not having mirror program.
mirror_program = "/usr/bin/debmirror"
if not os.path.exists(mirror_program):
utils.die(self.logger,"no %s found, please install it"%(mirror_program))
cmd = "" # command to run
has_rpm_list = False # flag indicating not to pull the whole repo
# detect cases that require special handling
if repo.rpm_list != "" and repo.rpm_list != []:
utils.die(self.logger,"has_rpm_list not yet supported on apt repos")
if not repo.arch:
utils.die(self.logger,"Architecture is required for apt repositories")
# built destination path for the repo
dest_path = os.path.join("/var/www/cobbler/repo_mirror", repo.name)
if repo.mirror_locally:
# NOTE: Dropping @@suite@@ replace as it is also dropped from
# from manage_import_debian_ubuntu.py due that repo has no os_version
# attribute. If it is added again it will break the Web UI!
#mirror = repo.mirror.replace("@@suite@@",repo.os_version)
mirror = repo.mirror
idx = mirror.find("://")
method = mirror[:idx]
mirror = mirror[idx+3:]
idx = mirror.find("/")
host = mirror[:idx]
mirror = mirror[idx+1:]
idx = mirror.rfind("/dists/")
suite = mirror[idx+7:]
mirror = mirror[:idx]
mirror_data = "--method=%s --host=%s --root=%s --dist=%s " % ( method , host , mirror , suite )
# FIXME : flags should come from repo instead of being hardcoded
rflags = "--passive --nocleanup"
for x in repo.yumopts:
if repo.yumopts[x]:
rflags += " %s %s" % ( x , repo.yumopts[x] )
else:
rflags += " %s" % x
cmd = "%s %s %s %s" % (mirror_program, rflags, mirror_data, dest_path)
if repo.arch == "src":
cmd = "%s --source" % cmd
else:
arch = repo.arch
if arch == "x86":
arch = "i386" # FIX potential arch errors
if arch == "x86_64":
arch = "amd64" # FIX potential arch errors
cmd = "%s --nosource -a %s" % (cmd, arch)
# Set's an environment variable for subprocess, otherwise debmirror will fail
# as it needs this variable to exist.
# FIXME: might this break anything? So far it doesn't
os.putenv("HOME", "/var/lib/cobbler")
rc = utils.subprocess_call(self.logger, cmd)
if rc !=0:
utils.die(self.logger,"cobbler reposync failed")
def yum_process_comps_file(self,comps_path,distro):
"""
When importing Fedora/EL certain parts of the install tree can also be used
as yum repos containing packages that might not yet be available via updates
in yum. This code identifies those areas.
"""
masterdir = "repodata"
if not os.path.exists(os.path.join(comps_path, "repodata")):
# older distros...
masterdir = "base"
# figure out what our comps file is ...
self.logger.info("looking for %(p1)s/%(p2)s/*comps*.xml" % { "p1" : comps_path, "p2" : masterdir })
files = glob.glob("%s/%s/*comps*.xml" % (comps_path, masterdir))
if len(files) == 0:
self.logger.info("no comps found here: %s" % os.path.join(comps_path, masterdir))
return # no comps xml file found
# pull the filename from the longer part
comps_file = files[0].split("/")[-1]
try:
# store the yum configs on the filesystem so we can use them later.
# and configure them in the kickstart post, etc
counter = len(distro.source_repos)
# find path segment for yum_url (changing filesystem path to http:// trailing fragment)
seg = comps_path.rfind("ks_mirror")
urlseg = comps_path[seg+10:]
# write a yum config file that shows how to use the repo.
if counter == 0:
dotrepo = "%s.repo" % distro.name
else:
dotrepo = "%s-%s.repo" % (distro.name, counter)
fname = os.path.join(self.settings.webdir, "ks_mirror", "config", "%s-%s.repo" % (distro.name, counter))
repo_url = "http://@@http_server@@/cobbler/ks_mirror/config/%s-%s.repo" % (distro.name, counter)
repo_url2 = "http://@@http_server@@/cobbler/ks_mirror/%s" % (urlseg)
distro.source_repos.append([repo_url,repo_url2])
# NOTE: the following file is now a Cheetah template, so it can be remapped
# during sync, that's why we have the @@http_server@@ left as templating magic.
# repo_url2 is actually no longer used. (?)
config_file = open(fname, "w+")
config_file.write("[core-%s]\n" % counter)
config_file.write("name=core-%s\n" % counter)
config_file.write("baseurl=http://@@http_server@@/cobbler/ks_mirror/%s\n" % (urlseg))
config_file.write("enabled=1\n")
config_file.write("gpgcheck=0\n")
config_file.write("priority=$yum_distro_priority\n")
config_file.close()
# don't run creatrepo twice -- this can happen easily for Xen and PXE, when
# they'll share same repo files.
if not self.found_repos.has_key(comps_path):
utils.remove_yum_olddata(comps_path)
cmd = "createrepo %s --groupfile %s %s" % (self.settings.createrepo_flags,os.path.join(comps_path, masterdir, comps_file), comps_path)
utils.subprocess_call(self.logger, cmd, shell=True)
self.found_repos[comps_path] = 1
# for older distros, if we have a "base" dir parallel with "repodata", we need to copy comps.xml up one...
p1 = os.path.join(comps_path, "repodata", "comps.xml")
p2 = os.path.join(comps_path, "base", "comps.xml")
if os.path.exists(p1) and os.path.exists(p2):
shutil.copyfile(p1,p2)
except:
self.logger.error("error launching createrepo (not installed?), ignoring")
utils.log_exc(self.logger)
def apt_sync(self, repo):
"""
Handle copying of http:// and ftp:// debian repos.
"""
# warn about not having mirror program.
mirror_program = "/usr/bin/debmirror"
if not os.path.exists(mirror_program):
utils.die(self.logger,
"no %s found, please install it" % (mirror_program))
cmd = "" # command to run
# detect cases that require special handling
if repo.rpm_list != "" and repo.rpm_list != []:
utils.die(self.logger,
"has_rpm_list not yet supported on apt repos")
if not repo.arch:
utils.die(self.logger,
"Architecture is required for apt repositories")
# built destination path for the repo
dest_path = os.path.join("/var/www/cobbler/repo_mirror", repo.name)
if repo.mirror_locally:
# NOTE: Dropping @@suite@@ replace as it is also dropped from
# from manage_import_debian_ubuntu.py due that repo has no os_version
# attribute. If it is added again it will break the Web UI!
# mirror = repo.mirror.replace("@@suite@@",repo.os_version)
mirror = repo.mirror
idx = mirror.find("://")
method = mirror[:idx]
mirror = mirror[idx + 3:]
idx = mirror.find("/")
host = mirror[:idx]
mirror = mirror[idx:]
dists = ",".join(repo.apt_dists)
components = ",".join(repo.apt_components)
mirror_data = "--method=%s --host=%s --root=%s --dist=%s --section=%s" % (
method, host, mirror, dists, components)
rflags = "--nocleanup"
for x in repo.yumopts:
if repo.yumopts[x]:
rflags += " %s %s" % (x, repo.yumopts[x])
else:
rflags += " %s" % x
cmd = "%s %s %s %s" % (mirror_program, rflags, mirror_data,
dest_path)
if repo.arch == "src":
cmd = "%s --source" % cmd
else:
arch = repo.arch
if arch == "x86":
arch = "i386" # FIX potential arch errors
if arch == "x86_64":
arch = "amd64" # FIX potential arch errors
cmd = "%s --nosource -a %s" % (cmd, arch)
# Set's an environment variable for subprocess, otherwise debmirror will fail
# as it needs this variable to exist.
# FIXME: might this break anything? So far it doesn't
os.putenv("HOME", "/var/lib/cobbler")
rc = utils.subprocess_call(self.logger, cmd)
if rc != 0:
utils.die(self.logger, "cobbler reposync failed")
def rsync_it(self,from_path,to_path):
from_path = "%s::%s" % (self.host, from_path)
cmd = "rsync -avzH %s %s" % (from_path, to_path)
rc = utils.subprocess_call(self.logger, cmd, shell=True)
if rc !=0:
self.logger.info("rsync failed")
def rhn_sync(self, repo):
"""
Handle mirroring of RHN repos.
"""
cmd = self.reposync_cmd() # reposync command
has_rpm_list = False # flag indicating not to pull the whole repo
# detect cases that require special handling
if repo.rpm_list != "" and repo.rpm_list != []:
has_rpm_list = True
# create yum config file for use by reposync
# FIXME: don't hardcode
dest_path = os.path.join(self.settings.webdir + "/repo_mirror",
repo.name)
temp_path = os.path.join(dest_path, ".origin")
if not os.path.isdir(temp_path):
# FIXME: there's a chance this might break the RHN D/L case
os.makedirs(temp_path)
# how we invoke reposync depends on whether this is RHN content or not.
# this is the somewhat more-complex RHN case.
# NOTE: this requires that you have entitlements for the server and you give the mirror as rhn://$channelname
if not repo.mirror_locally:
utils.die(self.logger,
"rhn:// repos do not work with --mirror-locally=1")
if has_rpm_list:
self.logger.warning(
"warning: --rpm-list is not supported for RHN content")
rest = repo.mirror[6:] # everything after rhn://
cmd = "%s %s --repo=%s --download_path=%s" % (
cmd, self.rflags, rest, self.settings.webdir + "/repo_mirror")
if repo.name != rest:
args = {"name": repo.name, "rest": rest}
utils.die(
self.logger,
"ERROR: repository %(name)s needs to be renamed %(rest)s as the name of the cobbler repository must match the name of the RHN channel"
% args)
if repo.arch == "i386":
# counter-intuitive, but we want the newish kernels too
repo.arch = "i686"
if repo.arch != "":
cmd = "%s -a %s" % (cmd, repo.arch)
# now regardless of whether we're doing yumdownloader or reposync
# or whether the repo was http://, ftp://, or rhn://, execute all queued
# commands here. Any failure at any point stops the operation.
if repo.mirror_locally:
utils.subprocess_call(self.logger, cmd)
# some more special case handling for RHN.
# create the config file now, because the directory didn't exist earlier
self.create_local_file(temp_path, repo, output=False)
# now run createrepo to rebuild the index
if repo.mirror_locally:
os.path.walk(dest_path, self.createrepo_walker, repo)
# create the config file the hosts will use to access the repository.
self.create_local_file(dest_path, repo)
def generate_standalone_iso(self, imagesdir, isolinuxdir, distname, filesource):
"""
Create bootable CD image to be used for handsoff CD installtions
"""
# Get the distro object for the requested distro
# and then get all of its descendants (profiles/sub-profiles/systems)
distro = self.api.find_distro(distname)
if distro is None:
utils.die(self.logger, "distro %s was not found, aborting" % distname)
descendants = distro.get_descendants()
if filesource is None:
# Try to determine the source from the distro kernel path
self.logger.debug("trying to locate source for distro")
found_source = False
(source_head, source_tail) = os.path.split(distro.kernel)
while source_tail != '':
if source_head == os.path.join(self.api.settings().webdir, "ks_mirror"):
filesource = os.path.join(source_head, source_tail)
found_source = True
self.logger.debug("found source in %s" % filesource)
break
(source_head, source_tail) = os.path.split(source_head)
# Can't find the source, raise an error
if not found_source:
utils.die(self.logger, "Error, no installation source found. When building a standalone ISO, you must specify a --source if the distro install tree is not hosted locally")
self.logger.info("copying kernels and initrds for standalone distro")
self.copy_boot_files(distro, isolinuxdir, None)
cmd = "rsync -rlptgu --exclude=boot.cat --exclude=TRANS.TBL --exclude=isolinux/ %s/ %s/../" % (filesource, isolinuxdir)
self.logger.info("- copying distro %s files (%s)" % (distname, cmd))
rc = utils.subprocess_call(self.logger, cmd, shell=True)
if rc:
utils.die(self.logger, "rsync of files failed")
self.logger.info("generating a isolinux.cfg")
isolinuxcfg = os.path.join(isolinuxdir, "isolinux.cfg")
cfg = open(isolinuxcfg, "w+")
cfg.write(self.iso_template)
for descendant in descendants:
data = utils.blender(self.api, False, descendant)
cfg.write("\n")
cfg.write("LABEL %s\n" % descendant.name)
cfg.write(" MENU LABEL %s\n" % descendant.name)
cfg.write(" kernel %s\n" % os.path.basename(distro.kernel))
append_line = " append initrd=%s" % os.path.basename(distro.initrd)
if distro.breed == "redhat":
append_line += " ks=cdrom:/isolinux/%s.cfg" % descendant.name
if distro.breed == "suse":
append_line += " autoyast=file:///isolinux/%s.cfg install=cdrom:///" % descendant.name
if "install" in data["kernel_options"]:
del data["kernel_options"]["install"]
if distro.breed in ["ubuntu", "debian"]:
append_line += " auto-install/enable=true preseed/file=/cdrom/isolinux/%s.cfg" % descendant.name
# add remaining kernel_options to append_line
append_line += self.add_remaining_kopts(data["kernel_options"])
cfg.write(append_line)
if descendant.COLLECTION_TYPE == 'profile':
kickstart_data = self.api.kickgen.generate_kickstart_for_profile(descendant.name)
elif descendant.COLLECTION_TYPE == 'system':
kickstart_data = self.api.kickgen.generate_kickstart_for_system(descendant.name)
if distro.breed == "redhat":
cdregex = re.compile("url .*\n", re.IGNORECASE)
kickstart_data = cdregex.sub("cdrom\n", kickstart_data)
ks_name = os.path.join(isolinuxdir, "%s.cfg" % descendant.name)
ks_file = open(ks_name, "w+")
ks_file.write(kickstart_data)
ks_file.close()
self.logger.info("done writing config")
cfg.write("\n")
cfg.write("MENU END\n")
cfg.close()
return
def yum_sync(self, repo):
"""
Handle copying of http:// and ftp:// yum repos.
"""
repo_mirror = repo.mirror
# warn about not having yum-utils. We don't want to require it in the package because
# RHEL4 and RHEL5U0 don't have it.
if not os.path.exists("/usr/bin/reposync"):
utils.die(self.logger, "no /usr/bin/reposync found, please install yum-utils")
cmd = "" # command to run
has_rpm_list = False # flag indicating not to pull the whole repo
# detect cases that require special handling
if repo.rpm_list != "" and repo.rpm_list != []:
has_rpm_list = True
# create yum config file for use by reposync
dest_path = os.path.join(self.settings.webdir + "/repo_mirror", repo.name)
temp_path = os.path.join(dest_path, ".origin")
if not os.path.isdir(temp_path) and repo.mirror_locally:
# FIXME: there's a chance this might break the RHN D/L case
os.makedirs(temp_path)
# create the config file that yum will use for the copying
if repo.mirror_locally:
temp_file = self.create_local_file(temp_path, repo, output=False)
if not has_rpm_list and repo.mirror_locally:
# if we have not requested only certain RPMs, use reposync
cmd = "/usr/bin/reposync %s --config=%s --repoid=%s --download_path=%s" % (self.rflags, temp_file, repo.name, self.settings.webdir + "/repo_mirror")
if repo.arch != "":
if repo.arch == "x86":
repo.arch = "i386" # FIX potential arch errors
if repo.arch == "i386":
# counter-intuitive, but we want the newish kernels too
cmd = "%s -a i686" % (cmd)
else:
cmd = "%s -a %s" % (cmd, repo.arch)
elif repo.mirror_locally:
# create the output directory if it doesn't exist
if not os.path.exists(dest_path):
os.makedirs(dest_path)
use_source = ""
if repo.arch == "src":
use_source = "--source"
# older yumdownloader sometimes explodes on --resolvedeps
# if this happens to you, upgrade yum & yum-utils
extra_flags = self.settings.yumdownloader_flags
cmd = "/usr/bin/yumdownloader %s %s --disablerepo=* --enablerepo=%s -c %s --destdir=%s %s" % (extra_flags, use_source, repo.name, temp_file, dest_path, " ".join(repo.rpm_list))
# now regardless of whether we're doing yumdownloader or reposync
# or whether the repo was http://, ftp://, or rhn://, execute all queued
# commands here. Any failure at any point stops the operation.
if repo.mirror_locally:
rc = utils.subprocess_call(self.logger, cmd)
if rc != 0:
utils.die(self.logger, "cobbler reposync failed")
repodata_path = os.path.join(dest_path, "repodata")
if not os.path.exists("/usr/bin/wget"):
utils.die(self.logger, "no /usr/bin/wget found, please install wget")
# grab repomd.xml and use it to download any metadata we can use
cmd2 = "/usr/bin/wget -q %s/repodata/repomd.xml -O %s/repomd.xml" % (repo_mirror, temp_path)
rc = utils.subprocess_call(self.logger, cmd2)
if rc == 0:
# create our repodata directory now, as any extra metadata we're
# about to download probably lives there
if not os.path.isdir(repodata_path):
os.makedirs(repodata_path)
rmd = yum.repoMDObject.RepoMD('', "%s/repomd.xml" % (temp_path))
for mdtype in rmd.repoData.keys():
# don't download metadata files that are created by default
if mdtype not in ["primary", "primary_db", "filelists", "filelists_db", "other", "other_db"]:
mdfile = rmd.getData(mdtype).location[1]
cmd3 = "/usr/bin/wget -q %s/%s -O %s/%s" % (repo_mirror, mdfile, dest_path, mdfile)
utils.subprocess_call(self.logger, cmd3)
if rc != 0:
utils.die(self.logger, "wget failed")
# now run createrepo to rebuild the index
if repo.mirror_locally:
os.path.walk(dest_path, self.createrepo_walker, repo)
# create the config file the hosts will use to access the repository.
self.create_local_file(dest_path, repo)
def generate_standalone_iso(self, imagesdir, isolinuxdir, distname,
filesource):
# Get the distro object for the requested distro
# and then get all of its descendants (profiles/sub-profiles/systems)
distro = self.api.find_distro(distname)
if distro is None:
utils.die(self.logger,
"distro %s was not found, aborting" % distname)
descendants = distro.get_descendants()
if filesource is None:
# Try to determine the source from the distro kernel path
self.logger.debug("trying to locate source for distro")
found_source = False
(source_head, source_tail) = os.path.split(distro.kernel)
while source_tail != '':
if source_head == os.path.join(self.api.settings().webdir,
"ks_mirror"):
filesource = os.path.join(source_head, source_tail)
found_source = True
self.logger.debug("found source in %s" % filesource)
break
(source_head, source_tail) = os.path.split(source_head)
# Can't find the source, raise an error
if not found_source:
utils.die(
self.logger,
" Error, no installation source found. When building a standalone ISO, you must specify a --source if the distro install tree is not hosted locally"
)
self.logger.info("copying kernels and initrds for standalone distro")
# tempdir/isolinux/$distro/vmlinuz, initrd.img
# FIXME: this will likely crash on non-Linux breeds
f1 = os.path.join(isolinuxdir, "vmlinuz")
f2 = os.path.join(isolinuxdir, "initrd.img")
if not os.path.exists(distro.kernel):
utils.die(self.logger, "path does not exist: %s" % distro.kernel)
if not os.path.exists(distro.initrd):
utils.die(self.logger, "path does not exist: %s" % distro.initrd)
shutil.copyfile(distro.kernel, f1)
shutil.copyfile(distro.initrd, f2)
cmd = "rsync -rlptgu --exclude=boot.cat --exclude=TRANS.TBL --exclude=isolinux/ %s/ %s/../" % (
filesource, isolinuxdir)
self.logger.info("- copying distro %s files (%s)" % (distname, cmd))
rc = utils.subprocess_call(self.logger, cmd, shell=True)
if rc:
utils.die(self.logger, "rsync of files failed")
self.logger.info("generating a isolinux.cfg")
isolinuxcfg = os.path.join(isolinuxdir, "isolinux.cfg")
cfg = open(isolinuxcfg, "w+")
cfg.write(HEADER) # fixme, use template
for descendant in descendants:
data = utils.blender(self.api, True, descendant)
cfg.write("\n")
cfg.write("LABEL %s\n" % descendant.name)
cfg.write(" MENU LABEL %s\n" % descendant.name)
cfg.write(" kernel vmlinuz\n")
data["kickstart"] = "cdrom:/isolinux/ks-%s.cfg" % descendant.name
append_line = " append initrd=initrd.img"
append_line = append_line + " ks=%s " % data["kickstart"]
append_line = append_line + " %s\n" % data["kernel_options"]
cfg.write(append_line)
if descendant.COLLECTION_TYPE == 'profile':
kickstart_data = self.api.kickgen.generate_kickstart_for_profile(
descendant.name)
elif descendant.COLLECTION_TYPE == 'system':
kickstart_data = self.api.kickgen.generate_kickstart_for_system(
descendant.name)
cdregex = re.compile("url .*\n", re.IGNORECASE)
kickstart_data = cdregex.sub("cdrom\n", kickstart_data)
ks_name = os.path.join(isolinuxdir, "ks-%s.cfg" % descendant.name)
ks_file = open(ks_name, "w+")
ks_file.write(kickstart_data)
ks_file.close()
self.logger.info("done writing config")
cfg.write("\n")
cfg.write("MENU END\n")
cfg.close()
return
def rsync_it(self,from_path,to_path):
from_path = "%s::%s" % (self.host, from_path)
cmd = "rsync -avzH %s %s" % (from_path, to_path)
rc = utils.subprocess_call(self.logger, cmd, shell=True)
if rc !=0:
self.logger.info("rsync failed")
def apt_sync(self, repo):
"""
Handle copying of http:// and ftp:// debian repos.
"""
repo_mirror = repo.mirror
# warn about not having mirror program.
mirror_program = "/usr/bin/debmirror"
if not os.path.exists(mirror_program):
utils.die(self.logger,
"no %s found, please install it" % (mirror_program))
cmd = "" # command to run
has_rpm_list = False # flag indicating not to pull the whole repo
# detect cases that require special handling
if repo.rpm_list != "" and repo.rpm_list != []:
utils.die(self.logger,
"has_rpm_list not yet supported on apt repos")
if not repo.arch:
utils.die(self.logger,
"Architecture is required for apt repositories")
# built destination path for the repo
dest_path = os.path.join("/var/www/cobbler/repo_mirror", repo.name)
if repo.mirror_locally:
mirror = repo.mirror.replace("@@suite@@", repo.os_version)
idx = mirror.find("://")
method = mirror[:idx]
mirror = mirror[idx + 3:]
idx = mirror.find("/")
host = mirror[:idx]
mirror = mirror[idx + 1:]
idx = mirror.rfind("/dists/")
suite = mirror[idx + 7:]
mirror = mirror[:idx]
mirror_data = "--method=%s --host=%s --root=%s --dist=%s " % (
method, host, mirror, suite)
# FIXME : flags should come from repo instead of being hardcoded
rflags = "--passive --nocleanup"
for x in repo.yumopts:
if repo.yumopts[x]:
rflags += " %s %s" % (x, repo.yumopts[x])
else:
rflags += " %s" % x
cmd = "%s %s %s %s" % (mirror_program, rflags, mirror_data,
dest_path)
if repo.arch == "src":
cmd = "%s --source" % cmd
else:
arch = repo.arch
if arch == "x86":
arch = "i386" # FIX potential arch errors
if arch == "x86_64":
arch = "amd64" # FIX potential arch errors
cmd = "%s --nosource -a %s" % (cmd, arch)
rc = utils.subprocess_call(self.logger, cmd)
if rc != 0:
utils.die(self.logger, "cobbler reposync failed")
