def import_tree(self,mirror_url,mirror_name,network_root=None,kickstart_file=None,rsync_flags=None,arch=None,breed=None,os_version=None,logger=None):
"""
Automatically import a directory tree full of distribution files.
mirror_url can be a string that represents a path, a user@host
syntax for SSH, or an rsync:// address. If mirror_url is a
filesystem path and mirroring is not desired, set network_root
to something like "nfs://path/to/mirror_url/root"
"""
self.log("import_tree",[mirror_url, mirror_name, network_root, kickstart_file, rsync_flags])
# both --path and --name are required arguments
if mirror_url is None:
self.log("import failed. no --path specified")
return False
if mirror_name is None:
self.log("import failed. no --name specified")
return False
path = os.path.normpath("%s/ks_mirror/%s" % (self.settings().webdir, mirror_name))
if arch is not None:
arch = arch.lower()
if arch == "x86":
# be consistent
arch = "i386"
if path.split("-")[-1] != arch:
path += ("-%s" % arch)
# we need to mirror (copy) the files
self.log("importing from a network location, running rsync to fetch the files first")
utils.mkdir(path)
# prevent rsync from creating the directory name twice
# if we are copying via rsync
if not mirror_url.endswith("/"):
mirror_url = "%s/" % mirror_url
if mirror_url.startswith("http://") or mirror_url.startswith("ftp://") or mirror_url.startswith("nfs://"):
# http mirrors are kind of primative. rsync is better.
# that's why this isn't documented in the manpage and we don't support them.
# TODO: how about adding recursive FTP as an option?
self.log("unsupported protocol")
return False
else:
# good, we're going to use rsync..
# we don't use SSH for public mirrors and local files.
# presence of user@host syntax means use SSH
spacer = ""
if not mirror_url.startswith("rsync://") and not mirror_url.startswith("/"):
spacer = ' -e "ssh" '
rsync_cmd = RSYNC_CMD
if rsync_flags:
rsync_cmd = rsync_cmd + " " + rsync_flags
# if --available-as was specified, limit the files we
# pull down via rsync to just those that are critical
# to detecting what the distro is
if network_root is not None:
rsync_cmd = rsync_cmd + " --include-from=/etc/cobbler/import_rsync_whitelist"
# kick off the rsync now
utils.run_this(rsync_cmd, (spacer, mirror_url, path), self.logger)
if network_root is not None:
# in addition to mirroring, we're going to assume the path is available
# over http, ftp, and nfs, perhaps on an external filer. scanning still requires
# --mirror is a filesystem path, but --available-as marks the network path.
# this allows users to point the path at a directory containing just the network
# boot files while the rest of the distro files are available somewhere else.
# find the filesystem part of the path, after the server bits, as each distro
# URL needs to be calculated relative to this.
if not network_root.endswith("/"):
network_root = network_root + "/"
valid_roots = [ "nfs://", "ftp://", "http://" ]
for valid_root in valid_roots:
if network_root.startswith(valid_root):
break
else:
self.log("Network root given to --available-as must be nfs://, ftp://, or http://")
return False
if network_root.startswith("nfs://"):
try:
(a,b,rest) = network_root.split(":",3)
except:
self.log("Network root given to --available-as is missing a colon, please see the manpage example.")
return False
#importer_modules = self.get_modules_in_category("manage/import")
#for importer_module in importer_modules:
# manager = importer_module.get_import_manager(self._config,logger)
# try:
# (found,pkgdir) = manager.check_for_signature(path,breed)
# if found:
# self.log("running import manager: %s" % manager.what())
# return manager.run(pkgdir,mirror_name,path,network_root,kickstart_file,rsync_flags,arch,breed,os_version)
# except:
# self.log("an exception occured while running the import manager")
# self.log("error was: %s" % sys.exc_info()[1])
# continue
#self.log("No import managers found a valid signature at the location specified")
## FIXME: since we failed, we should probably remove the
## path tree we created above so we don't leave cruft around
#return False
import_module = self.get_module_by_name("manage_import_signatures").get_import_manager(self._config,logger)
return import_module.run(path,mirror_name,network_root,kickstart_file,arch,breed,os_version)
def run(self,pkgdir,mirror,mirror_name,network_root=None,kickstart_file=None,rsync_flags=None,arch=None,breed=None,os_version=None):
self.pkgdir = pkgdir
self.mirror = mirror
self.mirror_name = mirror_name
self.network_root = network_root
self.kickstart_file = kickstart_file
self.rsync_flags = rsync_flags
self.arch = arch
self.breed = breed
self.os_version = os_version
# some fixups for the XMLRPC interface, which does not use "None"
if self.arch == "": self.arch = None
if self.mirror == "": self.mirror = None
if self.mirror_name == "": self.mirror_name = None
if self.kickstart_file == "": self.kickstart_file = None
if self.os_version == "": self.os_version = None
if self.rsync_flags == "": self.rsync_flags = None
if self.network_root == "": self.network_root = None
# If no breed was specified on the command line, set it to "freebsd" for this module
if self.breed == None:
self.breed = "freebsd"
# debug log stuff for testing
#self.logger.info("self.pkgdir = %s" % str(self.pkgdir))
#self.logger.info("self.mirror = %s" % str(self.mirror))
#self.logger.info("self.mirror_name = %s" % str(self.mirror_name))
#self.logger.info("self.network_root = %s" % str(self.network_root))
#self.logger.info("self.kickstart_file = %s" % str(self.kickstart_file))
#self.logger.info("self.rsync_flags = %s" % str(self.rsync_flags))
#self.logger.info("self.arch = %s" % str(self.arch))
#self.logger.info("self.breed = %s" % str(self.breed))
#self.logger.info("self.os_version = %s" % str(self.os_version))
# both --import and --name are required arguments
if self.mirror is None:
utils.die(self.logger,"import failed. no --path specified")
if self.mirror_name is None:
utils.die(self.logger,"import failed. no --name specified")
# if --arch is supplied, validate it to ensure it's valid
if self.arch is not None and self.arch != "":
self.arch = self.arch.lower()
if self.arch == "x86":
# be consistent
self.arch = "i386"
if self.arch not in self.get_valid_arches():
utils.die(self.logger,"arch must be one of: %s" % string.join(self.get_valid_arches(),", "))
# if we're going to do any copying, set where to put things
# and then make sure nothing is already there.
self.path = os.path.normpath( "%s/ks_mirror/%s" % (self.settings.webdir, self.mirror_name) )
self.rootdir = os.path.normpath( "%s/ks_mirror/%s" % (self.settings.webdir, self.mirror_name) )
if os.path.exists(self.path) and self.arch is None:
# FIXME : Raise exception even when network_root is given ?
utils.die(self.logger,"Something already exists at this import location (%s). You must specify --arch to avoid potentially overwriting existing files." % self.path)
# import takes a --kickstart for forcing selection that can't be used in all circumstances
if self.kickstart_file and not self.breed:
utils.die(self.logger,"Kickstart file can only be specified when a specific breed is selected")
if self.os_version and not self.breed:
utils.die(self.logger,"OS version can only be specified when a specific breed is selected")
if self.breed and self.breed.lower() not in self.get_valid_breeds():
utils.die(self.logger,"Supplied import breed is not supported by this module")
# if --arch is supplied, make sure the user is not importing a path with a different
# arch, which would just be silly.
if self.arch:
# append the arch path to the name if the arch is not already
# found in the name.
for x in self.get_valid_arches():
if self.path.lower().find(x) != -1:
if self.arch != x :
utils.die(self.logger,"Architecture found on pathname (%s) does not fit the one given in command line (%s)"%(x,self.arch))
break
else:
# FIXME : This is very likely removed later at get_proposed_name, and the guessed arch appended again
self.path += ("-%s" % self.arch)
# make the output path and mirror content but only if not specifying that a network
# accessible support location already exists (this is --available-as on the command line)
if self.network_root is None:
# we need to mirror (copy) the files
utils.mkdir(self.path)
# prevent rsync from creating the directory name twice
# if we are copying via rsync
if not self.mirror.endswith("/"):
self.mirror = "%s/" % self.mirror
if self.mirror.startswith("http://") or self.mirror.startswith("ftp://") or self.mirror.startswith("nfs://"):
# http mirrors are kind of primative. rsync is better.
# that's why this isn't documented in the manpage and we don't support them.
# TODO: how about adding recursive FTP as an option?
utils.die(self.logger,"unsupported protocol")
else:
# good, we're going to use rsync..
# we don't use SSH for public mirrors and local files.
# presence of user@host syntax means use SSH
spacer = ""
if not self.mirror.startswith("rsync://") and not self.mirror.startswith("/"):
spacer = ' -e "ssh" '
rsync_cmd = RSYNC_CMD
if self.rsync_flags:
rsync_cmd = rsync_cmd + " " + self.rsync_flags
# kick off the rsync now
utils.run_this(rsync_cmd, (spacer, self.mirror, self.path), self.logger)
else:
# rather than mirroring, we're going to assume the path is available
# over http, ftp, and nfs, perhaps on an external filer. scanning still requires
# --mirror is a filesystem path, but --available-as marks the network path
if not os.path.exists(self.mirror):
utils.die(self.logger, "path does not exist: %s" % self.mirror)
# find the filesystem part of the path, after the server bits, as each distro
# URL needs to be calculated relative to this.
if not self.network_root.endswith("/"):
self.network_root = self.network_root + "/"
self.path = os.path.normpath( self.mirror )
valid_roots = [ "nfs://", "ftp://", "http://" ]
for valid_root in valid_roots:
if self.network_root.startswith(valid_root):
break
else:
utils.die(self.logger, "Network root given to --available-as must be nfs://, ftp://, or http://")
if self.network_root.startswith("nfs://"):
try:
(a,b,rest) = self.network_root.split(":",3)
except:
utils.die(self.logger, "Network root given to --available-as is missing a colon, please see the manpage example.")
# now walk the filesystem looking for distributions that match certain patterns
self.logger.info("adding distros")
distros_added = []
# FIXME : search below self.path for isolinux configurations or known directories from TRY_LIST
os.path.walk(self.path, self.distro_adder, distros_added)
# find out if we can auto-create any repository records from the install tree
if self.network_root is None:
self.logger.info("associating repos")
# FIXME: this automagic is not possible (yet) without mirroring
self.repo_finder(distros_added)
# find the most appropriate answer files for each profile object
self.logger.info("associating answerfiles")
self.kickstart_finder(distros_added)
# ensure bootloaders are present
self.api.pxegen.copy_bootloaders()
return True
def import_tree(self,mirror_url,mirror_name,network_root=None,kickstart_file=None,rsync_flags=None,arch=None,breed=None,os_version=None,logger=None):
"""
Automatically import a directory tree full of distribution files.
mirror_url can be a string that represents a path, a user@host
syntax for SSH, or an rsync:// address. If mirror_url is a
filesystem path and mirroring is not desired, set network_root
to something like "nfs://path/to/mirror_url/root"
"""
self.log("import_tree",[mirror_url, mirror_name, network_root, kickstart_file, rsync_flags])
# both --path and --name are required arguments
if mirror_url is None:
self.log("import failed. no --path specified")
return False
if mirror_name is None:
self.log("import failed. no --name specified")
return False
path = os.path.normpath("%s/ks_mirror/%s" % (self.settings().webdir, mirror_name))
if arch is not None:
arch = arch.lower()
if arch == "x86":
# be consistent
arch = "i386"
path += ("-%s" % arch)
# we need to mirror (copy) the files
self.log("importing from a network location, running rsync to fetch the files first")
utils.mkdir(path)
# prevent rsync from creating the directory name twice
# if we are copying via rsync
if not mirror_url.endswith("/"):
mirror_url = "%s/" % mirror_url
if mirror_url.startswith("http://") or mirror_url.startswith("ftp://") or mirror_url.startswith("nfs://"):
# http mirrors are kind of primative. rsync is better.
# that's why this isn't documented in the manpage and we don't support them.
# TODO: how about adding recursive FTP as an option?
self.log("unsupported protocol")
return False
else:
# good, we're going to use rsync..
# we don't use SSH for public mirrors and local files.
# presence of user@host syntax means use SSH
spacer = ""
if not mirror_url.startswith("rsync://") and not mirror_url.startswith("/"):
spacer = ' -e "ssh" '
rsync_cmd = RSYNC_CMD
if rsync_flags:
rsync_cmd = rsync_cmd + " " + rsync_flags
# if --available-as was specified, limit the files we
# pull down via rsync to just those that are critical
# to detecting what the distro is
if network_root is not None:
rsync_cmd = rsync_cmd + " --include-from=/etc/cobbler/import_rsync_whitelist"
# kick off the rsync now
utils.run_this(rsync_cmd, (spacer, mirror_url, path), self.logger)
if network_root is not None:
# in addition to mirroring, we're going to assume the path is available
# over http, ftp, and nfs, perhaps on an external filer. scanning still requires
# --mirror is a filesystem path, but --available-as marks the network path.
# this allows users to point the path at a directory containing just the network
# boot files while the rest of the distro files are available somewhere else.
# find the filesystem part of the path, after the server bits, as each distro
# URL needs to be calculated relative to this.
if not network_root.endswith("/"):
network_root = network_root + "/"
valid_roots = [ "nfs://", "ftp://", "http://" ]
for valid_root in valid_roots:
if network_root.startswith(valid_root):
break
else:
self.log("Network root given to --available-as must be nfs://, ftp://, or http://")
return False
if network_root.startswith("nfs://"):
try:
(a,b,rest) = network_root.split(":",3)
except:
self.log("Network root given to --available-as is missing a colon, please see the manpage example.")
return False
#importer_modules = self.get_modules_in_category("manage/import")
#for importer_module in importer_modules:
# manager = importer_module.get_import_manager(self._config,logger)
# try:
# (found,pkgdir) = manager.check_for_signature(path,breed)
# if found:
# self.log("running import manager: %s" % manager.what())
# return manager.run(pkgdir,mirror_name,path,network_root,kickstart_file,rsync_flags,arch,breed,os_version)
# except:
# self.log("an exception occured while running the import manager")
# self.log("error was: %s" % sys.exc_info()[1])
# continue
#self.log("No import managers found a valid signature at the location specified")
## FIXME: since we failed, we should probably remove the
## path tree we created above so we don't leave cruft around
#return False
import_module = self.get_module_by_name("manage_import_signatures").get_import_manager(self._config,logger)
return import_module.run(path,mirror_name,network_root,kickstart_file,arch,breed,os_version)
def run(self,pkgdir,mirror,mirror_name,network_root=None,kickstart_file=None,rsync_flags=None,arch=None,breed=None,os_version=None):
self.pkgdir = pkgdir
self.mirror = mirror
self.mirror_name = mirror_name
self.network_root = network_root
self.kickstart_file = kickstart_file
self.rsync_flags = rsync_flags
self.arch = arch
self.breed = breed
self.os_version = os_version
# some fixups for the XMLRPC interface, which does not use "None"
if self.arch == "": self.arch = None
if self.mirror == "": self.mirror = None
if self.mirror_name == "": self.mirror_name = None
if self.kickstart_file == "": self.kickstart_file = None
if self.os_version == "": self.os_version = None
if self.rsync_flags == "": self.rsync_flags = None
if self.network_root == "": self.network_root = None
# If no breed was specified on the command line, set it to "redhat" for this module
if self.breed == None:
self.breed = "redhat"
# debug log stuff for testing
#self.logger.info("self.pkgdir = %s" % str(self.pkgdir))
#self.logger.info("self.mirror = %s" % str(self.mirror))
#self.logger.info("self.mirror_name = %s" % str(self.mirror_name))
#self.logger.info("self.network_root = %s" % str(self.network_root))
#self.logger.info("self.kickstart_file = %s" % str(self.kickstart_file))
#self.logger.info("self.rsync_flags = %s" % str(self.rsync_flags))
#self.logger.info("self.arch = %s" % str(self.arch))
#self.logger.info("self.breed = %s" % str(self.breed))
#self.logger.info("self.os_version = %s" % str(self.os_version))
# both --import and --name are required arguments
if self.mirror is None:
utils.die(self.logger,"import failed. no --path specified")
if self.mirror_name is None:
utils.die(self.logger,"import failed. no --name specified")
# if --arch is supplied, validate it to ensure it's valid
if self.arch is not None and self.arch != "":
self.arch = self.arch.lower()
if self.arch == "x86":
# be consistent
self.arch = "i386"
if self.arch not in self.get_valid_arches():
utils.die(self.logger,"arch must be one of: %s" % string.join(self.get_valid_arches(),", "))
# if we're going to do any copying, set where to put things
# and then make sure nothing is already there.
self.path = os.path.normpath( "%s/ks_mirror/%s" % (self.settings.webdir, self.mirror_name) )
if os.path.exists(self.path) and self.arch is None:
# FIXME : Raise exception even when network_root is given ?
utils.die(self.logger,"Something already exists at this import location (%s). You must specify --arch to avoid potentially overwriting existing files." % self.path)
# import takes a --kickstart for forcing selection that can't be used in all circumstances
if self.kickstart_file and not self.breed:
utils.die(self.logger,"Kickstart file can only be specified when a specific breed is selected")
if self.os_version and not self.breed:
utils.die(self.logger,"OS version can only be specified when a specific breed is selected")
if self.breed and self.breed.lower() not in self.get_valid_breeds():
utils.die(self.logger,"Supplied import breed is not supported by this module")
# if --arch is supplied, make sure the user is not importing a path with a different
# arch, which would just be silly.
if self.arch:
# append the arch path to the name if the arch is not already
# found in the name.
for x in self.get_valid_arches():
if self.path.lower().find(x) != -1:
if self.arch != x :
utils.die(self.logger,"Architecture found on pathname (%s) does not fit the one given in command line (%s)"%(x,self.arch))
break
else:
# FIXME : This is very likely removed later at get_proposed_name, and the guessed arch appended again
self.path += ("-%s" % self.arch)
# make the output path and mirror content but only if not specifying that a network
# accessible support location already exists (this is --available-as on the command line)
if self.network_root is None:
# we need to mirror (copy) the files
utils.mkdir(self.path)
# prevent rsync from creating the directory name twice
# if we are copying via rsync
if not self.mirror.endswith("/"):
self.mirror = "%s/" % self.mirror
if self.mirror.startswith("http://") or self.mirror.startswith("ftp://") or self.mirror.startswith("nfs://"):
# http mirrors are kind of primative. rsync is better.
# that's why this isn't documented in the manpage and we don't support them.
# TODO: how about adding recursive FTP as an option?
utils.die(self.logger,"unsupported protocol")
else:
# good, we're going to use rsync..
# we don't use SSH for public mirrors and local files.
# presence of user@host syntax means use SSH
spacer = ""
if not self.mirror.startswith("rsync://") and not self.mirror.startswith("/"):
spacer = ' -e "ssh" '
rsync_cmd = RSYNC_CMD
if self.rsync_flags:
rsync_cmd = rsync_cmd + " " + self.rsync_flags
# kick off the rsync now
utils.run_this(rsync_cmd, (spacer, self.mirror, self.path), self.logger)
else:
# rather than mirroring, we're going to assume the path is available
# over http, ftp, and nfs, perhaps on an external filer. scanning still requires
# --mirror is a filesystem path, but --available-as marks the network path
if not os.path.exists(self.mirror):
utils.die(self.logger, "path does not exist: %s" % self.mirror)
# find the filesystem part of the path, after the server bits, as each distro
# URL needs to be calculated relative to this.
if not self.network_root.endswith("/"):
self.network_root = self.network_root + "/"
self.path = os.path.normpath( self.mirror )
valid_roots = [ "nfs://", "ftp://", "http://" ]
for valid_root in valid_roots:
if self.network_root.startswith(valid_root):
break
else:
utils.die(self.logger, "Network root given to --available-as must be nfs://, ftp://, or http://")
if self.network_root.startswith("nfs://"):
try:
(a,b,rest) = self.network_root.split(":",3)
except:
utils.die(self.logger, "Network root given to --available-as is missing a colon, please see the manpage example.")
# now walk the filesystem looking for distributions that match certain patterns
self.logger.info("adding distros")
distros_added = []
# FIXME : search below self.path for isolinux configurations or known directories from TRY_LIST
os.path.walk(self.path, self.distro_adder, distros_added)
# find out if we can auto-create any repository records from the install tree
if self.network_root is None:
self.logger.info("associating repos")
# FIXME: this automagic is not possible (yet) without mirroring
self.repo_finder(distros_added)
# find the most appropriate answer files for each profile object
self.logger.info("associating kickstarts")
self.kickstart_finder(distros_added)
# ensure bootloaders are present
self.api.pxegen.copy_bootloaders()
return True
