def copy_files_to_image(self): # create image myfiles = [] if "netboot/packages" in self.settings: if isinstance(self.settings['netboot/packages'], str): loopy = [self.settings["netboot/packages"]] else: loopy = self.settings["netboot/packages"] for x in loopy: if "netboot/packages/" + x + "/files" in self.settings: if isinstance( type(self.settings['netboot/packages/' + x + '/files']), str): myfiles.extend(self.settings["netboot/packages/" + x + "/files"]) else: myfiles.append(self.settings["netboot/packages/" + x + "/files"]) if "netboot/extra_files" in self.settings: if isinstance(self.settings['netboot/extra_files'], list): myfiles.extend(self.settings["netboot/extra_files"]) else: myfiles.append(self.settings["netboot/extra_files"]) try: cmd([self.settings['controller_file'], 'image'] + myfiles, env=self.env) except CatalystError: self.unbind() raise CatalystError("netboot build aborting due to error.", print_traceback=True)
def __init__(self,spec,addlargs): self.required_values=[ "boot/kernel" ] self.valid_values=self.required_values[:] self.valid_values.extend([ "netboot2/packages", "netboot2/use", "netboot2/extra_files", "netboot2/overlay", "netboot2/busybox_config", "netboot2/root_overlay", "netboot2/linuxrc" ]) try: if "netboot2/packages" in addlargs: if isinstance(addlargs['netboot2/packages'], str): loopy=[addlargs["netboot2/packages"]] else: loopy=addlargs["netboot2/packages"] for x in loopy: self.valid_values.append("netboot2/packages/"+x+"/files") except: raise CatalystError("configuration error in netboot2/packages.") StageBase.__init__(self,spec,addlargs) self.settings["merge_path"]=normpath("/tmp/image/")
def ensure_dirs(path, gid=-1, uid=-1, mode=0o755, minimal=True, failback=None, fatal=False): '''Wrapper to snakeoil.osutil's ensure_dirs() This additionally allows for failures to run cleanup or other code and/or raise fatal errors. :param path: directory to ensure exists on disk :param gid: a valid GID to set any created directories to :param uid: a valid UID to set any created directories to :param mode: permissions to set any created directories to :param minimal: boolean controlling whether or not the specified mode must be enforced, or is the minimal permissions necessary. For example, if mode=0o755, minimal=True, and a directory exists with mode 0707, this will restore the missing group perms resulting in 757. :param failback: function to run in the event of a failed attemp to create the directory. :return: True if the directory could be created/ensured to have those permissions, False if not. ''' succeeded = snakeoil_ensure_dirs( path, gid=gid, uid=uid, mode=mode, minimal=minimal) if not succeeded: if failback: failback() if fatal: raise CatalystError( "Failed to create directory: %s" % path, print_traceback=True) return succeeded
def __init__(self, spec, addlargs): self.valid_values = [ "netboot/kernel/sources", "netboot/kernel/config", "netboot/kernel/prebuilt", "netboot/busybox_config", "netboot/extra_files", "netboot/packages" ] self.required_values = [] try: if "netboot/packages" in addlargs: if type(addlargs["netboot/packages"]) == types.StringType: loopy = [addlargs["netboot/packages"]] else: loopy = addlargs["netboot/packages"] # for x in loopy: # self.required_values.append("netboot/packages/"+x+"/files") except: raise CatalystError("configuration error in netboot/packages.") StageBase.__init__(self, spec, addlargs) self.set_build_kernel_vars(addlargs) if "netboot/busybox_config" in addlargs: file_locate(self.settings, ["netboot/busybox_config"]) # Custom Kernel Tarball --- use that instead ... # unless the user wants specific CFLAGS/CXXFLAGS, let's use -Os for envvar in "CFLAGS", "CXXFLAGS": if envvar not in os.environ and envvar not in addlargs: self.settings[envvar] = "-Os -pipe"
def copy_files_to_image(self): # create image myfiles = [] if "netboot/packages" in self.settings: if type(self.settings["netboot/packages"]) == types.StringType: loopy = [self.settings["netboot/packages"]] else: loopy = self.settings["netboot/packages"] for x in loopy: if "netboot/packages/" + x + "/files" in self.settings: if type(self.settings["netboot/packages/" + x + "/files"]) == types.ListType: myfiles.extend(self.settings["netboot/packages/" + x + "/files"]) else: myfiles.append(self.settings["netboot/packages/" + x + "/files"]) if "netboot/extra_files" in self.settings: if type(self.settings["netboot/extra_files"]) == types.ListType: myfiles.extend(self.settings["netboot/extra_files"]) else: myfiles.append(self.settings["netboot/extra_files"]) try: cmd(self.settings["controller_file"]+\ " image " + list_bashify(myfiles),env=self.env) except CatalystError: self.unbind() raise CatalystError("netboot build aborting due to error.", print_traceback=True)
def __init__(self, spec, addlargs): self.valid_values = [ "netboot/kernel/sources", "netboot/kernel/config", "netboot/kernel/prebuilt", "netboot/busybox_config", "netboot/extra_files", "netboot/packages" ] self.required_values = [] try: # XXX: This code does nothing because the for loop below is disabled. if "netboot/packages" in addlargs: if isinstance(addlargs['netboot/packages'], str): _loopy = [addlargs["netboot/packages"]] else: _loopy = addlargs["netboot/packages"] # for x in loopy: # self.required_values.append("netboot/packages/"+x+"/files") except: raise CatalystError("configuration error in netboot/packages.") StageBase.__init__(self, spec, addlargs) if "netboot/busybox_config" in addlargs: file_locate(self.settings, ["netboot/busybox_config"]) # Custom Kernel Tarball --- use that instead ... # unless the user wants specific CFLAGS/CXXFLAGS, let's use -Os for envvar in "CFLAGS", "CXXFLAGS": if envvar not in os.environ and envvar not in addlargs: self.settings[envvar] = "-Os -pipe"
def update_ebuild_repo(self) -> str: repouri = 'https://anongit.gentoo.org/git/repo/sync/gentoo.git' if self.ebuild_repo.is_dir(): git_cmds = [ [self.git, '-C', self.gitdir, 'fetch', '--quiet', '--depth=1'], [ self.git, '-C', self.gitdir, 'update-ref', 'HEAD', 'FETCH_HEAD' ], [self.git, '-C', self.gitdir, 'gc', '--quiet'], ] else: git_cmds = [ [ self.git, 'clone', '--quiet', '--depth=1', '--bare', # Set some config options to enable git gc to clean everything # except what we just fetched. See git-gc(1). '-c', 'gc.reflogExpire=0', '-c', 'gc.reflogExpireUnreachable=0', '-c', 'gc.rerereresolved=0', '-c', 'gc.rerereunresolved=0', '-c', 'gc.pruneExpire=now', '--branch=stable', repouri, self.gitdir ], ] try: for cmd in git_cmds: log.notice('>>> ' + ' '.join(cmd)) subprocess.run(cmd, capture_output=True, check=True, encoding='utf-8', close_fds=False) sp = subprocess.run( [self.git, '-C', self.gitdir, 'rev-parse', 'stable'], stdout=subprocess.PIPE, capture_output=True, check=True, encoding='utf-8', close_fds=False) return sp.stdout.rstrip() except subprocess.CalledProcessError as e: raise CatalystError(f'{e.cmd} failed with return code' f'{e.returncode}\n' f'{e.output}\n')
def parse_file(self, filename): try: with open(filename, "r") as myf: self.lines = myf.readlines() except: raise CatalystError("Could not open file " + filename, print_traceback=True) self.filename = filename self.parse()
def __init__(self,myspec): builder.generic.__init__(self,myspec) if self.settings["buildarch"]=="sparc64": if not os.path.exists("/bin/linux32") and not os.path.exists("/usr/bin/linux32"): raise CatalystError("required executable linux32 not found " "(\"emerge setarch\" to fix.)", print_traceback=True) self.settings["CHROOT"]="linux32 chroot" self.settings["crosscompile"] = False; else: self.settings["CHROOT"]="chroot"
def create_netboot_files(self): # finish it all up try: cmd(self.settings["controller_file"] + " finish", env=self.env) except CatalystError: self.unbind() raise CatalystError("netboot build aborting due to error.", print_traceback=True) # end print "netboot: build finished !"
def create_netboot_files(self): # finish it all up try: cmd([self.settings['controller_file'], 'finish'], env=self.env) except CatalystError: self.unbind() raise CatalystError("netboot build aborting due to error.", print_traceback=True) # end log.notice('netboot: build finished !')
def build_target(addlargs): try: target = addlargs["target"].replace('-', '_') module = import_module(target) target = getattr(module, target)(conf_values, addlargs) except AttributeError as e: raise CatalystError("Target \"%s\" not available." % target, print_traceback=True) from e except CatalystError: return False return target.run()
def move_kernels(self): # we're done, move the kernels to builds/* # no auto resume here as we always want the # freshest images moved try: cmd([self.settings['controller_file'], 'final'], env=self.env) log.notice('Netboot Build Finished!') except CatalystError: self.unbind() raise CatalystError("Failed to move kernel images!", print_traceback=True)
def build_busybox(self): # build busybox if "netboot/busybox_config" in self.settings: mycmd = self.settings["netboot/busybox_config"] else: mycmd = "" try: cmd(self.settings["controller_file"]+" busybox "+ mycmd,env=self.env) except CatalystError: self.unbind() raise CatalystError("netboot build aborting due to error.", print_traceback=True)
def generate_hash(self, file_, hash_="crc32"): '''Prefered method of generating a hash for the passed in file_ @param file_: the file to generate the hash for @param hash_: the hash algorythm to use @returns the hash result ''' try: return getattr(self, self.hash_map[hash_].func)(file_, hash_) except: raise CatalystError("Error generating hash, is appropriate " + \ "utility installed on your system?", print_traceback=True)
def run_local(self): # tinderbox # example call: "grp.sh run xmms vim sys-apps/gleep" try: if os.path.exists(self.settings["controller_file"]): cmd(self.settings["controller_file"]+" run "+\ list_bashify(self.settings["tinderbox/packages"]),"run script failed.",env=self.env) except CatalystError: self.unbind() raise CatalystError("Tinderbox aborting due to error.", print_traceback=True)
def unpack(self): _unpack = True display_msg = None clst_unpack_hash = self.resume.get("unpack") if os.path.isdir(self.settings["source_path"]): unpack_cmd = "rsync -a --delete " + self.settings[ "source_path"] + " " + self.settings["chroot_path"] display_msg="\nStarting rsync from "+self.settings["source_path"]+"\nto "+\ self.settings["chroot_path"]+" (This may take some time) ...\n" error_msg = "Rsync of " + self.settings[ "source_path"] + " to " + self.settings[ "chroot_path"] + " failed." invalid_snapshot = False if "autoresume" in self.settings["options"]: if os.path.isdir(self.settings["source_path"]) and \ self.resume.is_enabled("unpack"): print "Resume point detected, skipping unpack operation..." _unpack = False elif "source_path_hash" in self.settings: if self.settings["source_path_hash"] != clst_unpack_hash: invalid_snapshot = True if _unpack: self.mount_safety_check() if invalid_snapshot: print "No Valid Resume point detected, cleaning up ..." self.clear_autoresume() self.clear_chroot() ensure_dirs(self.settings["chroot_path"] + "/tmp", mode=1777) if "pkgcache" in self.settings["options"]: ensure_dirs(self.settings["pkgcache_path"], mode=0755) if not display_msg: raise CatalystError( "Could not find appropriate source.\n" "Please check the 'source_subpath' " "setting in the spec file.", print_traceback=True) print display_msg cmd(unpack_cmd, error_msg, env=self.env) if "source_path_hash" in self.settings: self.resume.enable("unpack", data=self.settings["source_path_hash"]) else: self.resume.enable("unpack")
def hard_lock(self, max_wait=14400): """Does the NFS, hardlink shuffle to ensure locking on the disk. We create a PRIVATE lockfile, that is just a placeholder on the disk. Then we HARDLINK the real lockfile to that private file. If our file can 2 references, then we have the lock. :) Otherwise we lather, rise, and repeat. We default to a 4 hour timeout. """ self.myhardlock = self.hardlock_name(self.lockdir) start_time = time.time() reported_waiting = False while (time.time() < (start_time + max_wait)): # We only need it to exist. self.myfd = os.open(self.myhardlock, os.O_CREAT | os.O_RDWR, 0660) os.close(self.myfd) self.add_hardlock_file_to_cleanup() if not os.path.exists(self.myhardlock): raise CatalystError( "FileNotFound: Created lockfile is missing: " "%(filename)s" % {"filename": self.myhardlock}, print_traceback=True) try: os.link(self.myhardlock, self.lockfile) except SystemExit: raise except Exception: # if "DEBUG" in self.settings: # print "lockfile(): Hardlink: Link failed." # print "Exception: ",e pass if self.hardlink_is_mine(self.myhardlock, self.lockfile): # We have the lock. if reported_waiting: print return True if reported_waiting: writemsg(".") else: reported_waiting = True print print "Waiting on (hardlink) lockfile: (one '.' per 3 seconds)" print "Lockfile: " + self.lockfile time.sleep(3) os.unlink(self.myhardlock) return False
def run_local(self): for pkgset in self.settings["grp"]: # example call: "grp.sh run pkgset cd1 xmms vim sys-apps/gleep" mypackages = list_bashify(self.settings["grp/" + pkgset + "/packages"]) try: cmd(self.settings["controller_file"]+" run "+self.settings["grp/"+pkgset+"/type"]\ +" "+pkgset+" "+mypackages,env=self.env) except CatalystError: self.unbind() raise CatalystError("GRP build aborting due to error.", print_traceback=True)
def run_local(self): # tinderbox # example call: "grp.sh run xmms vim sys-apps/gleep" try: if os.path.exists(self.settings["controller_file"]): cmd([self.settings['controller_file'], 'run'] + self.settings['tinderbox/packages'], env=self.env) except CatalystError: self.unbind() raise CatalystError("Tinderbox aborting due to error.", print_traceback=True)
def build_busybox(self): # build busybox if "netboot/busybox_config" in self.settings: mycmd = [self.settings['netboot/busybox_config']] else: mycmd = [] try: cmd([self.settings['controller_file'], 'busybox'] + mycmd, env=self.env) except CatalystError: self.unbind() raise CatalystError("netboot build aborting due to error.", print_traceback=True)
def run_local(self): for pkgset in self.settings["grp"]: # example call: "grp.sh run pkgset cd1 xmms vim sys-apps/gleep" try: cmd([ self.settings['controller_file'], 'run', self.settings['grp/' + pkgset + '/type'], pkgset ] + self.settings['grp/' + pkgset + '/packages'], env=self.env) except CatalystError: self.unbind() raise CatalystError("GRP build aborting due to error.", print_traceback=True)
def build_target(addlargs): try: target = addlargs["target"].replace('-', '_') module = import_module(target) target = getattr(module, target)(conf_values, addlargs) except AttributeError: raise CatalystError("Target \"%s\" not available." % target, print_traceback=True) try: target.run() except: print "!!! catalyst: Error encountered during run of target " + \ addlargs["target"] sys.exit(1)
def __init__(self, spec, addlargs): try: if "netboot/packages" in addlargs: if isinstance(addlargs['netboot/packages'], str): loopy = [addlargs["netboot/packages"]] else: loopy = addlargs["netboot/packages"] for x in loopy: self.valid_values |= {"netboot/packages/" + x + "/files"} except: raise CatalystError("configuration error in netboot/packages.") StageBase.__init__(self, spec, addlargs) self.settings["merge_path"] = normpath("/tmp/image/")
def set_source_path(self): self.settings["source_path"] = normpath( self.settings["storedir"] + "/builds/" + self.settings["source_subpath"].rstrip('/') + ".tar.bz2") if os.path.isfile(self.settings["source_path"]): self.settings["source_path_hash"] = \ self.settings["hash_map"].generate_hash( self.settings["source_path"]) else: self.settings["source_path"] = normpath( self.settings["storedir"] + "/tmp/" + self.settings["source_subpath"] + '/') if not os.path.exists(self.settings["source_path"]): raise CatalystError("Source Path: " + self.settings["source_path"] + " does not exist.", print_traceback=True)
def copy_files_to_image(self): # copies specific files from the buildroot to merge_path myfiles = [] # check for autoresume point if "autoresume" in self.settings["options"] \ and self.resume.is_enabled("copy_files_to_image"): log.notice( 'Resume point detected, skipping target path setup operation...' ) else: if "netboot/packages" in self.settings: if isinstance(self.settings['netboot/packages'], str): loopy = [self.settings["netboot/packages"]] else: loopy = self.settings["netboot/packages"] for x in loopy: if "netboot/packages/" + x + "/files" in self.settings: if isinstance( self.settings['netboot/packages/' + x + '/files'], list): myfiles.extend(self.settings["netboot/packages/" + x + "/files"]) else: myfiles.append(self.settings["netboot/packages/" + x + "/files"]) if "netboot/extra_files" in self.settings: if isinstance(self.settings['netboot/extra_files'], list): myfiles.extend(self.settings["netboot/extra_files"]) else: myfiles.append(self.settings["netboot/extra_files"]) try: cmd([self.settings['controller_file'], 'image'] + myfiles, env=self.env) except CatalystError: self.unbind() raise CatalystError("Failed to copy files to image!", print_traceback=True) self.resume.enable("copy_files_to_image")
def run_local(self): # what modules do we want to blacklist? if "livecd/modblacklist" in self.settings: path = normpath(self.settings["chroot_path"] + "/etc/modprobe.d/blacklist.conf") try: with open(path, "a") as myf: myf.write("\n#Added by Catalyst:") # workaround until config.py is using configparser if isinstance(self.settings["livecd/modblacklist"], str): self.settings["livecd/modblacklist"] = self.settings[ "livecd/modblacklist"].split() for x in self.settings["livecd/modblacklist"]: myf.write("\nblacklist " + x) except Exception as e: raise CatalystError("Couldn't open " + self.settings["chroot_path"] + "/etc/modprobe.d/blacklist.conf.", print_traceback=True) from e
def run_local(self): # what modules do we want to blacklist? if "livecd/modblacklist" in self.settings: try: myf = open( self.settings["chroot_path"] + "/etc/modprobe.d/blacklist.conf", "a") except: self.unbind() raise CatalystError("Couldn't open " + self.settings["chroot_path"] + "/etc/modprobe.d/blacklist.conf.", print_traceback=True) myf.write("\n#Added by Catalyst:") # workaround until config.py is using configparser if isinstance(self.settings["livecd/modblacklist"], str): self.settings["livecd/modblacklist"] = self.settings[ "livecd/modblacklist"].split() for x in self.settings["livecd/modblacklist"]: myf.write("\nblacklist " + x) myf.close()
def __init__(self, spec, addlargs): self.required_values=["version_stamp","target","subarch",\ "rel_type","profile","snapshot","source_subpath"] self.valid_values = self.required_values[:] self.valid_values.extend(["grp/use"]) if "grp" not in addlargs: raise CatalystError( "Required value \"grp\" not specified in spec.") self.required_values.extend(["grp"]) if type(addlargs["grp"]) == types.StringType: addlargs["grp"] = [addlargs["grp"]] if "grp/use" in addlargs: if type(addlargs["grp/use"]) == types.StringType: addlargs["grp/use"] = [addlargs["grp/use"]] for x in addlargs["grp"]: self.required_values.append("grp/" + x + "/packages") self.required_values.append("grp/" + x + "/type") StageBase.__init__(self, spec, addlargs)
def fcntl_lock(self, locktype): if self.myfd == None: if not os.path.exists(os.path.dirname(self.lockdir)): raise CatalystError("DirectoryNotFound: %s" % os.path.dirname(self.lockdir), print_traceback=True) if not os.path.exists(self.lockfile): old_mask = os.umask(000) self.myfd = os.open(self.lockfile, os.O_CREAT | os.O_RDWR, 0660) try: if os.stat(self.lockfile).st_gid != self.gid: os.chown(self.lockfile, os.getuid(), self.gid) except SystemExit, e: raise except OSError, e: if e[0] == 2: #XXX: No such file or directory return self.fcntl_locking(locktype) else: writemsg( "Cannot chown a lockfile. This could cause inconvenience later.\n" ) os.umask(old_mask)