def copy_files_to_image(self):
# create image
myfiles = []
if "netboot/packages" in self.settings:
if isinstance(self.settings['netboot/packages'], str):
loopy = [self.settings["netboot/packages"]]
else:
loopy = self.settings["netboot/packages"]
for x in loopy:
if "netboot/packages/" + x + "/files" in self.settings:
if isinstance(
type(self.settings['netboot/packages/' + x +
'/files']), str):
myfiles.extend(self.settings["netboot/packages/" + x +
"/files"])
else:
myfiles.append(self.settings["netboot/packages/" + x +
"/files"])
if "netboot/extra_files" in self.settings:
if isinstance(self.settings['netboot/extra_files'], list):
myfiles.extend(self.settings["netboot/extra_files"])
else:
myfiles.append(self.settings["netboot/extra_files"])
try:
cmd([self.settings['controller_file'], 'image'] + myfiles,
env=self.env)
except CatalystError:
self.unbind()
raise CatalystError("netboot build aborting due to error.",
print_traceback=True)
def __init__(self,spec,addlargs):
self.required_values=[
"boot/kernel"
]
self.valid_values=self.required_values[:]
self.valid_values.extend([
"netboot2/packages",
"netboot2/use",
"netboot2/extra_files",
"netboot2/overlay",
"netboot2/busybox_config",
"netboot2/root_overlay",
"netboot2/linuxrc"
])
try:
if "netboot2/packages" in addlargs:
if isinstance(addlargs['netboot2/packages'], str):
loopy=[addlargs["netboot2/packages"]]
else:
loopy=addlargs["netboot2/packages"]
for x in loopy:
self.valid_values.append("netboot2/packages/"+x+"/files")
except:
raise CatalystError("configuration error in netboot2/packages.")
StageBase.__init__(self,spec,addlargs)
self.settings["merge_path"]=normpath("/tmp/image/")
def ensure_dirs(path, gid=-1, uid=-1, mode=0o755, minimal=True,
failback=None, fatal=False):
'''Wrapper to snakeoil.osutil's ensure_dirs()
This additionally allows for failures to run
cleanup or other code and/or raise fatal errors.
:param path: directory to ensure exists on disk
:param gid: a valid GID to set any created directories to
:param uid: a valid UID to set any created directories to
:param mode: permissions to set any created directories to
:param minimal: boolean controlling whether or not the specified mode
must be enforced, or is the minimal permissions necessary. For example,
if mode=0o755, minimal=True, and a directory exists with mode 0707,
this will restore the missing group perms resulting in 757.
:param failback: function to run in the event of a failed attemp
to create the directory.
:return: True if the directory could be created/ensured to have those
permissions, False if not.
'''
succeeded = snakeoil_ensure_dirs(
path, gid=gid, uid=uid, mode=mode, minimal=minimal)
if not succeeded:
if failback:
failback()
if fatal:
raise CatalystError(
"Failed to create directory: %s" % path, print_traceback=True)
return succeeded
def __init__(self, spec, addlargs):
self.valid_values = [
"netboot/kernel/sources", "netboot/kernel/config",
"netboot/kernel/prebuilt", "netboot/busybox_config",
"netboot/extra_files", "netboot/packages"
]
self.required_values = []
try:
if "netboot/packages" in addlargs:
if type(addlargs["netboot/packages"]) == types.StringType:
loopy = [addlargs["netboot/packages"]]
else:
loopy = addlargs["netboot/packages"]
# for x in loopy:
# self.required_values.append("netboot/packages/"+x+"/files")
except:
raise CatalystError("configuration error in netboot/packages.")
StageBase.__init__(self, spec, addlargs)
self.set_build_kernel_vars(addlargs)
if "netboot/busybox_config" in addlargs:
file_locate(self.settings, ["netboot/busybox_config"])
# Custom Kernel Tarball --- use that instead ...
# unless the user wants specific CFLAGS/CXXFLAGS, let's use -Os
for envvar in "CFLAGS", "CXXFLAGS":
if envvar not in os.environ and envvar not in addlargs:
self.settings[envvar] = "-Os -pipe"
def copy_files_to_image(self):
# create image
myfiles = []
if "netboot/packages" in self.settings:
if type(self.settings["netboot/packages"]) == types.StringType:
loopy = [self.settings["netboot/packages"]]
else:
loopy = self.settings["netboot/packages"]
for x in loopy:
if "netboot/packages/" + x + "/files" in self.settings:
if type(self.settings["netboot/packages/" + x +
"/files"]) == types.ListType:
myfiles.extend(self.settings["netboot/packages/" + x +
"/files"])
else:
myfiles.append(self.settings["netboot/packages/" + x +
"/files"])
if "netboot/extra_files" in self.settings:
if type(self.settings["netboot/extra_files"]) == types.ListType:
myfiles.extend(self.settings["netboot/extra_files"])
else:
myfiles.append(self.settings["netboot/extra_files"])
try:
cmd(self.settings["controller_file"]+\
" image " + list_bashify(myfiles),env=self.env)
except CatalystError:
self.unbind()
raise CatalystError("netboot build aborting due to error.",
print_traceback=True)
def __init__(self, spec, addlargs):
self.valid_values = [
"netboot/kernel/sources", "netboot/kernel/config",
"netboot/kernel/prebuilt", "netboot/busybox_config",
"netboot/extra_files", "netboot/packages"
]
self.required_values = []
try:
# XXX: This code does nothing because the for loop below is disabled.
if "netboot/packages" in addlargs:
if isinstance(addlargs['netboot/packages'], str):
_loopy = [addlargs["netboot/packages"]]
else:
_loopy = addlargs["netboot/packages"]
# for x in loopy:
# self.required_values.append("netboot/packages/"+x+"/files")
except:
raise CatalystError("configuration error in netboot/packages.")
StageBase.__init__(self, spec, addlargs)
if "netboot/busybox_config" in addlargs:
file_locate(self.settings, ["netboot/busybox_config"])
# Custom Kernel Tarball --- use that instead ...
# unless the user wants specific CFLAGS/CXXFLAGS, let's use -Os
for envvar in "CFLAGS", "CXXFLAGS":
if envvar not in os.environ and envvar not in addlargs:
self.settings[envvar] = "-Os -pipe"
def update_ebuild_repo(self) -> str:
repouri = 'https://anongit.gentoo.org/git/repo/sync/gentoo.git'
if self.ebuild_repo.is_dir():
git_cmds = [
[self.git, '-C', self.gitdir, 'fetch', '--quiet', '--depth=1'],
[
self.git, '-C', self.gitdir, 'update-ref', 'HEAD',
'FETCH_HEAD'
],
[self.git, '-C', self.gitdir, 'gc', '--quiet'],
]
else:
git_cmds = [
[
self.git,
'clone',
'--quiet',
'--depth=1',
'--bare',
# Set some config options to enable git gc to clean everything
# except what we just fetched. See git-gc(1).
'-c',
'gc.reflogExpire=0',
'-c',
'gc.reflogExpireUnreachable=0',
'-c',
'gc.rerereresolved=0',
'-c',
'gc.rerereunresolved=0',
'-c',
'gc.pruneExpire=now',
'--branch=stable',
repouri,
self.gitdir
],
]
try:
for cmd in git_cmds:
log.notice('>>> ' + ' '.join(cmd))
subprocess.run(cmd,
capture_output=True,
check=True,
encoding='utf-8',
close_fds=False)
sp = subprocess.run(
[self.git, '-C', self.gitdir, 'rev-parse', 'stable'],
stdout=subprocess.PIPE,
capture_output=True,
check=True,
encoding='utf-8',
close_fds=False)
return sp.stdout.rstrip()
except subprocess.CalledProcessError as e:
raise CatalystError(f'{e.cmd} failed with return code'
f'{e.returncode}\n'
f'{e.output}\n')
def parse_file(self, filename):
try:
with open(filename, "r") as myf:
self.lines = myf.readlines()
except:
raise CatalystError("Could not open file " + filename,
print_traceback=True)
self.filename = filename
self.parse()
def __init__(self,myspec):
builder.generic.__init__(self,myspec)
if self.settings["buildarch"]=="sparc64":
if not os.path.exists("/bin/linux32") and not os.path.exists("/usr/bin/linux32"):
raise CatalystError("required executable linux32 not found "
"(\"emerge setarch\" to fix.)", print_traceback=True)
self.settings["CHROOT"]="linux32 chroot"
self.settings["crosscompile"] = False;
else:
self.settings["CHROOT"]="chroot"
def create_netboot_files(self):
# finish it all up
try:
cmd(self.settings["controller_file"] + " finish", env=self.env)
except CatalystError:
self.unbind()
raise CatalystError("netboot build aborting due to error.",
print_traceback=True)
# end
print "netboot: build finished !"
def create_netboot_files(self):
# finish it all up
try:
cmd([self.settings['controller_file'], 'finish'], env=self.env)
except CatalystError:
self.unbind()
raise CatalystError("netboot build aborting due to error.",
print_traceback=True)
# end
log.notice('netboot: build finished !')
def build_target(addlargs):
try:
target = addlargs["target"].replace('-', '_')
module = import_module(target)
target = getattr(module, target)(conf_values, addlargs)
except AttributeError as e:
raise CatalystError("Target \"%s\" not available." % target,
print_traceback=True) from e
except CatalystError:
return False
return target.run()
def move_kernels(self):
# we're done, move the kernels to builds/*
# no auto resume here as we always want the
# freshest images moved
try:
cmd([self.settings['controller_file'], 'final'], env=self.env)
log.notice('Netboot Build Finished!')
except CatalystError:
self.unbind()
raise CatalystError("Failed to move kernel images!",
print_traceback=True)
def build_busybox(self):
# build busybox
if "netboot/busybox_config" in self.settings:
mycmd = self.settings["netboot/busybox_config"]
else:
mycmd = ""
try:
cmd(self.settings["controller_file"]+" busybox "+ mycmd,env=self.env)
except CatalystError:
self.unbind()
raise CatalystError("netboot build aborting due to error.",
print_traceback=True)
def generate_hash(self, file_, hash_="crc32"):
'''Prefered method of generating a hash for the passed in file_
@param file_: the file to generate the hash for
@param hash_: the hash algorythm to use
@returns the hash result
'''
try:
return getattr(self, self.hash_map[hash_].func)(file_, hash_)
except:
raise CatalystError("Error generating hash, is appropriate " + \
"utility installed on your system?", print_traceback=True)
def run_local(self):
# tinderbox
# example call: "grp.sh run xmms vim sys-apps/gleep"
try:
if os.path.exists(self.settings["controller_file"]):
cmd(self.settings["controller_file"]+" run "+\
list_bashify(self.settings["tinderbox/packages"]),"run script failed.",env=self.env)
except CatalystError:
self.unbind()
raise CatalystError("Tinderbox aborting due to error.",
print_traceback=True)
def unpack(self):
_unpack = True
display_msg = None
clst_unpack_hash = self.resume.get("unpack")
if os.path.isdir(self.settings["source_path"]):
unpack_cmd = "rsync -a --delete " + self.settings[
"source_path"] + " " + self.settings["chroot_path"]
display_msg="\nStarting rsync from "+self.settings["source_path"]+"\nto "+\
self.settings["chroot_path"]+" (This may take some time) ...\n"
error_msg = "Rsync of " + self.settings[
"source_path"] + " to " + self.settings[
"chroot_path"] + " failed."
invalid_snapshot = False
if "autoresume" in self.settings["options"]:
if os.path.isdir(self.settings["source_path"]) and \
self.resume.is_enabled("unpack"):
print "Resume point detected, skipping unpack operation..."
_unpack = False
elif "source_path_hash" in self.settings:
if self.settings["source_path_hash"] != clst_unpack_hash:
invalid_snapshot = True
if _unpack:
self.mount_safety_check()
if invalid_snapshot:
print "No Valid Resume point detected, cleaning up ..."
self.clear_autoresume()
self.clear_chroot()
ensure_dirs(self.settings["chroot_path"] + "/tmp", mode=1777)
if "pkgcache" in self.settings["options"]:
ensure_dirs(self.settings["pkgcache_path"], mode=0755)
if not display_msg:
raise CatalystError(
"Could not find appropriate source.\n"
"Please check the 'source_subpath' "
"setting in the spec file.",
print_traceback=True)
print display_msg
cmd(unpack_cmd, error_msg, env=self.env)
if "source_path_hash" in self.settings:
self.resume.enable("unpack",
data=self.settings["source_path_hash"])
else:
self.resume.enable("unpack")
def hard_lock(self, max_wait=14400):
"""Does the NFS, hardlink shuffle to ensure locking on the disk.
We create a PRIVATE lockfile, that is just a placeholder on the disk.
Then we HARDLINK the real lockfile to that private file.
If our file can 2 references, then we have the lock. :)
Otherwise we lather, rise, and repeat.
We default to a 4 hour timeout.
"""
self.myhardlock = self.hardlock_name(self.lockdir)
start_time = time.time()
reported_waiting = False
while (time.time() < (start_time + max_wait)):
# We only need it to exist.
self.myfd = os.open(self.myhardlock, os.O_CREAT | os.O_RDWR, 0660)
os.close(self.myfd)
self.add_hardlock_file_to_cleanup()
if not os.path.exists(self.myhardlock):
raise CatalystError(
"FileNotFound: Created lockfile is missing: "
"%(filename)s" % {"filename": self.myhardlock},
print_traceback=True)
try:
os.link(self.myhardlock, self.lockfile)
except SystemExit:
raise
except Exception:
# if "DEBUG" in self.settings:
# print "lockfile(): Hardlink: Link failed."
# print "Exception: ",e
pass
if self.hardlink_is_mine(self.myhardlock, self.lockfile):
# We have the lock.
if reported_waiting:
print
return True
if reported_waiting:
writemsg(".")
else:
reported_waiting = True
print
print "Waiting on (hardlink) lockfile: (one '.' per 3 seconds)"
print "Lockfile: " + self.lockfile
time.sleep(3)
os.unlink(self.myhardlock)
return False
def run_local(self):
for pkgset in self.settings["grp"]:
# example call: "grp.sh run pkgset cd1 xmms vim sys-apps/gleep"
mypackages = list_bashify(self.settings["grp/" + pkgset +
"/packages"])
try:
cmd(self.settings["controller_file"]+" run "+self.settings["grp/"+pkgset+"/type"]\
+" "+pkgset+" "+mypackages,env=self.env)
except CatalystError:
self.unbind()
raise CatalystError("GRP build aborting due to error.",
print_traceback=True)
def run_local(self):
# tinderbox
# example call: "grp.sh run xmms vim sys-apps/gleep"
try:
if os.path.exists(self.settings["controller_file"]):
cmd([self.settings['controller_file'], 'run'] +
self.settings['tinderbox/packages'],
env=self.env)
except CatalystError:
self.unbind()
raise CatalystError("Tinderbox aborting due to error.",
print_traceback=True)
def build_busybox(self):
# build busybox
if "netboot/busybox_config" in self.settings:
mycmd = [self.settings['netboot/busybox_config']]
else:
mycmd = []
try:
cmd([self.settings['controller_file'], 'busybox'] + mycmd,
env=self.env)
except CatalystError:
self.unbind()
raise CatalystError("netboot build aborting due to error.",
print_traceback=True)
def run_local(self):
for pkgset in self.settings["grp"]:
# example call: "grp.sh run pkgset cd1 xmms vim sys-apps/gleep"
try:
cmd([
self.settings['controller_file'], 'run',
self.settings['grp/' + pkgset + '/type'], pkgset
] + self.settings['grp/' + pkgset + '/packages'],
env=self.env)
except CatalystError:
self.unbind()
raise CatalystError("GRP build aborting due to error.",
print_traceback=True)
def build_target(addlargs):
try:
target = addlargs["target"].replace('-', '_')
module = import_module(target)
target = getattr(module, target)(conf_values, addlargs)
except AttributeError:
raise CatalystError("Target \"%s\" not available." % target,
print_traceback=True)
try:
target.run()
except:
print "!!! catalyst: Error encountered during run of target " + \
addlargs["target"]
sys.exit(1)
def __init__(self, spec, addlargs):
try:
if "netboot/packages" in addlargs:
if isinstance(addlargs['netboot/packages'], str):
loopy = [addlargs["netboot/packages"]]
else:
loopy = addlargs["netboot/packages"]
for x in loopy:
self.valid_values |= {"netboot/packages/" + x + "/files"}
except:
raise CatalystError("configuration error in netboot/packages.")
StageBase.__init__(self, spec, addlargs)
self.settings["merge_path"] = normpath("/tmp/image/")
def set_source_path(self):
self.settings["source_path"] = normpath(
self.settings["storedir"] + "/builds/" +
self.settings["source_subpath"].rstrip('/') + ".tar.bz2")
if os.path.isfile(self.settings["source_path"]):
self.settings["source_path_hash"] = \
self.settings["hash_map"].generate_hash(
self.settings["source_path"])
else:
self.settings["source_path"] = normpath(
self.settings["storedir"] + "/tmp/" +
self.settings["source_subpath"] + '/')
if not os.path.exists(self.settings["source_path"]):
raise CatalystError("Source Path: " +
self.settings["source_path"] +
" does not exist.",
print_traceback=True)
def copy_files_to_image(self):
# copies specific files from the buildroot to merge_path
myfiles = []
# check for autoresume point
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("copy_files_to_image"):
log.notice(
'Resume point detected, skipping target path setup operation...'
)
else:
if "netboot/packages" in self.settings:
if isinstance(self.settings['netboot/packages'], str):
loopy = [self.settings["netboot/packages"]]
else:
loopy = self.settings["netboot/packages"]
for x in loopy:
if "netboot/packages/" + x + "/files" in self.settings:
if isinstance(
self.settings['netboot/packages/' + x + '/files'],
list):
myfiles.extend(self.settings["netboot/packages/" + x +
"/files"])
else:
myfiles.append(self.settings["netboot/packages/" + x +
"/files"])
if "netboot/extra_files" in self.settings:
if isinstance(self.settings['netboot/extra_files'], list):
myfiles.extend(self.settings["netboot/extra_files"])
else:
myfiles.append(self.settings["netboot/extra_files"])
try:
cmd([self.settings['controller_file'], 'image'] + myfiles,
env=self.env)
except CatalystError:
self.unbind()
raise CatalystError("Failed to copy files to image!",
print_traceback=True)
self.resume.enable("copy_files_to_image")
def run_local(self):
# what modules do we want to blacklist?
if "livecd/modblacklist" in self.settings:
path = normpath(self.settings["chroot_path"] +
"/etc/modprobe.d/blacklist.conf")
try:
with open(path, "a") as myf:
myf.write("\n#Added by Catalyst:")
# workaround until config.py is using configparser
if isinstance(self.settings["livecd/modblacklist"], str):
self.settings["livecd/modblacklist"] = self.settings[
"livecd/modblacklist"].split()
for x in self.settings["livecd/modblacklist"]:
myf.write("\nblacklist " + x)
except Exception as e:
raise CatalystError("Couldn't open " +
self.settings["chroot_path"] +
"/etc/modprobe.d/blacklist.conf.",
print_traceback=True) from e
def run_local(self):
# what modules do we want to blacklist?
if "livecd/modblacklist" in self.settings:
try:
myf = open(
self.settings["chroot_path"] +
"/etc/modprobe.d/blacklist.conf", "a")
except:
self.unbind()
raise CatalystError("Couldn't open " +
self.settings["chroot_path"] +
"/etc/modprobe.d/blacklist.conf.",
print_traceback=True)
myf.write("\n#Added by Catalyst:")
# workaround until config.py is using configparser
if isinstance(self.settings["livecd/modblacklist"], str):
self.settings["livecd/modblacklist"] = self.settings[
"livecd/modblacklist"].split()
for x in self.settings["livecd/modblacklist"]:
myf.write("\nblacklist " + x)
myf.close()
def __init__(self, spec, addlargs):
self.required_values=["version_stamp","target","subarch",\
"rel_type","profile","snapshot","source_subpath"]
self.valid_values = self.required_values[:]
self.valid_values.extend(["grp/use"])
if "grp" not in addlargs:
raise CatalystError(
"Required value \"grp\" not specified in spec.")
self.required_values.extend(["grp"])
if type(addlargs["grp"]) == types.StringType:
addlargs["grp"] = [addlargs["grp"]]
if "grp/use" in addlargs:
if type(addlargs["grp/use"]) == types.StringType:
addlargs["grp/use"] = [addlargs["grp/use"]]
for x in addlargs["grp"]:
self.required_values.append("grp/" + x + "/packages")
self.required_values.append("grp/" + x + "/type")
StageBase.__init__(self, spec, addlargs)
def fcntl_lock(self, locktype):
if self.myfd == None:
if not os.path.exists(os.path.dirname(self.lockdir)):
raise CatalystError("DirectoryNotFound: %s" %
os.path.dirname(self.lockdir),
print_traceback=True)
if not os.path.exists(self.lockfile):
old_mask = os.umask(000)
self.myfd = os.open(self.lockfile, os.O_CREAT | os.O_RDWR,
0660)
try:
if os.stat(self.lockfile).st_gid != self.gid:
os.chown(self.lockfile, os.getuid(), self.gid)
except SystemExit, e:
raise
except OSError, e:
if e[0] == 2: #XXX: No such file or directory
return self.fcntl_locking(locktype)
else:
writemsg(
"Cannot chown a lockfile. This could cause inconvenience later.\n"
)
os.umask(old_mask)
