def empty(self):
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("empty"):
log.notice('Resume point detected, skipping empty operation...')
else:
if "netboot2/empty" in self.settings:
if isinstance(self.settings['netboot2/empty'], str):
self.settings["netboot2/empty"] = self.settings[
"netboot2/empty"].split()
for x in self.settings["netboot2/empty"]:
myemp = self.settings["chroot_path"] + self.settings[
"merge_path"] + x
if not os.path.isdir(myemp):
log.warning(
'not a directory or does not exist, skipping "empty" operation: %s',
x)
continue
log.info('Emptying directory %s', x)
# stat the dir, delete the dir, recreate the dir and set
# the proper perms and ownership
mystat = os.stat(myemp)
shutil.rmtree(myemp)
ensure_dirs(myemp, mode=0o755)
os.chown(myemp, mystat[ST_UID], mystat[ST_GID])
os.chmod(myemp, mystat[ST_MODE])
self.resume.enable("empty")
def __init__(self, lockdir): self.gid = 250 self.lockfile = os.path.join(lockdir, '.catalyst_lock') ensure_dirs(lockdir) fileutils.touch(self.lockfile, mode=0o664) os.chown(self.lockfile, -1, self.gid) self.lock = osutils.FsLock(self.lockfile)
def base_dirs(self):
if os.uname()[0] == "FreeBSD":
# baselayout no longer creates the .keep files in proc and dev for FreeBSD as it
# would create them too late...we need them earlier before bind mounting filesystems
# since proc and dev are not writeable, so...create them here
ensure_dirs(self.settings["stage_path"] + "/proc")
ensure_dirs(self.settings["stage_path"] + "/dev")
if not os.path.isfile(self.settings["stage_path"] + "/proc/.keep"):
try:
proc_keepfile = open(
self.settings["stage_path"] + "/proc/.keep", "w")
proc_keepfile.write('')
proc_keepfile.close()
except IOError:
print "!!! Failed to create %s" % (
self.settings["stage_path"] + "/dev/.keep")
if not os.path.isfile(self.settings["stage_path"] + "/dev/.keep"):
try:
dev_keepfile = open(
self.settings["stage_path"] + "/dev/.keep", "w")
dev_keepfile.write('')
dev_keepfile.close()
except IOError:
print "!!! Failed to create %s" % (
self.settings["stage_path"] + "/dev/.keep")
else:
pass
def run(self):
if "purgeonly" in self.settings["options"]:
self.purge()
return
if "purge" in self.settings["options"]:
self.purge()
self.setup()
print "Creating Portage tree snapshot "+self.settings["version_stamp"]+\
" from "+self.settings["portdir"]+"..."
mytmp = self.settings["tmp_path"]
ensure_dirs(mytmp)
cmd("rsync -a --delete --exclude /packages/ --exclude /distfiles/ " +
"--exclude /local/ --exclude CVS/ --exclude .svn --filter=H_**/files/digest-* "
+ self.settings["portdir"] + "/ " + mytmp +
"/%s/" % self.settings["repo_name"],
"Snapshot failure",
env=self.env)
print "Compressing Portage snapshot tarball..."
cmd("tar -I lbzip2 -cf " + self.settings["snapshot_path"] + " -C " +
mytmp + " " + self.settings["repo_name"],
"Snapshot creation failure",
env=self.env)
self.gen_contents_file(self.settings["snapshot_path"])
self.gen_digest_file(self.settings["snapshot_path"])
self.cleanup()
print "snapshot: complete!"
def set_mounts(self):
# stage_path/proc probably doesn't exist yet, so create it
ensure_dirs(self.settings["stage_path"] + "/proc")
# alter the mount mappings to bind mount proc onto it
self.mounts.append("stage1root/proc")
self.target_mounts["stage1root/proc"] = "/tmp/stage1root/proc"
self.mountmap["stage1root/proc"] = "/proc"
def set_mounts(self):
# stage_path/proc probably doesn't exist yet, so create it
ensure_dirs(self.settings["stage_path"]+"/proc")
# alter the mount mappings to bind mount proc onto it
self.mounts.append("stage1root/proc")
self.target_mounts["stage1root/proc"] = "/tmp/stage1root/proc"
self.mountmap["stage1root/proc"] = "/proc"
def run(self):
if "purgeonly" in self.settings["options"]:
self.purge()
return True
if "purge" in self.settings["options"]:
self.purge()
success = True
self.setup()
log.notice('Creating %s tree snapshot %s from %s ...',
self.settings["repo_name"], self.settings['version_stamp'],
self.settings['portdir'])
mytmp=self.settings["tmp_path"]
ensure_dirs(mytmp)
cmd(['rsync', '-a', '--no-o', '--no-g', '--delete',
'--exclude=/packages/',
'--exclude=/distfiles/',
'--exclude=/local/',
'--exclude=CVS/',
'--exclude=.svn',
'--exclude=.git/',
'--filter=H_**/files/digest-*',
self.settings['portdir'] + '/',
mytmp + '/' + self.settings['repo_name'] + '/'],
env=self.env)
log.notice('Compressing %s snapshot tarball ...', self.settings["repo_name"])
compressor = CompressMap(self.settings["compress_definitions"],
env=self.env, default_mode=self.settings['compression_mode'],
comp_prog=self.settings["comp_prog"])
infodict = compressor.create_infodict(
source=self.settings["repo_name"],
destination=self.settings["snapshot_path"],
basedir=mytmp,
filename=self.settings["snapshot_path"],
mode=self.settings["compression_mode"],
auto_extension=True
)
if not compressor.compress(infodict):
success = False
log.error('Snapshot compression failure')
else:
filename = '.'.join([self.settings["snapshot_path"],
compressor.extension(self.settings["compression_mode"])])
log.notice('Snapshot successfully written to %s', filename)
self.gen_contents_file(filename)
self.gen_digest_file(filename)
if "keepwork" not in self.settings["options"]:
self.cleanup()
if success:
log.info('snapshot: complete!')
return success
def run(self):
if "purgeonly" in self.settings["options"]:
self.purge()
return True
if "purge" in self.settings["options"]:
self.purge()
success = True
self.setup()
log.notice('Creating %s tree snapshot %s from %s ...',
self.settings["repo_name"], self.settings['version_stamp'],
self.settings['portdir'])
mytmp=self.settings["tmp_path"]
ensure_dirs(mytmp)
cmd(['rsync', '-a', '--no-o', '--no-g', '--delete',
'--exclude=/packages/',
'--exclude=/distfiles/',
'--exclude=/local/',
'--exclude=CVS/',
'--exclude=.svn',
'--filter=H_**/files/digest-*',
self.settings['portdir'] + '/',
mytmp + '/' + self.settings['repo_name'] + '/'],
env=self.env)
log.notice('Compressing %s snapshot tarball ...', self.settings["repo_name"])
compressor = CompressMap(self.settings["compress_definitions"],
env=self.env, default_mode=self.settings['compression_mode'],
comp_prog=self.settings["comp_prog"])
infodict = compressor.create_infodict(
source=self.settings["repo_name"],
destination=self.settings["snapshot_path"],
basedir=mytmp,
filename=self.settings["snapshot_path"],
mode=self.settings["compression_mode"],
auto_extension=True
)
if not compressor.compress(infodict):
success = False
log.error('Snapshot compression failure')
else:
filename = '.'.join([self.settings["snapshot_path"],
compressor.extension(self.settings["compression_mode"])])
log.notice('Snapshot successfully written to %s', filename)
self.gen_contents_file(filename)
self.gen_digest_file(filename)
if "keepwork" not in self.settings["options"]:
self.cleanup()
if success:
log.info('snapshot: complete!')
return success
def set_target_path(self):
self.settings["target_path"]=normpath(self.settings["storedir"]+"/builds/"+\
self.settings["target_subpath"])
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("setup_target_path"):
log.notice('Resume point detected, skipping target path setup operation...')
else:
# first clean up any existing target stuff
clear_path(self.settings['target_path'])
self.resume.enable("setup_target_path")
ensure_dirs(self.settings["storedir"]+"/builds/")
def unpack(self):
_unpack = True
display_msg = None
clst_unpack_hash = self.resume.get("unpack")
if os.path.isdir(self.settings["source_path"]):
unpack_cmd = "rsync -a --delete " + self.settings[
"source_path"] + " " + self.settings["chroot_path"]
display_msg="\nStarting rsync from "+self.settings["source_path"]+"\nto "+\
self.settings["chroot_path"]+" (This may take some time) ...\n"
error_msg = "Rsync of " + self.settings[
"source_path"] + " to " + self.settings[
"chroot_path"] + " failed."
invalid_snapshot = False
if "autoresume" in self.settings["options"]:
if os.path.isdir(self.settings["source_path"]) and \
self.resume.is_enabled("unpack"):
print "Resume point detected, skipping unpack operation..."
_unpack = False
elif "source_path_hash" in self.settings:
if self.settings["source_path_hash"] != clst_unpack_hash:
invalid_snapshot = True
if _unpack:
self.mount_safety_check()
if invalid_snapshot:
print "No Valid Resume point detected, cleaning up ..."
self.clear_autoresume()
self.clear_chroot()
ensure_dirs(self.settings["chroot_path"] + "/tmp", mode=1777)
if "pkgcache" in self.settings["options"]:
ensure_dirs(self.settings["pkgcache_path"], mode=0755)
if not display_msg:
raise CatalystError(
"Could not find appropriate source.\n"
"Please check the 'source_subpath' "
"setting in the spec file.",
print_traceback=True)
print display_msg
cmd(unpack_cmd, error_msg, env=self.env)
if "source_path_hash" in self.settings:
self.resume.enable("unpack",
data=self.settings["source_path_hash"])
else:
self.resume.enable("unpack")
def set_target_path(self):
self.settings["target_path"]=normpath(self.settings["storedir"]+"/builds/"+\
self.settings["target_subpath"])
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("setup_target_path"):
log.notice('Resume point detected, skipping target path setup operation...')
else:
# first clean up any existing target stuff
if os.path.isfile(self.settings["target_path"]):
cmd("rm -f "+self.settings["target_path"], \
"Could not remove existing file: "+self.settings["target_path"],env=self.env)
self.resume.enable("setup_target_path")
ensure_dirs(self.settings["storedir"]+"/builds/")
def base_dirs(self):
if os.uname()[0] == "FreeBSD":
# baselayout no longer creates the .keep files in proc and dev for FreeBSD as it
# would create them too late...we need them earlier before bind mounting filesystems
# since proc and dev are not writeable, so...create them here
ensure_dirs(self.settings["stage_path"] + "/proc")
ensure_dirs(self.settings["stage_path"] + "/dev")
for f in ('/proc', '/dev'):
f = self.settings['stage_path'] + f + '/.keep'
if not os.path.isfile(f):
try:
fileutils.touch(f)
except IOError:
log.error('Failed to create %s', f)
def purge(self):
myemp=self.settings["tmp_path"]
if os.path.isdir(myemp):
log.notice('Emptying directory %s', myemp)
# stat the dir, delete the dir, recreate the dir and set
# the proper perms and ownership
mystat=os.stat(myemp)
# There's no easy way to change flags recursively in python
if os.uname()[0] == "FreeBSD":
os.system("chflags -R noschg "+myemp)
shutil.rmtree(myemp)
ensure_dirs(myemp, mode=0o755)
os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
os.chmod(myemp,mystat[ST_MODE])
def base_dirs(self):
if os.uname()[0] == "FreeBSD":
# baselayout no longer creates the .keep files in proc and dev for FreeBSD as it
# would create them too late...we need them earlier before bind mounting filesystems
# since proc and dev are not writeable, so...create them here
ensure_dirs(self.settings["stage_path"]+"/proc")
ensure_dirs(self.settings["stage_path"]+"/dev")
for f in ('/proc', '/dev'):
f = self.settings['stage_path'] + f + '/.keep'
if not os.path.isfile(f):
try:
fileutils.touch(f)
except IOError:
log.error('Failed to create %s', f)
def set_target_path(self):
self.settings["target_path"] = normpath(
self.settings["storedir"] + "/builds/" +
self.settings["target_subpath"] + '/')
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("setup_target_path"):
print "Resume point detected, skipping target path setup operation..."
else:
# first clean up any existing target stuff
if os.path.exists(self.settings["target_path"]):
cmd("rm -rf "+self.settings["target_path"],\
"Could not remove existing directory: "+self.settings["target_path"],env=self.env)
self.resume.enable("setup_target_path")
ensure_dirs(self.settings["target_path"])
def purge(self):
myemp = self.settings["tmp_path"]
if os.path.isdir(myemp):
print "Emptying directory", myemp
"""
stat the dir, delete the dir, recreate the dir and set
the proper perms and ownership
"""
mystat = os.stat(myemp)
""" There's no easy way to change flags recursively in python """
if os.uname()[0] == "FreeBSD":
os.system("chflags -R noschg " + myemp)
shutil.rmtree(myemp)
ensure_dirs(myemp, mode=0755)
os.chown(myemp, mystat[ST_UID], mystat[ST_GID])
os.chmod(myemp, mystat[ST_MODE])
def run(self):
if "purgeonly" in self.settings["options"]:
self.purge()
return True
if "purge" in self.settings["options"]:
self.purge()
success = True
self.setup()
log.notice('Creating Portage tree snapshot %s from %s ...',
self.settings['version_stamp'], self.settings['portdir'])
mytmp=self.settings["tmp_path"]
ensure_dirs(mytmp)
target_snapshot = self.settings["portdir"] + "/ " + mytmp + "/%s/" % self.settings["repo_name"]
cmd("rsync -a --no-o --no-g --delete --exclude /packages/ --exclude /distfiles/ " +
"--exclude /local/ --exclude CVS/ --exclude .svn --filter=H_**/files/digest-* " +
target_snapshot,
"Snapshot failure", env=self.env)
log.notice('Compressing Portage snapshot tarball ...')
compressor = CompressMap(self.settings["compress_definitions"],
env=self.env, default_mode=self.settings['compression_mode'])
infodict = compressor.create_infodict(
source=self.settings["repo_name"],
destination=self.settings["snapshot_path"],
basedir=mytmp,
filename=self.settings["snapshot_path"],
mode=self.settings["compression_mode"],
auto_extension=True
)
if not compressor.compress(infodict):
success = False
log.error('Snapshot compression failure')
else:
filename = '.'.join([self.settings["snapshot_path"],
compressor.extension(self.settings["compression_mode"])])
log.notice('Snapshot successfully written to %s', filename)
self.gen_contents_file(filename)
self.gen_digest_file(filename)
self.cleanup()
if success:
log.info('snapshot: complete!')
return success
def empty(self):
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("empty"):
log.notice('Resume point detected, skipping empty operation...')
else:
if "netboot2/empty" in self.settings:
if isinstance(self.settings['netboot2/empty'], str):
self.settings["netboot2/empty"]=self.settings["netboot2/empty"].split()
for x in self.settings["netboot2/empty"]:
myemp=self.settings["chroot_path"] + self.settings["merge_path"] + x
if not os.path.isdir(myemp):
log.warning('not a directory or does not exist, skipping "empty" operation: %s', x)
continue
log.info('Emptying directory %s', x)
# stat the dir, delete the dir, recreate the dir and set
# the proper perms and ownership
mystat=os.stat(myemp)
shutil.rmtree(myemp)
ensure_dirs(myemp, mode=0o755)
os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
os.chmod(myemp,mystat[ST_MODE])
self.resume.enable("empty")
def __init__(self, lockdir):
ensure_dirs(lockdir)
lockfile = os.path.join(lockdir, '.catalyst_lock')
Lock.__init__(self, lockfile)
def setup(self):
x = normpath(self.settings["storedir"] + "/snapshots")
ensure_dirs(x)
def __init__(self, basedir, mode=0755):
self.basedir = basedir
ensure_dirs(basedir, mode=mode, fatal=True)
self._points = {}
self._init_points_()
def __init__(self, basedir, mode=0o755):
self.basedir = basedir
ensure_dirs(basedir, mode=mode, fatal=True)
self._points = {}
self._init_points_()
def setup(self): x=normpath(self.settings["storedir"]+"/snapshots") ensure_dirs(x)