def cmd(mycmd, env=None, debug=False, fail_func=None):
"""Run the external |mycmd|.
If |mycmd| is a string, then it's assumed to be a bash snippet and will
be run through bash. Otherwise, it's a standalone command line and will
be run directly.
"""
log.debug('cmd: %r', mycmd)
sys.stdout.flush()
if env is None:
env = {}
if 'BASH_ENV' not in env:
env = env.copy()
env['BASH_ENV'] = '/etc/spork/is/not/valid/profile.env'
args = []
if isinstance(mycmd, str):
args.append(BASH_BINARY)
if debug:
args.append('-x')
args.extend(['-c', mycmd])
else:
args.extend(mycmd)
log.debug('args: %r', args)
proc = Popen(args, env=env)
ret = proc.wait()
if ret:
if fail_func:
log.error('cmd(%r) exited %s; running fail_func().', args, ret)
fail_func()
raise CatalystError('cmd(%r) exited %s' % (args, ret),
print_traceback=False)
def cmd(mycmd, env=None, debug=False, fail_func=None):
"""Run the external |mycmd|.
If |mycmd| is a string, then it's assumed to be a bash snippet and will
be run through bash. Otherwise, it's a standalone command line and will
be run directly.
"""
log.debug('cmd: %r', mycmd)
sys.stdout.flush()
if env is None:
env = {}
if 'BASH_ENV' not in env:
env = env.copy()
env['BASH_ENV'] = '/etc/spork/is/not/valid/profile.env'
args = []
if isinstance(mycmd, str):
args.append(BASH_BINARY)
if debug:
args.append('-x')
args.extend(['-c', mycmd])
else:
args.extend(mycmd)
log.debug('args: %r', args)
proc = Popen(args, env=env)
ret = proc.wait()
if ret:
if fail_func:
log.error('cmd(%r) exited %s; running fail_func().', args, ret)
fail_func()
raise CatalystError('cmd(%r) exited %s' % (args, ret),
print_traceback=False)
def clear_dir(target,
mode=0o755,
chg_flags=False,
remove=False,
clear_nondir=True):
'''Universal directory clearing function
@target: string, path to be cleared or removed
@mode: integer, desired mode to set the directory to
@chg_flags: boolean used for FreeBSD hosts
@remove: boolean, passed through to clear_dir()
@return boolean
'''
log.debug('start: %s', target)
if not target:
log.debug('no target... returning')
return False
mystat = None
if os.path.isdir(target) and not os.path.islink(target):
log.notice('Emptying directory: %s', target)
# stat the dir, delete the dir, recreate the dir and set
# the proper perms and ownership
try:
log.debug('os.stat()')
mystat = os.stat(target)
# There's no easy way to change flags recursively in python
if chg_flags and os.uname()[0] == "FreeBSD":
cmd(['chflags', '-R', 'noschg', target])
log.debug('shutil.rmtree()')
shutil.rmtree(target)
except Exception:
log.error('clear_dir failed', exc_info=True)
return False
elif os.path.exists(target):
if clear_nondir:
log.debug("Clearing (unlinking) non-directory: %s", target)
os.unlink(target)
else:
log.info('clear_dir failed: %s: is not a directory', target)
return False
else:
log.debug("Condidtions not met to clear: %s", target)
log.debug(" isdir: %s", os.path.isdir(target))
log.debug(" islink: %s", os.path.islink(target))
log.debug(" exists: %s", os.path.exists(target))
if not remove:
log.debug('ensure_dirs()')
ensure_dirs(target, mode=mode)
if mystat:
os.chown(target, mystat[ST_UID], mystat[ST_GID])
os.chmod(target, mystat[ST_MODE])
log.debug('DONE, returning True')
return True
def run(self):
if "purgeonly" in self.settings["options"]:
self.purge()
return True
if "purge" in self.settings["options"]:
self.purge()
success = True
self.setup()
log.notice('Creating %s tree snapshot %s from %s ...',
self.settings["repo_name"], self.settings['version_stamp'],
self.settings['portdir'])
mytmp=self.settings["tmp_path"]
ensure_dirs(mytmp)
cmd(['rsync', '-a', '--no-o', '--no-g', '--delete',
'--exclude=/packages/',
'--exclude=/distfiles/',
'--exclude=/local/',
'--exclude=CVS/',
'--exclude=.svn',
'--exclude=.git/',
'--filter=H_**/files/digest-*',
self.settings['portdir'] + '/',
mytmp + '/' + self.settings['repo_name'] + '/'],
env=self.env)
log.notice('Compressing %s snapshot tarball ...', self.settings["repo_name"])
compressor = CompressMap(self.settings["compress_definitions"],
env=self.env, default_mode=self.settings['compression_mode'],
comp_prog=self.settings["comp_prog"])
infodict = compressor.create_infodict(
source=self.settings["repo_name"],
destination=self.settings["snapshot_path"],
basedir=mytmp,
filename=self.settings["snapshot_path"],
mode=self.settings["compression_mode"],
auto_extension=True
)
if not compressor.compress(infodict):
success = False
log.error('Snapshot compression failure')
else:
filename = '.'.join([self.settings["snapshot_path"],
compressor.extension(self.settings["compression_mode"])])
log.notice('Snapshot successfully written to %s', filename)
self.gen_contents_file(filename)
self.gen_digest_file(filename)
if "keepwork" not in self.settings["options"]:
self.cleanup()
if success:
log.info('snapshot: complete!')
return success
def run(self):
if "purgeonly" in self.settings["options"]:
self.purge()
return True
if "purge" in self.settings["options"]:
self.purge()
success = True
self.setup()
log.notice('Creating %s tree snapshot %s from %s ...',
self.settings["repo_name"], self.settings['version_stamp'],
self.settings['portdir'])
mytmp=self.settings["tmp_path"]
ensure_dirs(mytmp)
cmd(['rsync', '-a', '--no-o', '--no-g', '--delete',
'--exclude=/packages/',
'--exclude=/distfiles/',
'--exclude=/local/',
'--exclude=CVS/',
'--exclude=.svn',
'--filter=H_**/files/digest-*',
self.settings['portdir'] + '/',
mytmp + '/' + self.settings['repo_name'] + '/'],
env=self.env)
log.notice('Compressing %s snapshot tarball ...', self.settings["repo_name"])
compressor = CompressMap(self.settings["compress_definitions"],
env=self.env, default_mode=self.settings['compression_mode'],
comp_prog=self.settings["comp_prog"])
infodict = compressor.create_infodict(
source=self.settings["repo_name"],
destination=self.settings["snapshot_path"],
basedir=mytmp,
filename=self.settings["snapshot_path"],
mode=self.settings["compression_mode"],
auto_extension=True
)
if not compressor.compress(infodict):
success = False
log.error('Snapshot compression failure')
else:
filename = '.'.join([self.settings["snapshot_path"],
compressor.extension(self.settings["compression_mode"])])
log.notice('Snapshot successfully written to %s', filename)
self.gen_contents_file(filename)
self.gen_digest_file(filename)
if "keepwork" not in self.settings["options"]:
self.cleanup()
if success:
log.info('snapshot: complete!')
return success
def clear_dir(target, mode=0o755, chg_flags=False, remove=False,
clear_nondir=True):
'''Universal directory clearing function
@target: string, path to be cleared or removed
@mode: integer, desired mode to set the directory to
@chg_flags: boolean used for FreeBSD hosts
@remove: boolean, passed through to clear_dir()
@return boolean
'''
log.debug('start: %s', target)
if not target:
log.debug('no target... returning')
return False
mystat = None
if os.path.isdir(target) and not os.path.islink(target):
log.notice('Emptying directory: %s', target)
# stat the dir, delete the dir, recreate the dir and set
# the proper perms and ownership
try:
log.debug('os.stat()')
mystat = os.stat(target)
# There's no easy way to change flags recursively in python
if chg_flags and os.uname()[0] == "FreeBSD":
cmd(['chflags', '-R', 'noschg', target])
log.debug('shutil.rmtree()')
shutil.rmtree(target)
except Exception:
log.error('clear_dir failed', exc_info=True)
return False
elif os.path.exists(target):
if clear_nondir:
log.debug("Clearing (unlinking) non-directory: %s", target)
os.unlink(target)
else:
log.info('clear_dir failed: %s: is not a directory', target)
return False
else:
log.debug("Condidtions not met to clear: %s", target)
log.debug(" isdir: %s", os.path.isdir(target))
log.debug(" islink: %s", os.path.islink(target))
log.debug(" exists: %s", os.path.exists(target))
if not remove:
log.debug('ensure_dirs()')
ensure_dirs(target, mode=mode)
if mystat:
os.chown(target, mystat[ST_UID], mystat[ST_GID])
os.chmod(target, mystat[ST_MODE])
log.debug('DONE, returning True')
return True
def base_dirs(self):
if os.uname()[0] == "FreeBSD":
# baselayout no longer creates the .keep files in proc and dev for FreeBSD as it
# would create them too late...we need them earlier before bind mounting filesystems
# since proc and dev are not writeable, so...create them here
ensure_dirs(self.settings["stage_path"] + "/proc")
ensure_dirs(self.settings["stage_path"] + "/dev")
for f in ('/proc', '/dev'):
f = self.settings['stage_path'] + f + '/.keep'
if not os.path.isfile(f):
try:
fileutils.touch(f)
except IOError:
log.error('Failed to create %s', f)
def base_dirs(self):
if os.uname()[0] == "FreeBSD":
# baselayout no longer creates the .keep files in proc and dev for FreeBSD as it
# would create them too late...we need them earlier before bind mounting filesystems
# since proc and dev are not writeable, so...create them here
ensure_dirs(self.settings["stage_path"]+"/proc")
ensure_dirs(self.settings["stage_path"]+"/dev")
for f in ('/proc', '/dev'):
f = self.settings['stage_path'] + f + '/.keep'
if not os.path.isfile(f):
try:
fileutils.touch(f)
except IOError:
log.error('Failed to create %s', f)
def disable(self, point):
'''Sets the resume point 'OFF'
@param point: string. name of the resume point to disable
@return boolean
'''
if point not in self._points:
return True
try:
os.unlink(self._points[point])
self._points.pop(point)
except Exception as e:
log.error('AutoResumeError: %s', e)
return False
return True
def disable(self, point):
'''Sets the resume point 'OFF'
@param point: string. name of the resume point to disable
@return boolean
'''
if point not in self._points:
return True
try:
os.unlink(self._points[point])
self._points.pop(point)
except Exception as e:
log.error('AutoResumeError: %s', e)
return False
return True
def run(self):
if "purgeonly" in self.settings["options"]:
self.purge()
return True
if "purge" in self.settings["options"]:
self.purge()
success = True
self.setup()
log.notice('Creating Portage tree snapshot %s from %s ...',
self.settings['version_stamp'], self.settings['portdir'])
mytmp=self.settings["tmp_path"]
ensure_dirs(mytmp)
target_snapshot = self.settings["portdir"] + "/ " + mytmp + "/%s/" % self.settings["repo_name"]
cmd("rsync -a --no-o --no-g --delete --exclude /packages/ --exclude /distfiles/ " +
"--exclude /local/ --exclude CVS/ --exclude .svn --filter=H_**/files/digest-* " +
target_snapshot,
"Snapshot failure", env=self.env)
log.notice('Compressing Portage snapshot tarball ...')
compressor = CompressMap(self.settings["compress_definitions"],
env=self.env, default_mode=self.settings['compression_mode'])
infodict = compressor.create_infodict(
source=self.settings["repo_name"],
destination=self.settings["snapshot_path"],
basedir=mytmp,
filename=self.settings["snapshot_path"],
mode=self.settings["compression_mode"],
auto_extension=True
)
if not compressor.compress(infodict):
success = False
log.error('Snapshot compression failure')
else:
filename = '.'.join([self.settings["snapshot_path"],
compressor.extension(self.settings["compression_mode"])])
log.notice('Snapshot successfully written to %s', filename)
self.gen_contents_file(filename)
self.gen_digest_file(filename)
self.cleanup()
if success:
log.info('snapshot: complete!')
return success
def get(self, point, no_lf=True):
'''Gets any data stored inside a resume point
@param point: string. name of the resume point to enable
@return data: string of information stored, or None
'''
if point in self._points:
try:
with open(self._points[point], 'r') as myf:
data = myf.read()
if data and no_lf:
data = data.replace('\n', ' ')
except OSError as e:
log.error('AutoResumeError: %s', e)
return None
return data
return None
def get(self, point, no_lf=True):
'''Gets any data stored inside a resume point
@param point: string. name of the resume point to enable
@return data: string of information stored, or None
'''
if point in self._points:
try:
with open(self._points[point], 'r') as myf:
data = myf.read()
if data and no_lf:
data = data.replace('\n', ' ')
except OSError as e:
log.error('AutoResumeError: %s', e)
return None
return data
return None
def clear_dir(target, mode=0o755, remove=False):
'''Universal directory clearing function
@target: string, path to be cleared or removed
@mode: integer, desired mode to set the directory to
@remove: boolean, passed through to clear_dir()
@return boolean
'''
log.debug('start: %s', target)
if not target:
log.debug('no target... returning')
return False
mystat = None
if os.path.isdir(target) and not os.path.islink(target):
log.notice('Emptying directory: %s', target)
# stat the dir, delete the dir, recreate the dir and set
# the proper perms and ownership
try:
log.debug('os.stat()')
mystat = os.stat(target)
log.debug('shutil.rmtree()')
shutil.rmtree(target)
except Exception:
log.error('clear_dir failed', exc_info=True)
return False
elif os.path.exists(target):
log.debug("Clearing (unlinking) non-directory: %s", target)
os.unlink(target)
else:
log.debug("Conditions not met to clear: %s", target)
log.debug(" isdir: %s", os.path.isdir(target))
log.debug(" islink: %s", os.path.islink(target))
log.debug(" exists: %s", os.path.exists(target))
if not remove:
log.debug('ensure_dirs()')
ensure_dirs(target, mode=mode)
if mystat:
os.chown(target, mystat[ST_UID], mystat[ST_GID])
os.chmod(target, mystat[ST_MODE])
log.debug('DONE, returning True')
return True
def run(self):
if self.settings['snapshot_treeish'] == 'stable':
treeish = self.update_ebuild_repo()
else:
treeish = self.settings['snapshot_treeish']
self.set_snapshot(treeish)
git_cmd = [
self.git, '-C', self.gitdir, 'archive', '--format=tar', treeish
]
tar2sqfs_cmd = [
command('tar2sqfs'),
str(self.snapshot), '-q', '-f', '-j1', '-c', 'gzip'
]
log.notice('Creating %s tree snapshot %s from %s',
self.settings['repo_name'], treeish, self.gitdir)
log.notice('>>> ' + ' '.join([*git_cmd, '|']))
log.notice(' ' + ' '.join(tar2sqfs_cmd))
lockfile = self.snapshot.with_suffix('.lock')
with write_lock(lockfile):
git = subprocess.Popen(git_cmd,
stdout=subprocess.PIPE,
stderr=sys.stderr,
close_fds=False)
tar2sqfs = subprocess.Popen(tar2sqfs_cmd,
stdin=git.stdout,
stdout=sys.stdout,
stderr=sys.stderr,
close_fds=False)
git.stdout.close()
git.wait()
tar2sqfs.wait()
if tar2sqfs.returncode == 0:
log.notice('Wrote snapshot to %s', self.snapshot)
else:
log.error('Failed to create snapshot')
return tar2sqfs.returncode == 0
def enable(self, point, data=None):
'''Sets the resume point 'ON'
@param point: string. name of the resume point to enable
@param data: string of information to store, or None
@return boolean
'''
if point in self._points and not data:
return True
fname = pjoin(self.basedir, point)
if data:
with open(fname,"w") as myf:
myf.write(data)
else:
try:
fileutils.touch(fname)
self._points[point] = fname
except Exception as e:
log.error('AutoResumeError: %s', e)
return False
return True
def move_path(src, dest):
'''Move a source target to a new destination
:param src: source path to move
:param dest: destination path to move it to
:returns: boolean
'''
log.debug('Start move_path(%s, %s)', src, dest)
if os.path.isdir(src) and not os.path.islink(src):
if os.path.exists(dest):
log.warning('Removing existing target destination: %s', dest)
if not clear_dir(dest, remove=True):
return False
log.debug('Moving source...')
try:
shutil.move(src, dest)
except Exception:
log.error('move_path failed', exc_info=True)
return False
return True
return False
def enable(self, point, data=None):
'''Sets the resume point 'ON'
@param point: string. name of the resume point to enable
@param data: string of information to store, or None
@return boolean
'''
if point in self._points and not data:
return True
fname = pjoin(self.basedir, point)
if data:
with open(fname, "w") as myf:
myf.write(data)
else:
try:
fileutils.touch(fname)
self._points[point] = fname
except Exception as e:
log.error('AutoResumeError: %s', e)
return False
return True
def move_path(src, dest):
'''Move a source target to a new destination
:param src: source path to move
:param dest: destination path to move it to
:returns: boolean
'''
log.debug('Start move_path(%s, %s)', src, dest)
if os.path.isdir(src) and not os.path.islink(src):
if os.path.exists(dest):
log.warning('Removing existing target destination: %s', dest)
if not clear_dir(dest, remove=True):
return False
log.debug('Moving source...')
try:
shutil.move(src, dest)
except Exception:
log.error('move_path failed', exc_info=True)
return False
return True
return False
def cmd(mycmd, myexc="", env=None, debug=False, fail_func=None):
if env is None:
env = {}
log.debug('cmd: %r', mycmd)
sys.stdout.flush()
args=[BASH_BINARY]
if "BASH_ENV" not in env:
env = env.copy()
env["BASH_ENV"] = "/etc/spork/is/not/valid/profile.env"
if debug:
args.append("-x")
args.append("-c")
args.append(mycmd)
log.debug('args: %r', args)
proc = Popen(args, env=env)
if proc.wait() != 0:
if fail_func:
log.error('CMD(), NON-Zero command return. Running fail_func().')
fail_func()
raise CatalystError("cmd() NON-zero return value from: %s" % myexc,
print_traceback=False)
def clean_stage1(self):
'''seedcache is enabled, so salvage the /tmp/stage1root,
remove the seed chroot'''
log.notice('Salvaging the stage1root from the chroot path ...')
# move the self.settings["stage_path"] outside of the self.settings["chroot_path"]
tmp_path = normpath(self.settings["storedir"] + "/tmp/" + "stage1root")
if move_path(self.settings["stage_path"], tmp_path):
self.remove_chroot()
# move it to self.settings["chroot_path"]
if not move_path(tmp_path, self.settings["chroot_path"]):
log.error(
'clean_stage1 failed, see previous log messages for details'
)
return False
log.notice(
'Successfully moved and cleaned the stage1root for the seedcache'
)
return True
log.error(
'clean_stage1 failed to move the stage1root to a temporary loation'
)
return False
def clear_dir(target, mode=0o755, chg_flags=False, remove=False):
'''Universal directory clearing function
@target: string, path to be cleared or removed
@mode: integer, desired mode to set the directory to
@chg_flags: boolean used for FreeBSD hoosts
@remove: boolean, passed through to clear_dir()
@return boolean
'''
log.debug('start: %s', target)
if not target:
log.debug('no target... returning')
return False
if os.path.isdir(target):
log.info('Emptying directory: %s', target)
# stat the dir, delete the dir, recreate the dir and set
# the proper perms and ownership
try:
log.debug('os.stat()')
mystat = os.stat(target)
# There's no easy way to change flags recursively in python
if chg_flags and os.uname()[0] == "FreeBSD":
os.system("chflags -R noschg " + target)
log.debug('shutil.rmtree()')
shutil.rmtree(target)
if not remove:
log.debug('ensure_dirs()')
ensure_dirs(target, mode=mode)
os.chown(target, mystat[ST_UID], mystat[ST_GID])
os.chmod(target, mystat[ST_MODE])
except Exception:
log.error('clear_dir failed', exc_info=True)
return False
else:
log.info('clear_dir failed: %s: is not a directory', target)
log.debug('DONE, returning True')
return True
def clear_dir(target, mode=0o755, chg_flags=False, remove=False):
'''Universal directory clearing function
@target: string, path to be cleared or removed
@mode: integer, desired mode to set the directory to
@chg_flags: boolean used for FreeBSD hoosts
@remove: boolean, passed through to clear_dir()
@return boolean
'''
log.debug('start: %s', target)
if not target:
log.debug('no target... returning')
return False
if os.path.isdir(target):
log.info('Emptying directory: %s', target)
# stat the dir, delete the dir, recreate the dir and set
# the proper perms and ownership
try:
log.debug('os.stat()')
mystat=os.stat(target)
# There's no easy way to change flags recursively in python
if chg_flags and os.uname()[0] == "FreeBSD":
os.system("chflags -R noschg " + target)
log.debug('shutil.rmtree()')
shutil.rmtree(target)
if not remove:
log.debug('ensure_dirs()')
ensure_dirs(target, mode=mode)
os.chown(target, mystat[ST_UID], mystat[ST_GID])
os.chmod(target, mystat[ST_MODE])
except Exception:
log.error('clear_dir failed', exc_info=True)
return False
else:
log.info('clear_dir failed: %s: is not a directory', target)
log.debug('DONE, returning True')
return True
def __init__(self, message, print_traceback=False):
if message:
log.error('CatalystError: %s', message, exc_info=print_traceback)
def __init__(self, message, print_traceback=False):
if message:
log.error('CatalystError: %s', message, exc_info=print_traceback)
