def cmd(mycmd, env=None, debug=False, fail_func=None):
"""Run the external |mycmd|.
If |mycmd| is a string, then it's assumed to be a bash snippet and will
be run through bash. Otherwise, it's a standalone command line and will
be run directly.
"""
log.debug('cmd: %r', mycmd)
sys.stdout.flush()
if env is None:
env = {}
if 'BASH_ENV' not in env:
env = env.copy()
env['BASH_ENV'] = '/etc/spork/is/not/valid/profile.env'
args = []
if isinstance(mycmd, str):
args.append(BASH_BINARY)
if debug:
args.append('-x')
args.extend(['-c', mycmd])
else:
args.extend(mycmd)
log.debug('args: %r', args)
proc = Popen(args, env=env)
ret = proc.wait()
if ret:
if fail_func:
log.error('cmd(%r) exited %s; running fail_func().', args, ret)
fail_func()
raise CatalystError('cmd(%r) exited %s' % (args, ret),
print_traceback=False)
def cmd(mycmd, env=None, debug=False, fail_func=None):
"""Run the external |mycmd|.
If |mycmd| is a string, then it's assumed to be a bash snippet and will
be run through bash. Otherwise, it's a standalone command line and will
be run directly.
"""
log.debug('cmd: %r', mycmd)
sys.stdout.flush()
if env is None:
env = {}
if 'BASH_ENV' not in env:
env = env.copy()
env['BASH_ENV'] = '/etc/spork/is/not/valid/profile.env'
args = []
if isinstance(mycmd, str):
args.append(BASH_BINARY)
if debug:
args.append('-x')
args.extend(['-c', mycmd])
else:
args.extend(mycmd)
log.debug('args: %r', args)
proc = Popen(args, env=env)
ret = proc.wait()
if ret:
if fail_func:
log.error('cmd(%r) exited %s; running fail_func().', args, ret)
fail_func()
raise CatalystError('cmd(%r) exited %s' % (args, ret),
print_traceback=False)
def move_path(src, dest):
'''Move a source target to a new destination
:param src: source path to move
:param dest: destination path to move it to
:returns: boolean
'''
log.debug('Start move_path(%s, %s)', src, dest)
if os.path.isdir(src) and not os.path.islink(src):
if os.path.exists(dest):
log.warning('Removing existing target destination: %s', dest)
if not clear_dir(dest, remove=True):
return False
log.debug('Moving source...')
try:
shutil.move(src, dest)
except Exception:
log.error('move_path failed', exc_info=True)
return False
return True
return False
def move_path(src, dest):
'''Move a source target to a new destination
:param src: source path to move
:param dest: destination path to move it to
:returns: boolean
'''
log.debug('Start move_path(%s, %s)', src, dest)
if os.path.isdir(src) and not os.path.islink(src):
if os.path.exists(dest):
log.warning('Removing existing target destination: %s', dest)
if not clear_dir(dest, remove=True):
return False
log.debug('Moving source...')
try:
shutil.move(src, dest)
except Exception:
log.error('move_path failed', exc_info=True)
return False
return True
return False
def cmd(mycmd, myexc="", env=None, debug=False, fail_func=None):
if env is None:
env = {}
log.debug('cmd: %r', mycmd)
sys.stdout.flush()
args=[BASH_BINARY]
if "BASH_ENV" not in env:
env = env.copy()
env["BASH_ENV"] = "/etc/spork/is/not/valid/profile.env"
if debug:
args.append("-x")
args.append("-c")
args.append(mycmd)
log.debug('args: %r', args)
proc = Popen(args, env=env)
if proc.wait() != 0:
if fail_func:
log.error('CMD(), NON-Zero command return. Running fail_func().')
fail_func()
raise CatalystError("cmd() NON-zero return value from: %s" % myexc,
print_traceback=False)
def calc_hash2(self, file_, hash_type):
'''
Calculate the hash for "file_"
@param file_: the file to generate the hash for
@param hash_: the hash algorythm to use
@returns the hash result
'''
_hash = self.hash_map[hash_type]
args = [_hash.cmd]
args.extend(_hash.args)
args.append(file_)
log.debug('args = %r', args)
source = Popen(args, stdout=PIPE)
output = source.communicate()
lines = output[0].decode('ascii').split('\n')
log.debug('output = %s', output)
header = lines[0]
h_f = lines[1].split()
hash_result = h_f[0]
short_file = os.path.split(h_f[1])[1]
result = header + "\n" + hash_result + " " + short_file + "\n"
log.info('%s (%s) = %s', header, short_file, result)
return result
def calc_hash2(self, file_, hash_type):
'''
Calculate the hash for "file_"
@param file_: the file to generate the hash for
@param hash_: the hash algorythm to use
@returns the hash result
'''
_hash = self.hash_map[hash_type]
args = [_hash.cmd]
args.extend(_hash.args)
args.append(file_)
log.debug('args = %r', args)
source = Popen(args, stdout=PIPE)
output = source.communicate()
lines = output[0].decode('ascii').split('\n')
log.debug('output = %s', output)
header = lines[0]
h_f = lines[1].split()
hash_result = h_f[0]
short_file = os.path.split(h_f[1])[1]
result = header + "\n" + hash_result + " " + short_file + "\n"
log.info('%s (%s) = %s', header, short_file, result)
return result
def clear_dir(target, mode=0o755, remove=False):
'''Universal directory clearing function
@target: string, path to be cleared or removed
@mode: integer, desired mode to set the directory to
@remove: boolean, passed through to clear_dir()
@return boolean
'''
log.debug('start: %s', target)
if not target:
log.debug('no target... returning')
return False
mystat = None
if os.path.isdir(target) and not os.path.islink(target):
log.notice('Emptying directory: %s', target)
# stat the dir, delete the dir, recreate the dir and set
# the proper perms and ownership
try:
log.debug('os.stat()')
mystat = os.stat(target)
log.debug('shutil.rmtree()')
shutil.rmtree(target)
except Exception:
log.error('clear_dir failed', exc_info=True)
return False
elif os.path.exists(target):
log.debug("Clearing (unlinking) non-directory: %s", target)
os.unlink(target)
else:
log.debug("Conditions not met to clear: %s", target)
log.debug(" isdir: %s", os.path.isdir(target))
log.debug(" islink: %s", os.path.islink(target))
log.debug(" exists: %s", os.path.exists(target))
if not remove:
log.debug('ensure_dirs()')
ensure_dirs(target, mode=mode)
if mystat:
os.chown(target, mystat[ST_UID], mystat[ST_GID])
os.chmod(target, mystat[ST_MODE])
log.debug('DONE, returning True')
return True
def clear_dir(target,
mode=0o755,
chg_flags=False,
remove=False,
clear_nondir=True):
'''Universal directory clearing function
@target: string, path to be cleared or removed
@mode: integer, desired mode to set the directory to
@chg_flags: boolean used for FreeBSD hosts
@remove: boolean, passed through to clear_dir()
@return boolean
'''
log.debug('start: %s', target)
if not target:
log.debug('no target... returning')
return False
mystat = None
if os.path.isdir(target) and not os.path.islink(target):
log.notice('Emptying directory: %s', target)
# stat the dir, delete the dir, recreate the dir and set
# the proper perms and ownership
try:
log.debug('os.stat()')
mystat = os.stat(target)
# There's no easy way to change flags recursively in python
if chg_flags and os.uname()[0] == "FreeBSD":
cmd(['chflags', '-R', 'noschg', target])
log.debug('shutil.rmtree()')
shutil.rmtree(target)
except Exception:
log.error('clear_dir failed', exc_info=True)
return False
elif os.path.exists(target):
if clear_nondir:
log.debug("Clearing (unlinking) non-directory: %s", target)
os.unlink(target)
else:
log.info('clear_dir failed: %s: is not a directory', target)
return False
else:
log.debug("Condidtions not met to clear: %s", target)
log.debug(" isdir: %s", os.path.isdir(target))
log.debug(" islink: %s", os.path.islink(target))
log.debug(" exists: %s", os.path.exists(target))
if not remove:
log.debug('ensure_dirs()')
ensure_dirs(target, mode=mode)
if mystat:
os.chown(target, mystat[ST_UID], mystat[ST_GID])
os.chmod(target, mystat[ST_MODE])
log.debug('DONE, returning True')
return True
def main(argv):
parser = get_parser()
opts = parser.parse_args(argv)
# Initialize the logger before anything else.
log_level = opts.log_level
if log_level is None:
if opts.debug:
log_level = 'debug'
elif opts.verbose:
log_level = 'info'
else:
log_level = 'notice'
log.setup_logging(log_level, output=opts.log_file, debug=opts.debug,
color=opts.color)
# Parse the command line options.
myconfigs = opts.configs
if not myconfigs:
myconfigs = [DEFAULT_CONFIG_FILE]
myspecfile = opts.file
mycmdline = opts.cli[:]
if opts.snapshot:
mycmdline.append('target=snapshot')
mycmdline.append('version_stamp=' + opts.snapshot)
conf_values['DEBUG'] = opts.debug
conf_values['VERBOSE'] = opts.debug or opts.verbose
options = set()
if opts.fetchonly:
options.add('fetch')
if opts.purge:
options.add('purge')
if opts.purgeonly:
options.add('purgeonly')
if opts.purgetmponly:
options.add('purgetmponly')
if opts.clear_autoresume:
options.add('clear-autoresume')
# Make sure we have some work before moving further.
if not myspecfile and not mycmdline:
parser.error('please specify one of either -f or -C or -s')
# made it this far so start by outputting our version info
version()
# import configuration file and import our main module using those settings
parse_config(myconfigs)
conf_values["options"].update(options)
log.debug('conf_values[options] = %s', conf_values['options'])
# initialize our contents generator
contents_map = ContentsMap(CONTENTS_DEFINITIONS)
conf_values["contents_map"] = contents_map
# initialze our hash and contents generators
hash_map = HashMap(HASH_DEFINITIONS)
conf_values["hash_map"] = hash_map
# initialize our (de)compression definitions
conf_values['decompress_definitions'] = DECOMPRESS_DEFINITIONS
conf_values['compress_definitions'] = COMPRESS_DEFINITIONS
# TODO add capability to config/spec new definitions
# Start checking that digests are valid now that hash_map is initialized
if "digests" in conf_values:
digests = set(conf_values['digests'].split())
valid_digests = set(HASH_DEFINITIONS.keys())
# Use the magic keyword "auto" to use all algos that are available.
skip_missing = False
if 'auto' in digests:
skip_missing = True
digests.remove('auto')
if not digests:
digests = set(valid_digests)
# First validate all the requested digests are valid keys.
if digests - valid_digests:
log.critical(
'These are not valid digest entries:\n'
'%s\n'
'Valid digest entries:\n'
'%s',
', '.join(digests - valid_digests),
', '.join(sorted(valid_digests)))
# Then check for any programs that the hash func requires.
for digest in digests:
try:
process.find_binary(hash_map.hash_map[digest].cmd)
except process.CommandNotFound:
# In auto mode, just ignore missing support.
if skip_missing:
digests.remove(digest)
continue
log.critical(
'The "%s" binary needed by digest "%s" was not found. '
'It needs to be in your system path.',
hash_map.hash_map[digest].cmd, digest)
# Now reload the config with our updated value.
conf_values['digests'] = ' '.join(digests)
if "hash_function" in conf_values:
if conf_values["hash_function"] not in HASH_DEFINITIONS:
log.critical(
'%s is not a valid hash_function entry\n'
'Valid hash_function entries:\n'
'%s', HASH_DEFINITIONS.keys())
try:
process.find_binary(hash_map.hash_map[conf_values["hash_function"]].cmd)
except process.CommandNotFound:
log.critical(
'The "%s" binary needed by hash_function "%s" was not found. '
'It needs to be in your system path.',
hash_map.hash_map[conf_values['hash_function']].cmd,
conf_values['hash_function'])
addlargs={}
if myspecfile:
spec = catalyst.config.SpecParser(myspecfile)
addlargs.update(spec.get_values())
if mycmdline:
try:
cmdline = catalyst.config.ConfigParser()
cmdline.parse_lines(mycmdline)
addlargs.update(cmdline.get_values())
except CatalystError:
log.critical('Could not parse commandline')
if "target" not in addlargs:
raise CatalystError("Required value \"target\" not specified.")
if os.getuid() != 0:
# catalyst cannot be run as a normal user due to chroots, mounts, etc
log.critical('This script requires root privileges to operate')
# everything is setup, so the build is a go
try:
success = build_target(addlargs)
except KeyboardInterrupt:
log.critical('Catalyst build aborted due to user interrupt (Ctrl-C)')
if not success:
sys.exit(2)
sys.exit(0)
def clear_dir(target, mode=0o755, chg_flags=False, remove=False):
'''Universal directory clearing function
@target: string, path to be cleared or removed
@mode: integer, desired mode to set the directory to
@chg_flags: boolean used for FreeBSD hoosts
@remove: boolean, passed through to clear_dir()
@return boolean
'''
log.debug('start: %s', target)
if not target:
log.debug('no target... returning')
return False
if os.path.isdir(target):
log.info('Emptying directory: %s', target)
# stat the dir, delete the dir, recreate the dir and set
# the proper perms and ownership
try:
log.debug('os.stat()')
mystat=os.stat(target)
# There's no easy way to change flags recursively in python
if chg_flags and os.uname()[0] == "FreeBSD":
os.system("chflags -R noschg " + target)
log.debug('shutil.rmtree()')
shutil.rmtree(target)
if not remove:
log.debug('ensure_dirs()')
ensure_dirs(target, mode=mode)
os.chown(target, mystat[ST_UID], mystat[ST_GID])
os.chmod(target, mystat[ST_MODE])
except Exception:
log.error('clear_dir failed', exc_info=True)
return False
else:
log.info('clear_dir failed: %s: is not a directory', target)
log.debug('DONE, returning True')
return True
def clear_dir(target, mode=0o755, chg_flags=False, remove=False,
clear_nondir=True):
'''Universal directory clearing function
@target: string, path to be cleared or removed
@mode: integer, desired mode to set the directory to
@chg_flags: boolean used for FreeBSD hosts
@remove: boolean, passed through to clear_dir()
@return boolean
'''
log.debug('start: %s', target)
if not target:
log.debug('no target... returning')
return False
mystat = None
if os.path.isdir(target) and not os.path.islink(target):
log.notice('Emptying directory: %s', target)
# stat the dir, delete the dir, recreate the dir and set
# the proper perms and ownership
try:
log.debug('os.stat()')
mystat = os.stat(target)
# There's no easy way to change flags recursively in python
if chg_flags and os.uname()[0] == "FreeBSD":
cmd(['chflags', '-R', 'noschg', target])
log.debug('shutil.rmtree()')
shutil.rmtree(target)
except Exception:
log.error('clear_dir failed', exc_info=True)
return False
elif os.path.exists(target):
if clear_nondir:
log.debug("Clearing (unlinking) non-directory: %s", target)
os.unlink(target)
else:
log.info('clear_dir failed: %s: is not a directory', target)
return False
else:
log.debug("Condidtions not met to clear: %s", target)
log.debug(" isdir: %s", os.path.isdir(target))
log.debug(" islink: %s", os.path.islink(target))
log.debug(" exists: %s", os.path.exists(target))
if not remove:
log.debug('ensure_dirs()')
ensure_dirs(target, mode=mode)
if mystat:
os.chown(target, mystat[ST_UID], mystat[ST_GID])
os.chmod(target, mystat[ST_MODE])
log.debug('DONE, returning True')
return True
def clear_dir(target, mode=0o755, chg_flags=False, remove=False):
'''Universal directory clearing function
@target: string, path to be cleared or removed
@mode: integer, desired mode to set the directory to
@chg_flags: boolean used for FreeBSD hoosts
@remove: boolean, passed through to clear_dir()
@return boolean
'''
log.debug('start: %s', target)
if not target:
log.debug('no target... returning')
return False
if os.path.isdir(target):
log.info('Emptying directory: %s', target)
# stat the dir, delete the dir, recreate the dir and set
# the proper perms and ownership
try:
log.debug('os.stat()')
mystat = os.stat(target)
# There's no easy way to change flags recursively in python
if chg_flags and os.uname()[0] == "FreeBSD":
os.system("chflags -R noschg " + target)
log.debug('shutil.rmtree()')
shutil.rmtree(target)
if not remove:
log.debug('ensure_dirs()')
ensure_dirs(target, mode=mode)
os.chown(target, mystat[ST_UID], mystat[ST_GID])
os.chmod(target, mystat[ST_MODE])
except Exception:
log.error('clear_dir failed', exc_info=True)
return False
else:
log.info('clear_dir failed: %s: is not a directory', target)
log.debug('DONE, returning True')
return True
