def parse_config(config_files):
for config_file in config_files:
log.notice('Loading configuration file: %s', config_file)
try:
with open(config_file, 'rb') as f:
config = tomli.load(f)
for key in config:
if key not in valid_config_file_values:
log.critical("Unknown option '%s' in config file %s", key,
config_file)
conf_values.update(config)
except Exception as e:
log.critical('Could not find parse configuration file: %s: %s',
config_file, e)
# print out any options messages
for opt in conf_values['options']:
if opt in option_messages:
log.info(option_messages[opt])
if "envscript" in conf_values:
log.info('Envscript support enabled.')
# take care of any variable substitutions that may be left
for x in list(conf_values):
if isinstance(conf_values[x], str):
conf_values[x] = conf_values[x] % conf_values
def import_module(target):
"""
import catalyst's own modules
(i.e. targets and the arch modules)
"""
try:
mod_name = "catalyst.targets." + target
module = __import__(mod_name, [], [], ["not empty"])
except ImportError:
log.critical('Python module import error: %s', target, exc_info=True)
return module
def import_module(target):
"""
import catalyst's own modules
(i.e. targets and the arch modules)
"""
try:
mod_name = "catalyst.targets." + target
module = __import__(mod_name, [],[], ["not empty"])
except ImportError:
log.critical('Python module import error: %s', target, exc_info=True)
return module
def parse_config(config_files):
# search a couple of different areas for the main config file
myconf = {}
# try and parse the config file "config_file"
for config_file in config_files:
log.notice('Loading configuration file: %s', config_file)
try:
config = catalyst.config.ConfigParser(config_file)
myconf.update(config.get_values())
except Exception as e:
log.critical('Could not find parse configuration file: %s: %s',
config_file, e)
# now, load up the values into conf_values so that we can use them
for x in list(confdefaults):
if x in myconf:
if x == 'options':
conf_values[x] = set(myconf[x].split())
elif x in ["decompressor_search_order"]:
conf_values[x] = myconf[x].split()
else:
conf_values[x] = myconf[x]
else:
conf_values[x] = confdefaults[x]
# add our python base directory to use for loading target arch's
conf_values["PythonDir"] = os.path.dirname(os.path.realpath(__file__))
# print out any options messages
for opt in conf_values['options']:
if opt in option_messages:
log.info(option_messages[opt])
for key in [
"digests", "envscript", "var_tmpfs_portage", "port_logdir",
"local_overlay"
]:
if key in myconf:
conf_values[key] = myconf[key]
if "contents" in myconf:
# replace '-' with '_' (for compatibility with existing configs)
conf_values["contents"] = myconf["contents"].replace("-", '_')
if "envscript" in myconf:
log.info('Envscript support enabled.')
# take care of any variable substitutions that may be left
for x in list(conf_values):
if isinstance(conf_values[x], str):
conf_values[x] = conf_values[x] % conf_values
def parse_config(config_files):
# search a couple of different areas for the main config file
myconf={}
# try and parse the config file "config_file"
for config_file in config_files:
log.notice('Loading configuration file: %s', config_file)
try:
config = catalyst.config.ConfigParser(config_file)
myconf.update(config.get_values())
except Exception as e:
log.critical('Could not find parse configuration file: %s: %s',
config_file, e)
# now, load up the values into conf_values so that we can use them
for x in list(confdefaults):
if x in myconf:
if x == 'options':
conf_values[x] = set(myconf[x].split())
elif x in ["decompressor_search_order"]:
conf_values[x] = myconf[x].split()
else:
conf_values[x]=myconf[x]
else:
conf_values[x]=confdefaults[x]
# add our python base directory to use for loading target arch's
conf_values["PythonDir"] = __selfpath__
# print out any options messages
for opt in conf_values['options']:
if opt in option_messages:
log.info(option_messages[opt])
for key in ["digests", "envscript", "var_tmpfs_portage", "port_logdir",
"local_overlay"]:
if key in myconf:
conf_values[key] = myconf[key]
if "contents" in myconf:
# replace '-' with '_' (for compatibility with existing configs)
conf_values["contents"] = myconf["contents"].replace("-", '_')
if "envscript" in myconf:
log.info('Envscript support enabled.')
# take care of any variable substitutions that may be left
for x in list(conf_values):
if isinstance(conf_values[x], str):
conf_values[x] = conf_values[x] % conf_values
def _main(parser, opts):
"""The "main" main function so we can trace/profile."""
# Initialize the logger before anything else.
log_level = opts.log_level
if log_level is None:
if opts.debug:
log_level = 'debug'
elif opts.verbose:
log_level = 'info'
else:
log_level = 'notice'
log.setup_logging(log_level,
output=opts.log_file,
debug=opts.debug,
color=opts.color)
# Parse the command line options.
myconfigs = opts.configs
if not myconfigs:
myconfigs = [DEFAULT_CONFIG_FILE]
myspecfile = opts.file
mycmdline = list()
if opts.snapshot:
mycmdline.append('target: snapshot')
mycmdline.append('snapshot_treeish: ' + opts.snapshot)
conf_values['DEBUG'] = opts.debug
conf_values['VERBOSE'] = opts.debug or opts.verbose
options = []
if opts.fetchonly:
options.append('fetch')
if opts.purge:
options.append('purge')
if opts.purgeonly:
options.append('purgeonly')
if opts.purgetmponly:
options.append('purgetmponly')
if opts.clear_autoresume:
options.append('clear-autoresume')
if opts.enter_chroot:
options.append('enter-chroot')
# Make sure we have some work before moving further.
if not myspecfile and not mycmdline:
parser.error('please specify one of either -f or -C or -s')
# made it this far so start by outputting our version info
version()
# import configuration file and import our main module using those settings
parse_config(myconfigs)
conf_values["options"].extend(options)
log.notice('conf_values[options] = %s', conf_values['options'])
# initialize our contents generator
contents_map = ContentsMap(CONTENTS_DEFINITIONS,
comp_prog=conf_values['comp_prog'],
decomp_opt=conf_values['decomp_opt'],
list_xattrs_opt=conf_values['list_xattrs_opt'])
conf_values["contents_map"] = contents_map
# initialize our (de)compression definitions
conf_values['decompress_definitions'] = DECOMPRESS_DEFINITIONS
conf_values['compress_definitions'] = COMPRESS_DEFINITIONS
# TODO add capability to config/spec new definitions
if "digests" in conf_values:
valid_digests = hashlib.algorithms_available
digests = set(conf_values['digests'])
conf_values['digests'] = digests
# First validate all the requested digests are valid keys.
if digests - valid_digests:
raise CatalystError('These are not valid digest entries:\n%s\n'
'Valid digest entries:\n%s' %
(', '.join(sorted(digests - valid_digests)),
', '.join(sorted(valid_digests))))
addlargs = {}
if myspecfile:
log.notice("Processing spec file: %s", myspecfile)
spec = catalyst.config.SpecParser(myspecfile)
addlargs.update(spec.get_values())
if mycmdline:
try:
cmdline = catalyst.config.SpecParser()
cmdline.parse_lines(mycmdline)
addlargs.update(cmdline.get_values())
except CatalystError:
log.critical('Could not parse commandline')
if "target" not in addlargs:
raise CatalystError("Required value \"target\" not specified.")
if os.getuid() != 0:
# catalyst cannot be run as a normal user due to chroots, mounts, etc
log.critical('This script requires root privileges to operate')
# Start off by creating unique namespaces to run in. Would be nice to
# use pid & user namespaces, but snakeoil's namespace module has signal
# transfer issues (CTRL+C doesn't propagate), and user namespaces need
# more work due to Gentoo build process (uses sudo/root/portage).
with namespace(uts=True, ipc=True, hostname='catalyst'):
# everything is setup, so the build is a go
try:
success = build_target(addlargs)
except KeyboardInterrupt:
log.critical(
'Catalyst build aborted due to user interrupt (Ctrl-C)')
if not success:
sys.exit(2)
sys.exit(0)
def _main(parser, opts):
"""The "main" main function so we can trace/profile."""
# Initialize the logger before anything else.
log_level = opts.log_level
if log_level is None:
if opts.debug:
log_level = 'debug'
elif opts.verbose:
log_level = 'info'
else:
log_level = 'notice'
log.setup_logging(log_level,
output=opts.log_file,
debug=opts.debug,
color=opts.color)
# Parse the command line options.
myconfigs = opts.configs
if not myconfigs:
myconfigs = [DEFAULT_CONFIG_FILE]
myspecfile = opts.file
mycmdline = opts.cli[:]
if opts.snapshot:
mycmdline.append('target=snapshot')
mycmdline.append('version_stamp=' + opts.snapshot)
conf_values['DEBUG'] = opts.debug
conf_values['VERBOSE'] = opts.debug or opts.verbose
options = set()
if opts.fetchonly:
options.add('fetch')
if opts.purge:
options.add('purge')
if opts.purgeonly:
options.add('purgeonly')
if opts.purgetmponly:
options.add('purgetmponly')
if opts.clear_autoresume:
options.add('clear-autoresume')
# Make sure we have some work before moving further.
if not myspecfile and not mycmdline:
parser.error('please specify one of either -f or -C or -s')
# made it this far so start by outputting our version info
version()
# import configuration file and import our main module using those settings
parse_config(myconfigs)
conf_values["options"].update(options)
log.notice('conf_values[options] = %s', conf_values['options'])
# initialize our contents generator
contents_map = ContentsMap(CONTENTS_DEFINITIONS,
comp_prog=conf_values['comp_prog'],
decomp_opt=conf_values['decomp_opt'],
list_xattrs_opt=conf_values['list_xattrs_opt'])
conf_values["contents_map"] = contents_map
# initialze our hash and contents generators
hash_map = HashMap(HASH_DEFINITIONS)
conf_values["hash_map"] = hash_map
# initialize our (de)compression definitions
conf_values['decompress_definitions'] = DECOMPRESS_DEFINITIONS
conf_values['compress_definitions'] = COMPRESS_DEFINITIONS
# TODO add capability to config/spec new definitions
# Start checking that digests are valid now that hash_map is initialized
if "digests" in conf_values:
digests = set(conf_values['digests'].split())
valid_digests = set(HASH_DEFINITIONS.keys())
# Use the magic keyword "auto" to use all algos that are available.
skip_missing = False
if 'auto' in digests:
skip_missing = True
digests.remove('auto')
if not digests:
digests = set(valid_digests)
# First validate all the requested digests are valid keys.
if digests - valid_digests:
log.critical(
'These are not valid digest entries:\n'
'%s\n'
'Valid digest entries:\n'
'%s', ', '.join(digests - valid_digests),
', '.join(sorted(valid_digests)))
# Then check for any programs that the hash func requires.
for digest in digests:
try:
process.find_binary(hash_map.hash_map[digest].cmd)
except process.CommandNotFound:
# In auto mode, just ignore missing support.
if skip_missing:
digests.remove(digest)
continue
log.critical(
'The "%s" binary needed by digest "%s" was not found. '
'It needs to be in your system path.',
hash_map.hash_map[digest].cmd, digest)
# Now reload the config with our updated value.
conf_values['digests'] = ' '.join(digests)
if "hash_function" in conf_values:
if conf_values["hash_function"] not in HASH_DEFINITIONS:
log.critical(
'%s is not a valid hash_function entry\n'
'Valid hash_function entries:\n'
'%s', conf_values["hash_function"], HASH_DEFINITIONS.keys())
try:
process.find_binary(
hash_map.hash_map[conf_values["hash_function"]].cmd)
except process.CommandNotFound:
log.critical(
'The "%s" binary needed by hash_function "%s" was not found. '
'It needs to be in your system path.',
hash_map.hash_map[conf_values['hash_function']].cmd,
conf_values['hash_function'])
# detect GNU sed
for sed in ('/usr/bin/gsed', '/bin/sed', '/usr/bin/sed'):
if os.path.exists(sed):
conf_values["sed"] = sed
break
addlargs = {}
if myspecfile:
log.notice("Processing spec file: %s", myspecfile)
spec = catalyst.config.SpecParser(myspecfile)
addlargs.update(spec.get_values())
if mycmdline:
try:
cmdline = catalyst.config.ConfigParser()
cmdline.parse_lines(mycmdline)
addlargs.update(cmdline.get_values())
except CatalystError:
log.critical('Could not parse commandline')
if "target" not in addlargs:
raise CatalystError("Required value \"target\" not specified.")
if os.getuid() != 0:
# catalyst cannot be run as a normal user due to chroots, mounts, etc
log.critical('This script requires root privileges to operate')
# Namespaces aren't supported on *BSDs at the moment. So let's check
# whether we're on Linux.
if os.uname().sysname in ["Linux", "linux"]:
# Start off by creating unique namespaces to run in. Would be nice to
# use pid & user namespaces, but snakeoil's namespace module has signal
# transfer issues (CTRL+C doesn't propagate), and user namespaces need
# more work due to Gentoo build process (uses sudo/root/portage).
namespaces.simple_unshare(mount=True,
uts=True,
ipc=True,
pid=False,
net=False,
user=False,
hostname='catalyst')
# everything is setup, so the build is a go
try:
success = build_target(addlargs)
except KeyboardInterrupt:
log.critical('Catalyst build aborted due to user interrupt (Ctrl-C)')
if not success:
sys.exit(2)
sys.exit(0)
def main(argv):
parser = get_parser()
opts = parser.parse_args(argv)
# Initialize the logger before anything else.
log_level = opts.log_level
if log_level is None:
if opts.debug:
log_level = 'debug'
elif opts.verbose:
log_level = 'info'
else:
log_level = 'notice'
log.setup_logging(log_level, output=opts.log_file, debug=opts.debug,
color=opts.color)
# Parse the command line options.
myconfigs = opts.configs
if not myconfigs:
myconfigs = [DEFAULT_CONFIG_FILE]
myspecfile = opts.file
mycmdline = opts.cli[:]
if opts.snapshot:
mycmdline.append('target=snapshot')
mycmdline.append('version_stamp=' + opts.snapshot)
conf_values['DEBUG'] = opts.debug
conf_values['VERBOSE'] = opts.debug or opts.verbose
options = set()
if opts.fetchonly:
options.add('fetch')
if opts.purge:
options.add('purge')
if opts.purgeonly:
options.add('purgeonly')
if opts.purgetmponly:
options.add('purgetmponly')
if opts.clear_autoresume:
options.add('clear-autoresume')
# Make sure we have some work before moving further.
if not myspecfile and not mycmdline:
parser.error('please specify one of either -f or -C or -s')
# made it this far so start by outputting our version info
version()
# import configuration file and import our main module using those settings
parse_config(myconfigs)
conf_values["options"].update(options)
log.debug('conf_values[options] = %s', conf_values['options'])
# initialize our contents generator
contents_map = ContentsMap(CONTENTS_DEFINITIONS)
conf_values["contents_map"] = contents_map
# initialze our hash and contents generators
hash_map = HashMap(HASH_DEFINITIONS)
conf_values["hash_map"] = hash_map
# initialize our (de)compression definitions
conf_values['decompress_definitions'] = DECOMPRESS_DEFINITIONS
conf_values['compress_definitions'] = COMPRESS_DEFINITIONS
# TODO add capability to config/spec new definitions
# Start checking that digests are valid now that hash_map is initialized
if "digests" in conf_values:
digests = set(conf_values['digests'].split())
valid_digests = set(HASH_DEFINITIONS.keys())
# Use the magic keyword "auto" to use all algos that are available.
skip_missing = False
if 'auto' in digests:
skip_missing = True
digests.remove('auto')
if not digests:
digests = set(valid_digests)
# First validate all the requested digests are valid keys.
if digests - valid_digests:
log.critical(
'These are not valid digest entries:\n'
'%s\n'
'Valid digest entries:\n'
'%s',
', '.join(digests - valid_digests),
', '.join(sorted(valid_digests)))
# Then check for any programs that the hash func requires.
for digest in digests:
try:
process.find_binary(hash_map.hash_map[digest].cmd)
except process.CommandNotFound:
# In auto mode, just ignore missing support.
if skip_missing:
digests.remove(digest)
continue
log.critical(
'The "%s" binary needed by digest "%s" was not found. '
'It needs to be in your system path.',
hash_map.hash_map[digest].cmd, digest)
# Now reload the config with our updated value.
conf_values['digests'] = ' '.join(digests)
if "hash_function" in conf_values:
if conf_values["hash_function"] not in HASH_DEFINITIONS:
log.critical(
'%s is not a valid hash_function entry\n'
'Valid hash_function entries:\n'
'%s', HASH_DEFINITIONS.keys())
try:
process.find_binary(hash_map.hash_map[conf_values["hash_function"]].cmd)
except process.CommandNotFound:
log.critical(
'The "%s" binary needed by hash_function "%s" was not found. '
'It needs to be in your system path.',
hash_map.hash_map[conf_values['hash_function']].cmd,
conf_values['hash_function'])
addlargs={}
if myspecfile:
spec = catalyst.config.SpecParser(myspecfile)
addlargs.update(spec.get_values())
if mycmdline:
try:
cmdline = catalyst.config.ConfigParser()
cmdline.parse_lines(mycmdline)
addlargs.update(cmdline.get_values())
except CatalystError:
log.critical('Could not parse commandline')
if "target" not in addlargs:
raise CatalystError("Required value \"target\" not specified.")
if os.getuid() != 0:
# catalyst cannot be run as a normal user due to chroots, mounts, etc
log.critical('This script requires root privileges to operate')
# everything is setup, so the build is a go
try:
success = build_target(addlargs)
except KeyboardInterrupt:
log.critical('Catalyst build aborted due to user interrupt (Ctrl-C)')
if not success:
sys.exit(2)
sys.exit(0)
