def _install_sdk_to_prefix(self, sdkname):
"""
Read recipe for sdkname, and install the SDK to the prefix.
"""
from pybombs import recipe
src_dir = self.prefix.src_dir
cfg_file = self.prefix.cfg_file
### Get the recipe
r = recipe.get_recipe(sdkname, target='sdk')
try:
self.log.trace("Switching CWD to {0}".format(src_dir))
if not op.isdir(src_dir):
os.mkdir(src_dir)
os.chdir(src_dir)
except:
self.log.error("Source dir required to install SDK.")
return False
### Install the actual SDK file
self.log.debug("Fetching SDK `{sdk}'".format(sdk=sdkname))
fetcher.Fetcher().fetch(r)
self.log.info("Installing SDK `{sdk}'".format(sdk=sdkname))
# Install command
cmd = r.var_replace_all(r.get_command('install'))
if subproc.monitor_process(cmd, shell=True, env=os.environ) == 0:
self.log.debug("Installation successful")
else:
self.log.error("Error installing SDK. Aborting.")
return False
# Clean up
files_to_delete = [op.normpath(op.join(src_dir, r.var_replace_all(x))) for x in r.clean]
if len(files_to_delete):
self.log.info("Cleaning up files...")
for ftd in files_to_delete:
if op.commonprefix((src_dir, ftd)) != src_dir:
self.log.warn("Not removing {ftd} -- outside source dir!".format(ftd=ftd))
continue
self.log.debug("Removing {ftd}...".format(ftd=ftd))
if op.isdir(ftd):
shutil.rmtree(ftd)
elif op.isfile(ftd):
os.remove(ftd)
else:
self.log.error("Not sure what this is: {ftd}".format(ftd=ftd))
return False
### Update the prefix-local config file
self.log.debug("Updating config file with SDK recipe info.")
try:
old_cfg_data = PBConfigFile(cfg_file).get()
except IOError:
self.log.debug("There doesn't seem to be a config file yet for this prefix.")
old_cfg_data = {}
# Filter out keys we don't care about:
sdk_recipe_keys_for_config = ('config', 'packages', 'categories', 'env')
sdk_cfg_data = {k: v for k, v in iteritems(r.get_dict()) if k in sdk_recipe_keys_for_config}
self.log.trace("New data: {new}".format(new=sdk_cfg_data))
cfg_data = dict_merge(old_cfg_data, sdk_cfg_data)
self.log.debug("Writing updated prefix config to `{0}'".format(cfg_file))
PBConfigFile(cfg_file).save(cfg_data)
return True
def remove_recipe_dir(self, alias):
"""
Remove a recipe alias and, if applicable, its cache.
"""
if not alias in self.cfg.get_named_recipe_dirs():
#self.log.error("Unknown recipe alias: {alias}".format(alias=alias))
remove_fail = 'Could not remove the recipe directory'
self.color_strips(remove_fail, 'red')
return False
# Remove from config file
cfg_filename = self.cfg.get_named_recipe_cfg_file(alias)
cfg_file = PBConfigFile(cfg_filename)
cfg_data = cfg_file.get()
cfg_data['recipes'].pop(alias, None)
cfg_file.save(cfg_data)
recipe_cache_dir = os.path.join(
os.path.split(cfg_filename)[0],
self.cfg.recipe_cache_dir,
alias,
)
# If the recipe cache is not inside a PyBOMBS dir, don't delete it.
if self.cfg.pybombs_dir not in recipe_cache_dir:
remove_success = 'Successfully removed the recipe directory'
self.color_strips(remove_success, 'blue')
return True
if os.path.exists(recipe_cache_dir):
#self.log.info("Removing directory: {cdir}".format(cdir=recipe_cache_dir))
shutil.rmtree(recipe_cache_dir)
remove_success = 'Successfully removed the recipe directory'
self.color_strips(remove_success, 'blue')
return True
def _remove_recipes(self):
"""
Remove a recipe alias and, if applicable, its cache.
"""
if not self.cfg.get_named_recipe_dirs().has_key(self.args.alias):
self.log.error(
"Unknown recipe alias: {alias}".format(alias=self.args.alias))
return -1
# Remove from config file
cfg_filename = self.cfg.get_named_recipe_cfg_file(self.args.alias)
cfg_file = PBConfigFile(cfg_filename)
cfg_data = cfg_file.get()
cfg_data['recipes'].pop(self.args.alias, None)
cfg_file.save(cfg_data)
recipe_cache_dir = os.path.join(
os.path.split(cfg_filename)[0],
self.cfg.recipe_cache_dir,
self.args.alias,
)
# If the recipe cache is not inside a PyBOMBS dir, don't delete it.
if self.cfg.pybombs_dir not in recipe_cache_dir:
return
if os.path.exists(recipe_cache_dir):
self.log.info(
"Removing directory: {cdir}".format(cdir=recipe_cache_dir))
shutil.rmtree(recipe_cache_dir)
def _run_pkg_or_cat(self):
" Handle `pybombs config --package or --category' "
if self.args.package is not None and self.args.category is not None:
self.log.error("Cannot provide both --package and --category!")
return -1
if self.args.package is not None:
config_target = 'packages'
pkg_cat = self.args.package
elif self.args.category is not None:
config_target = 'categories'
pkg_cat = self.args.category
cfg_file = PBConfigFile(self.cfg_file)
keys = [self.args.key]
verb = "Getting"
if self.args.key is None:
keys = cfg_file.data.get(config_target, {}).get(pkg_cat, {}).keys()
elif self.args.value is not None:
cfg_file.update(
{config_target: {
pkg_cat: {
self.args.key: self.args.value
}
}})
verb = "Setting"
print("{verb} keys for {target} {pkgcat}:".format(verb=verb,
target=config_target,
pkgcat=pkg_cat))
for key in keys:
print("{key}: {value}".format(key=key,
value=cfg_file.data.get(
config_target,
{}).get(pkg_cat, {}).get(key)))
def load(self):
"""
Load the inventory file.
If the file does not exist, initialize the internal content
with an empty dictionary.
This will override any internal state.
"""
self.log.debug("Trying to load inventory file {}...".format(self._filename))
self._invfile = PBConfigFile(self._filename)
def _merge_config_info_from_file(self, cfg_file, cfg_data):
"""
Load a config file, load its contents, and merge it into cfg_info.
Return the result.
"""
try:
self.log.debug('Inspecting config file: {0}'.format(cfg_file))
cfg_data_new = PBConfigFile(cfg_file).get()
except Exception:
self.log.debug('Well, looks like that failed.')
return cfg_data
return dict_merge(cfg_data, cfg_data_new)
def extract_cfg_items(filename, section, throw_ex=True):
"""
Read section from a config file and return it as a dict.
Will throw KeyError if section does not exist.
"""
cfg_data = PBConfigFile(filename).get() or {}
try:
return cfg_data[section]
except KeyError as e:
if throw_ex:
raise e
return {}
def _load_recipe_from_file(self, filename):
"""
Turn a .lwr file into a valid recipe datastructure.
"""
data = PBConfigFile(filename).get()
# Make sure dependencies is always a valid list:
if 'depends' in data and data['depends'] is not None:
if not isinstance(data['depends'], list):
data['depends'] = [data['depends'], ]
else:
data['depends'] = []
return data
def _run_env(self):
" Handle `pybombs config --env` "
cfg_file = PBConfigFile(self.cfg_file)
keys = [self.args.key]
verb = "Getting"
config_target = 'env'
if self.args.key is None:
keys = cfg_file.data.get(config_target, {}).keys()
elif self.args.value is not None:
cfg_file.update({config_target: {self.args.key: self.args.value}})
verb = "Setting"
print("{verb} environment variables:".format(verb=verb))
for key in keys:
print("{key}: {value}".format(key=key,
value=cfg_file.data.get(
config_target, {}).get(key)))
def _lint_recipe(self, recipe_file):
"""
Check if recipe_file is a valid recipe
"""
print("Linting recipe `{0}'".format(recipe_file))
# Basic file checks
try:
recipe_dict = PBConfigFile(recipe_file).get()
except IOError:
self.log.error("Can't open `{0}'".format(recipe_file))
return -1
except AttributeError:
self.log.error(
"Can't parse contents of file `{0}'".format(recipe_file))
return -1
if not isinstance(recipe_dict, dict):
self.log.error("Invalid recipe file. Not a dict.")
return -1
# Try loading as recipe
try:
rec = recipe.Recipe(recipe_file)
if not hasattr(rec, 'satisfy'):
print("[HMM] - No satisfy rules declared")
else:
for pkgtype in rec.satisfy.keys():
rec.get_package_reqs(pkgtype)
except PBException as ex:
print("[VERY BAD] - Recipe error: " + str(ex))
# Check keys
key_check = {
'HMM': ['source', 'depends'],
'BAD': ['inherit', 'category'],
}
for err_type, key_list in iteritems(key_check):
for key in key_list:
if not key in recipe_dict:
print("[{err}] Recipe doesn't have key: {key}".format(
err=err_type, key=key))
# Check dependencies is a list
dependencies = recipe_dict.get('depends', [])
if not isinstance(dependencies, list):
print("[VERY BAD] Dependencies is not a list: {}".format(
dependencies))
def _append_cfg_from_file(self, cfg_filename):
"""
Load file filename, interpret it as a config file
and append to cfg_cascade
Returns True if loading the config file was successful.
"""
self.log.debug(
"Reading config info from file: {0}".format(cfg_filename))
try:
cfg_data = PBConfigFile(cfg_filename).get()
except Exception as e:
self.log.debug("Parsing config file failed ({cfgf}).".format(
cfgf=cfg_filename))
self.cfg_cascade.append({})
return False
config_items = cfg_data.get('config', {})
self.log.debug('New config items: {items}'.format(items=config_items))
self.cfg_cascade.append(config_items)
return True
def update_cfg_file(self, new_data, cfg_file=None):
"""
Write new data to a config file.
If no config file is specified, the local config file
is used (e.g., on Linux, the one in ~/.pybombs/).
"""
if cfg_file is None:
cfg_file = self.local_cfg
if not os.path.isfile(cfg_file):
self.log.info("Creating new config file {0}".format(cfg_file))
path = os.path.split(cfg_file)[0]
if not os.path.isdir(path):
os.mkdir(path)
self.log.trace("Updating file {0} with new data: {1}".format(
cfg_file, new_data))
try:
return PBConfigFile(cfg_file).update(new_data)
except IOError:
self.log.error("Error opening config file {0}.".format(cfg_file))
return {}
def _run_config(self):
" Handle `pybombs config' "
if self.args.config_only:
cfg_data = PBConfigFile(self.cfg_file).get('config')
else:
cfg_data = self.cfg
keys = [self.args.key]
if self.args.key is None:
keys = cfg_data.keys()
elif self.args.value is not None:
self.cfg.set(self.args.key, self.args.value)
self.cfg.update_cfg_file(
new_data={'config': {
self.args.key: self.args.value
}},
cfg_file=self.cfg_file,
)
for key in keys:
print("{key}: {value}".format(key=key, value=cfg_data.get(key,
"")))
print(" - {help}".format(
help=self.cfg.get_help(key) or " Undocumented config option"))
def __init__(self, args, cfg_list, select_prefix=None):
self.log = pb_logging.logger.getChild("ConfigManager.PrefixInfo")
self.prefix_dir = None
self.prefix_cfg_dir = None
self.prefix_src = None
self.alias = None
self.src_dir = None
self.cfg_file = None
self.inv_file = None
self.inventory = None
self.recipe_dir = None
self.target_dir = None
self.env = os.environ.copy()
self.is_virtualenv = False
self._cfg_info = OrderedDict(self.default_config_info)
if select_prefix is not None:
args.prefix = select_prefix
# 1) Load the config info
for cfg_file in reversed(cfg_list):
self._cfg_info = \
self._merge_config_info_from_file(cfg_file, self._cfg_info)
# 2) Find the prefix directory
self._find_prefix_dir(args)
if self.prefix_dir is None:
self.log.debug("Cannot establish a prefix directory. This may cause issues down the line.")
self._set_attrs()
return
assert self.prefix_dir is not None
if self.alias is not None and self.alias in self._cfg_info['prefix_config_dir']:
self.prefix_cfg_dir = npath(self._cfg_info['prefix_config_dir'][self.alias])
self.log.debug("Choosing prefix config dir from alias: {0}".format(self.prefix_cfg_dir))
elif self.prefix_dir in self._cfg_info['prefix_config_dir']:
self.prefix_cfg_dir = npath(self._cfg_info['prefix_config_dir'][self.prefix_dir])
self.log.debug("Choosing prefix config dir from path lookup in prefix_config_dir: {0}".format(self.prefix_cfg_dir))
else:
self.prefix_cfg_dir = npath(os.path.join(self.prefix_dir, self.prefix_conf_dir))
self.log.debug("Choosing default prefix config dir: {0}".format(self.prefix_cfg_dir))
if not os.path.isdir(self.prefix_cfg_dir):
self.log.debug("Config dir does not yet exist.")
self.is_virtualenv = sysutils.is_virtualenv(self.prefix_dir)
if self.is_virtualenv:
self.log.info("Prefix is a Python virtualenv.")
# 3) Find the config file
self.cfg_file = npath(os.path.join(self.prefix_cfg_dir, ConfigManager.cfg_file_name))
config_section = {}
if not os.path.isfile(self.cfg_file):
self.log.debug("Prefix configuration file not found: {0}, assuming empty.".format(self.cfg_file))
else:
config_section = PBConfigFile(self.cfg_file).get('config')
self._cfg_info = self._merge_config_info_from_file(self.cfg_file, self._cfg_info)
# 4) Find the src dir
self.src_dir = npath(config_section.get('srcdir', os.path.join(self.prefix_dir, self.src_dir_name)))
self.log.debug("Prefix source dir is: {0}".format(self.src_dir))
if not os.path.isdir(self.src_dir):
self.log.debug("Source dir does not exist.")
# 5) Find the inventory file
self.inv_file = npath(os.path.join(self.prefix_cfg_dir, self.inv_file_name))
if not os.path.isfile(self.inv_file):
self.log.debug("Prefix inventory file not found: {0}".format(self.inv_file))
self.inventory = inventory.Inventory(inventory_file=self.inv_file)
# 6) Prefix-specific recipes. There's two places for these:
# - A 'recipes/' subdirectory
# - Anything declared in the config.yml file inside the prefix
self.recipe_dir = npath(config_section.get('recipes', os.path.join(self.prefix_cfg_dir, 'recipes')))
if os.path.isdir(self.recipe_dir):
self.log.debug("Prefix-local recipe dir is: {0}".format(self.recipe_dir))
else:
self.recipe_dir = None
# 7) Load environment
# If there's a setup_env option in the current config file, we use that
if self.setup_env_key in config_section:
self.log.debug('Loading environment from shell script: {0}'.format(config_section[self.setup_env_key]))
self.env = self._load_environ_from_script(config_section[self.setup_env_key])
else:
self.env = self._load_default_env(self.env)
# Set env vars that we always need
self.env[self.env_prefix_var] = self.prefix_dir
self.env[self.env_srcdir_var] = self.src_dir
# env: sections are always respected:
OLD_ENV = os.environ # Bit of an ugly hack to allow use of
os.environ = self.env # os.path.expandvars() on self.env
for k, v in iteritems(self._cfg_info['env']):
self.env[k.upper()] = os.path.expandvars(v.strip())
os.environ = OLD_ENV
# 8) Keep relevant config sections as attributes
self._set_attrs()
def load(self, select_prefix=None):
"""
Load the actual configuration. We put this outside the ctor so we can
reload on the same object. In that case, anything unsaved is reset!
"""
## Get command line args:
parser = argparse.ArgumentParser(add_help=False)
self.setup_parser(parser)
args = parser.parse_known_args()[0]
cfg_files = []
## Set verbosity level:
verb_offset = args.verbose - args.quiet
verb_level = pb_logging.default_log_level - 10 * verb_offset
if verb_level < pb_logging.OBNOXIOUS:
verb_level = pb_logging.OBNOXIOUS
pb_logging.logger.setLevel(verb_level)
self.yes = args.yes
## Set up logger:
self.log = pb_logging.logger.getChild("ConfigManager")
## Setup cfg_cascade:
# self.cfg_cascade is a list of dicts. The higher the index,
# the more important the dict.
# Zeroth layer: The default values.
self.cfg_cascade = [{k: v[0] for k, v in iteritems(self.defaults)},]
# Global defaults
global_cfg = os.path.join(self.global_base_dir, self.cfg_file_name)
if self._append_cfg_from_file(global_cfg):
cfg_files.insert(0, global_cfg)
# Home directory:
self.local_cfg_dir = self.get_pybombs_dir()
if not os.path.isdir(self.local_cfg_dir):
try:
self.log.debug("Creating local config dir {0}".format(self.local_cfg_dir))
os.mkdir(self.local_cfg_dir)
except (IOError, OSError):
self.log.debug("Failed.")
self.local_cfg = os.path.join(self.local_cfg_dir, self.cfg_file_name)
if self._append_cfg_from_file(self.local_cfg):
cfg_files.insert(0, self.local_cfg)
# Current prefix (don't know that yet -- so skip for now)
self.cfg_cascade.append({})
# Config file specified on command line:
if args.config_file is not None:
self._append_cfg_from_file(npath(args.config_file))
cfg_files.insert(0, npath(args.config_file))
else:
self.cfg_cascade.append({})
# Config args specified on command line:
cmd_line_opts = {}
for opt in args.config:
k, v = opt.split('=', 1)
cmd_line_opts[k] = v
self.cfg_cascade.append(cmd_line_opts)
# Append an empty one. This is what we use when set() is called
# to change settings at runtime.
self.cfg_cascade.append({})
# After this, no more dicts should be appended to cfg_cascade.
assert len(self.cfg_cascade) == self.LAYER_VOLATILE + 1
# Find recipe templates:
self._template_dir = os.path.join(self.module_dir, 'templates')
self.log.debug("Template directory: {0}".format(self._template_dir))
## Init prefix:
self._prefix_info = PrefixInfo(args, cfg_files, select_prefix)
# Add the prefix config file (if it exists)
prefix_config = self._prefix_info.cfg_file
if prefix_config is not None and os.path.exists(prefix_config):
cfg_files.insert(0, prefix_config)
self._append_cfg_from_file(prefix_config, self.LAYER_PREFIX)
## Init recipe-lists:
# Go through cfg files, then env variable, then command line args
self._recipe_locations = []
self._named_recipe_dirs = {}
self._named_recipe_sources = {}
self._named_recipe_cfg_files = {}
# From command line:
for r_loc in args.recipes:
if r_loc:
self._recipe_locations.append(npath(r_loc))
# From environment variable:
if len(os.environ.get("PYBOMBS_RECIPE_DIR", "").strip()):
self._recipe_locations += [
npath(x) \
for x in os.environ.get("PYBOMBS_RECIPE_DIR").split(os.pathsep) \
if len(x.strip())
]
# From prefix info:
if self._prefix_info.recipe_dir is not None:
self._recipe_locations.append(self._prefix_info.recipe_dir)
# From config files (from here, recipe locations are named):
for cfg_file in cfg_files:
recipe_locations = PBConfigFile(cfg_file).get('recipes')
for name, uri in reversed(recipe_locations.items()):
local_recipe_dir = self.resolve_recipe_uri(
uri, name, os.path.join(os.path.split(cfg_file)[0], 'recipes')
)
self._recipe_locations.append(local_recipe_dir)
self._named_recipe_dirs[name] = local_recipe_dir
self._named_recipe_sources[name] = uri
self._named_recipe_cfg_files[name] = cfg_file
# Internal recipe list:
self._recipe_locations.append(os.path.join(self.module_dir, 'recipes'))
self.log.debug("Full list of recipe locations: {0}".format(self._recipe_locations))
self.log.debug("Named recipe locations: {0}".format(self._named_recipe_sources))
