def _remove_recipes(self):
"""
Remove a recipe alias and, if applicable, its cache.
"""
if not self.cfg.get_named_recipe_dirs().has_key(self.args.alias):
self.log.error(
"Unknown recipe alias: {alias}".format(alias=self.args.alias))
return -1
# Remove from config file
cfg_filename = self.cfg.get_named_recipe_cfg_file(self.args.alias)
cfg_file = PBConfigFile(cfg_filename)
cfg_data = cfg_file.get()
cfg_data['recipes'].pop(self.args.alias, None)
cfg_file.save(cfg_data)
recipe_cache_dir = os.path.join(
os.path.split(cfg_filename)[0],
self.cfg.recipe_cache_dir,
self.args.alias,
)
# If the recipe cache is not inside a PyBOMBS dir, don't delete it.
if self.cfg.pybombs_dir not in recipe_cache_dir:
return
if os.path.exists(recipe_cache_dir):
self.log.info(
"Removing directory: {cdir}".format(cdir=recipe_cache_dir))
shutil.rmtree(recipe_cache_dir)
def remove_recipe_dir(self, alias):
"""
Remove a recipe alias and, if applicable, its cache.
"""
if not alias in self.cfg.get_named_recipe_dirs():
self.log.error("Unknown recipe alias: {alias}".format(alias=alias))
return False
# Remove from config file
cfg_filename = self.cfg.get_named_recipe_cfg_file(alias)
cfg_file = PBConfigFile(cfg_filename)
cfg_data = cfg_file.get()
cfg_data['recipes'].pop(alias, None)
cfg_file.save(cfg_data)
recipe_cache_dir = os.path.join(
os.path.split(cfg_filename)[0],
self.cfg.recipe_cache_dir,
alias,
)
# If the recipe cache is not inside a PyBOMBS dir, don't delete it.
if self.cfg.pybombs_dir not in recipe_cache_dir:
return True
if os.path.exists(recipe_cache_dir):
self.log.info("Removing directory: {cdir}".format(cdir=recipe_cache_dir))
shutil.rmtree(recipe_cache_dir)
return True
def remove_recipe_dir(self, alias):
"""
Remove a recipe alias and, if applicable, its cache.
"""
if not alias in self.cfg.get_named_recipe_dirs():
#self.log.error("Unknown recipe alias: {alias}".format(alias=alias))
remove_fail = 'Could not remove the recipe directory'
self.color_strips(remove_fail, 'red')
return False
# Remove from config file
cfg_filename = self.cfg.get_named_recipe_cfg_file(alias)
cfg_file = PBConfigFile(cfg_filename)
cfg_data = cfg_file.get()
cfg_data['recipes'].pop(alias, None)
cfg_file.save(cfg_data)
recipe_cache_dir = os.path.join(
os.path.split(cfg_filename)[0],
self.cfg.recipe_cache_dir,
alias,
)
# If the recipe cache is not inside a PyBOMBS dir, don't delete it.
if self.cfg.pybombs_dir not in recipe_cache_dir:
remove_success = 'Successfully removed the recipe directory'
self.color_strips(remove_success, 'blue')
return True
if os.path.exists(recipe_cache_dir):
#self.log.info("Removing directory: {cdir}".format(cdir=recipe_cache_dir))
shutil.rmtree(recipe_cache_dir)
remove_success = 'Successfully removed the recipe directory'
self.color_strips(remove_success, 'blue')
return True
def _run_pkg_or_cat(self):
" Handle `pybombs config --package or --category' "
if self.args.package is not None and self.args.category is not None:
self.log.error("Cannot provide both --package and --category!")
return -1
if self.args.package is not None:
config_target = 'packages'
pkg_cat = self.args.package
elif self.args.category is not None:
config_target = 'categories'
pkg_cat = self.args.category
cfg_file = PBConfigFile(self.cfg_file)
keys = [self.args.key]
verb = "Getting"
if self.args.key is None:
keys = cfg_file.data.get(config_target, {}).get(pkg_cat, {}).keys()
elif self.args.value is not None:
cfg_file.update({config_target: {pkg_cat: {self.args.key: self.args.value}}})
verb = "Setting"
print("{verb} keys for {target} {pkgcat}:".format(verb=verb, target=config_target, pkgcat=pkg_cat))
for key in keys:
print("{key}: {value}".format(
key=key,
value=cfg_file.data.get(config_target, {}).get(pkg_cat, {}).get(key))
)
def _run_pkg_or_cat(self):
" Handle `pybombs config --package or --category' "
if self.args.package is not None and self.args.category is not None:
self.log.error("Cannot provide both --package and --category!")
return -1
if self.args.package is not None:
config_target = 'packages'
pkg_cat = self.args.package
elif self.args.category is not None:
config_target = 'categories'
pkg_cat = self.args.category
cfg_file = PBConfigFile(self.cfg_file)
keys = [self.args.key]
verb = "Getting"
if self.args.key is None:
keys = cfg_file.data.get(config_target, {}).get(pkg_cat, {}).keys()
elif self.args.value is not None:
cfg_file.update(
{config_target: {
pkg_cat: {
self.args.key: self.args.value
}
}})
verb = "Setting"
print("{verb} keys for {target} {pkgcat}:".format(verb=verb,
target=config_target,
pkgcat=pkg_cat))
for key in keys:
print("{key}: {value}".format(key=key,
value=cfg_file.data.get(
config_target,
{}).get(pkg_cat, {}).get(key)))
def _install_sdk_to_prefix(self, sdkname):
"""
Read recipe for sdkname, and install the SDK to the prefix.
"""
from pybombs import recipe
src_dir = self.prefix.src_dir
cfg_file = self.prefix.cfg_file
### Get the recipe
r = recipe.get_recipe(sdkname, target='sdk')
try:
self.log.trace("Switching CWD to {0}".format(src_dir))
if not op.isdir(src_dir):
os.mkdir(src_dir)
os.chdir(src_dir)
except:
self.log.error("Source dir required to install SDK.")
return False
### Install the actual SDK file
self.log.debug("Fetching SDK `{sdk}'".format(sdk=sdkname))
fetcher.Fetcher().fetch(r)
self.log.info("Installing SDK `{sdk}'".format(sdk=sdkname))
# Install command
cmd = r.var_replace_all(r.get_command('install'))
if subproc.monitor_process(cmd, shell=True, env=os.environ) == 0:
self.log.debug("Installation successful")
else:
self.log.error("Error installing SDK. Aborting.")
return False
# Clean up
files_to_delete = [op.normpath(op.join(src_dir, r.var_replace_all(x))) for x in r.clean]
if len(files_to_delete):
self.log.info("Cleaning up files...")
for ftd in files_to_delete:
if op.commonprefix((src_dir, ftd)) != src_dir:
self.log.warn("Not removing {ftd} -- outside source dir!".format(ftd=ftd))
continue
self.log.debug("Removing {ftd}...".format(ftd=ftd))
if op.isdir(ftd):
shutil.rmtree(ftd)
elif op.isfile(ftd):
os.remove(ftd)
else:
self.log.error("Not sure what this is: {ftd}".format(ftd=ftd))
return False
### Update the prefix-local config file
self.log.debug("Updating config file with SDK recipe info.")
try:
old_cfg_data = PBConfigFile(cfg_file).get()
except IOError:
self.log.debug("There doesn't seem to be a config file yet for this prefix.")
old_cfg_data = {}
# Filter out keys we don't care about:
sdk_recipe_keys_for_config = ('config', 'packages', 'categories', 'env')
sdk_cfg_data = {k: v for k, v in iteritems(r.get_dict()) if k in sdk_recipe_keys_for_config}
self.log.trace("New data: {new}".format(new=sdk_cfg_data))
cfg_data = dict_merge(old_cfg_data, sdk_cfg_data)
self.log.debug("Writing updated prefix config to `{0}'".format(cfg_file))
PBConfigFile(cfg_file).save(cfg_data)
return True
def load(self):
"""
Load the inventory file.
If the file does not exist, initialize the internal content
with an empty dictionary.
This will override any internal state.
"""
self.log.debug("Trying to load inventory file {}...".format(self._filename))
self._invfile = PBConfigFile(self._filename)
def _load_recipe_from_file(self, filename):
"""
Turn a .lwr file into a valid recipe datastructure.
"""
data = PBConfigFile(filename).get()
# Make sure dependencies is always a valid list:
if data.has_key('depends') and data['depends'] is not None:
if not isinstance(data['depends'], list):
data['depends'] = [
data['depends'],
]
else:
data['depends'] = []
return data
def _run_env(self):
" Handle `pybombs config --env` "
cfg_file = PBConfigFile(self.cfg_file)
keys = [self.args.key]
verb = "Getting"
config_target = 'env'
if self.args.key is None:
keys = cfg_file.data.get(config_target, {}).keys()
elif self.args.value is not None:
cfg_file.update({config_target: {self.args.key: self.args.value}})
verb = "Setting"
print("{verb} environment variables:".format(verb=verb))
for key in keys:
print("{key}: {value}".format(key=key,
value=cfg_file.data.get(
config_target, {}).get(key)))
def _run_env(self):
" Handle `pybombs config --env` "
cfg_file = PBConfigFile(self.cfg_file)
keys = [self.args.key]
verb = "Getting"
config_target = 'env'
if self.args.key is None:
keys = cfg_file.data.get(config_target, {}).keys()
elif self.args.value is not None:
cfg_file.update({config_target: {self.args.key: self.args.value}})
verb = "Setting"
print("{verb} environment variables:".format(verb=verb))
for key in keys:
print("{key}: {value}".format(
key=key,
value=cfg_file.data.get(config_target, {}).get(key))
)
def load(self):
"""
Load the inventory file.
If the file does not exist, initialize the internal content
with an empty dictionary.
This will override any internal state.
"""
self.log.debug("Trying to load inventory file {0}...".format(self._filename))
self._invfile = PBConfigFile(self._filename)
def _append_cfg_from_file(self, cfg_filename):
"""
Load file filename, interpret it as a config file
and append to cfg_cascade
Returns True if loading the config file was successful.
"""
self.log.debug("Reading config info from file: {0}".format(cfg_filename))
try:
cfg_data = PBConfigFile(cfg_filename).get()
except Exception as e:
self.log.debug("Parsing config file failed ({cfgf}).".format(cfgf=cfg_filename))
self.cfg_cascade.append({})
return False
config_items = cfg_data.get('config', {})
self.log.debug('New config items: {items}'.format(items=config_items))
self.cfg_cascade.append(config_items)
return True
def _run_config(self):
" Handle `pybombs config' "
if self.args.config_only:
cfg_data = PBConfigFile(self.cfg_file).get('config')
else:
cfg_data = self.cfg
keys = [self.args.key]
if self.args.key is None:
keys = cfg_data.keys()
elif self.args.value is not None:
self.cfg.set(self.args.key, self.args.value)
self.cfg.update_cfg_file(
new_data={'config': {self.args.key: self.args.value}},
cfg_file=self.cfg_file,
)
for key in keys:
print("{key}: {value}".format(key=key, value=cfg_data.get(key, "")))
print(" - {help}".format(help=self.cfg.get_help(key) or " Undocumented config option"))
def _lint_recipe(self, recipe_file):
"""
Check if recipe_file is a valid recipe
"""
print("Linting recipe `{0}'".format(recipe_file))
# Basic file checks
try:
recipe_dict = PBConfigFile(recipe_file).get()
except IOError:
self.log.error("Can't open `{0}'".format(recipe_file))
return -1
except AttributeError:
self.log.error(
"Can't parse contents of file `{0}'".format(recipe_file))
return -1
if not isinstance(recipe_dict, dict):
self.log.error("Invalid recipe file. Not a dict.")
return -1
# Try loading as recipe
try:
rec = recipe.Recipe(recipe_file)
if not hasattr(rec, 'satisfy'):
print("[HMM] - No satisfy rules declared")
else:
for pkgtype in rec.satisfy.keys():
rec.get_package_reqs(pkgtype)
except PBException as ex:
print("[VERY BAD] - Recipe error: " + str(ex))
# Check keys
key_check = {
'HMM': ['source', 'depends'],
'BAD': ['inherit', 'category'],
}
for err_type, key_list in iteritems(key_check):
for key in key_list:
if not key in recipe_dict:
print("[{err}] Recipe doesn't have key: {key}".format(
err=err_type, key=key))
# Check dependencies is a list
dependencies = recipe_dict.get('depends', [])
if not isinstance(dependencies, list):
print("[VERY BAD] Dependencies is not a list: {}".format(
dependencies))
def _append_cfg_from_file(self, cfg_filename):
"""
Load file filename, interpret it as a config file
and append to cfg_cascade
Returns True if loading the config file was successful.
"""
self.log.debug(
"Reading config info from file: {0}".format(cfg_filename))
try:
cfg_data = PBConfigFile(cfg_filename).get()
except Exception as e:
self.log.debug("Parsing config file failed ({cfgf}).".format(
cfgf=cfg_filename))
self.cfg_cascade.append({})
return False
config_items = cfg_data.get('config', {})
self.log.debug('New config items: {items}'.format(items=config_items))
self.cfg_cascade.append(config_items)
return True
def _lint_recipe(self, recipe_file):
"""
Check if recipe_file is a valid recipe
"""
print("Linting recipe `{0}'".format(recipe_file))
# Basic file checks
try:
recipe_dict = PBConfigFile(recipe_file).get()
except IOError:
self.log.error("Can't open `{0}'".format(recipe_file))
return -1
except AttributeError:
self.log.error("Can't parse contents of file `{0}'".format(recipe_file))
return -1
if not isinstance(recipe_dict, dict):
self.log.error("Invalid recipe file. Not a dict.")
return -1
# Try loading as recipe
try:
rec = recipe.Recipe(recipe_file)
if not hasattr(rec, 'satisfy'):
print("[HMM] - No satisfy rules declared")
else:
for pkgtype in rec.satisfy.keys():
rec.get_package_reqs(pkgtype)
except PBException as ex:
print("[VERY BAD] - Recipe error: " + str(ex))
# Check keys
key_check = {
'HMM': ['source', 'depends'],
'BAD': ['inherit', 'category'],
}
for err_type, key_list in iteritems(key_check):
for key in key_list:
if not key in recipe_dict:
print("[{err}] Recipe doesn't have key: {key}".format(err=err_type, key=key))
# Check dependencies is a list
dependencies = recipe_dict.get('depends', [])
if not isinstance(dependencies, list):
print("[VERY BAD] Dependencies is not a list: {}"
.format(dependencies))
def _merge_config_info_from_file(self, cfg_file, cfg_data):
"""
Load a config file, load its contents, and merge it into cfg_info.
Return the result.
"""
try:
self.log.debug('Inspecting config file: {0}'.format(cfg_file))
cfg_data_new = PBConfigFile(cfg_file).get()
except Exception:
self.log.debug('Well, looks like that failed.')
return cfg_data
return dict_merge(cfg_data, cfg_data_new)
def extract_cfg_items(filename, section, throw_ex=True):
"""
Read section from a config file and return it as a dict.
Will throw KeyError if section does not exist.
"""
cfg_data = PBConfigFile(filename).get() or {}
try:
return cfg_data[section]
except KeyError as e:
if throw_ex:
raise e
return {}
def _run_config(self):
" Handle `pybombs config' "
if self.args.config_only:
cfg_data = PBConfigFile(self.cfg_file).get('config')
else:
cfg_data = self.cfg
keys = [self.args.key]
if self.args.key is None:
keys = cfg_data.keys()
elif self.args.value is not None:
self.cfg.set(self.args.key, self.args.value)
self.cfg.update_cfg_file(
new_data={'config': {
self.args.key: self.args.value
}},
cfg_file=self.cfg_file,
)
for key in keys:
print("{key}: {value}".format(key=key, value=cfg_data.get(key,
"")))
print(" - {help}".format(
help=self.cfg.get_help(key) or " Undocumented config option"))
def _load_recipe_from_file(self, filename):
"""
Turn a .lwr file into a valid recipe datastructure.
"""
data = PBConfigFile(filename).get()
# Make sure dependencies is always a valid list:
if 'depends' in data and data['depends'] is not None:
if not isinstance(data['depends'], Sequence):
data['depends'] = [
data['depends'],
]
else:
data['depends'] = list(
data['depends']) # not every Sequence is a list
else:
data['depends'] = []
return data
def update_cfg_file(self, new_data, cfg_file=None):
"""
Write new data to a config file.
If no config file is specified, the local config file
is used (e.g., on Linux, the one in ~/.pybombs/).
"""
if cfg_file is None:
cfg_file = self.local_cfg
if not os.path.isfile(cfg_file):
self.log.info("Creating new config file {0}".format(cfg_file))
path = os.path.split(cfg_file)[0]
if not os.path.isdir(path):
os.mkdir(path)
self.log.trace("Updating file {0} with new data: {1}".format(
cfg_file, new_data))
try:
return PBConfigFile(cfg_file).update(new_data)
except IOError:
self.log.error("Error opening config file {0}.".format(cfg_file))
return {}
def load(self, select_prefix=None):
"""
Load the actual configuration. We put this outside the ctor so we can
reload on the same object. In that case, anything unsaved is reset!
"""
## Get command line args:
parser = argparse.ArgumentParser(add_help=False)
self.setup_parser(parser)
args = parser.parse_known_args()[0]
cfg_files = []
## Set verbosity level:
verb_offset = args.verbose - args.quiet
verb_level = pb_logging.default_log_level - 10 * verb_offset
if verb_level < pb_logging.TRACE:
verb_level = pb_logging.TRACE
pb_logging.logger.setLevel(verb_level)
self.yes = args.yes
## Set up logger:
self.log = pb_logging.logger.getChild("ConfigManager")
## Setup cfg_cascade:
# self.cfg_cascade is a list of dicts. The higher the index,
# the more important the dict.
# Zeroth layer: The default values.
self.cfg_cascade = [{k: v[0] for k, v in iteritems(self.defaults)},]
# Global defaults
global_cfg = os.path.join(self.global_base_dir, self.cfg_file_name)
if self._append_cfg_from_file(global_cfg):
cfg_files.insert(0, global_cfg)
# Home directory:
self.local_cfg_dir = self.get_pybombs_dir()
if not os.path.isdir(self.local_cfg_dir):
try:
self.log.debug("Creating local config dir {0}".format(self.local_cfg_dir))
os.mkdir(self.local_cfg_dir)
except (IOError, OSError):
self.log.debug("Failed.")
self.local_cfg = os.path.join(self.local_cfg_dir, self.cfg_file_name)
if self._append_cfg_from_file(self.local_cfg):
cfg_files.insert(0, self.local_cfg)
# Current prefix (don't know that yet -- so skip for now)
self.cfg_cascade.append({})
# Config file specified on command line:
if args.config_file is not None:
self._append_cfg_from_file(npath(args.config_file))
cfg_files.insert(0, npath(args.config_file))
else:
self.cfg_cascade.append({})
# Config args specified on command line:
cmd_line_opts = {}
for opt in args.config:
k, v = opt.split('=', 1)
cmd_line_opts[k] = v
self.cfg_cascade.append(cmd_line_opts)
# Append an empty one. This is what we use when set() is called
# to change settings at runtime.
self.cfg_cascade.append({})
# After this, no more dicts should be appended to cfg_cascade.
assert len(self.cfg_cascade) == self.LAYER_VOLATILE + 1
# Find recipe templates:
self._template_dir = os.path.join(self.module_dir, 'templates')
self.log.debug("Template directory: {0}".format(self._template_dir))
## Init prefix:
self._prefix_info = PrefixInfo(args, cfg_files, select_prefix)
self._config_reference = 'prefix'
# Add the prefix config file (if it exists)
prefix_config = self._prefix_info.cfg_file
if prefix_config is not None and os.path.exists(prefix_config):
cfg_files.insert(0, prefix_config)
self._append_cfg_from_file(prefix_config, self.LAYER_PREFIX)
## Init recipe-lists:
# Go through cfg files, then env variable, then command line args
self._recipe_locations = []
self._named_recipe_dirs = {}
self._named_recipe_sources = {}
self._named_recipe_cfg_files = {}
# From command line:
for r_loc in args.recipes:
if r_loc:
self._recipe_locations.append(npath(r_loc))
# From environment variable:
if len(os.environ.get("PYBOMBS_RECIPE_DIR", "").strip()):
self._recipe_locations += [
npath(x) \
for x in os.environ.get("PYBOMBS_RECIPE_DIR").split(os.pathsep) \
if len(x.strip())
]
# From prefix info:
if self._prefix_info.recipe_dir is not None:
self._recipe_locations.append(self._prefix_info.recipe_dir)
# From config files (from here, recipe locations are named):
for cfg_file in cfg_files:
recipe_locations = PBConfigFile(cfg_file).get('recipes')
for name, uri in reversed(recipe_locations.items()):
local_recipe_dir = self.resolve_recipe_uri(
uri, name, os.path.join(os.path.split(cfg_file)[0], 'recipes')
)
self._recipe_locations.append(local_recipe_dir)
self._named_recipe_dirs[name] = local_recipe_dir
self._named_recipe_sources[name] = uri
self._named_recipe_cfg_files[name] = cfg_file
# Internal recipe list:
self._recipe_locations.append(os.path.join(self.module_dir, 'recipes'))
self.log.debug("Full list of recipe locations: {0}".format(self._recipe_locations))
self.log.debug("Named recipe locations: {0}".format(self._named_recipe_sources))
# Detect Python version of the prefix (this is *not* necessarily
# the Python version that is used for executing this script! The
# prefix could be, for example, a virtualenv with its custom Python
# version.)
self._detect_python_version()
self.log.info("Prefix Python version is: {}"
.format(self.get('python_ver')))
class Inventory(object):
"""
Inventory Manager.
Every prefix has an inventory, a list of packages that are
installed and in which state they current are.
Except for save(), none of the methods actually writes to the
inventory file.
"""
_states = {
'none': (
0,
'Package is not installed or fetched',
),
'fetched': (
10,
'Package source is in prefix, but not installed.',
),
'configured': (
20,
'Package is downloaded and configured.',
),
'built': (
30,
'Package is compiled.',
),
'installed': (40, 'Package is installed into current prefix.'),
}
def __init__(
self,
inventory_file=None,
):
self._filename = inventory_file
self.log = pb_logging.logger.getChild("Inventory")
self._state_names = {}
for state in self._states.keys():
setattr(self, "STATE_{0}".format(state.upper()),
self._states[state][0])
self._state_names[self._states[state][0]] = state
self.load()
def load(self):
"""
Load the inventory file.
If the file does not exist, initialize the internal content
with an empty dictionary.
This will override any internal state.
"""
self.log.debug("Trying to load inventory file {0}...".format(
self._filename))
self._invfile = PBConfigFile(self._filename)
def save(self):
"""
Save current state of inventory to the inventory file.
This will overwrite any existing file with the same name
and without warning. If the file didn't exist, it will be
created.
"""
self.log.debug("Saving inventory to file {0}...".format(
self._filename))
if not os.path.isdir(os.path.split(self._filename)[0]):
os.mkdir(os.path.split(self._filename)[0])
self._invfile.save()
def has(self, pkg):
"""
Returns true if the package pkg is in the inventory.
"""
return pkg in self._invfile.data
def remove(self, pkg):
"""
Remove package pkg from the inventory.
"""
if self.has(pkg):
del self._invfile.data[pkg]
self._invfile.save()
def get_state(self, pkg):
"""
Return a package's state.
See the documentation for Inventory for a list of valid states.
If pkg does not exist, returns None.
"""
try:
return self._invfile.data[pkg]["state"]
except KeyError:
return self.STATE_NONE
def set_state(self, pkg, state):
"""
Sets the state of pkg to state.
If pkg does not exist, add that package to the list.
"""
if isinstance(state, str):
try:
state = self._states[state][0]
except KeyError:
try:
state = int(state)
except ValueError:
pass
if not state in self.get_valid_states():
raise ValueError("Invalid state: {0}".format(state))
if not self.has(pkg):
self._invfile.data[pkg] = {}
self.log.debug("Setting state to `{0}'".format(
self._state_names[state]))
self._invfile.update({pkg: {'state': state}})
def get_version(self, pkg, default_version=None):
"""
Return a package's version.
This throws a PBException if the package doesn't exist.
If no version was set, return default_version (defaults to None).
"""
if not self.has(pkg):
raise PBException(
"Cannot get version for package {0} if it's not in the inventory!"
.format(pkg))
try:
return self._invfile.data[pkg]["version"]
except KeyError:
return default_version
def set_version(self, pkg, version):
"""
Sets the version of pkg to version.
This throws a PBException if the package doesn't exist.
"""
if not self.has(pkg):
raise PBException(
"Cannot set version for package {0} if it's not in the inventory!"
.format(pkg))
self.log.debug("Setting version to {0}".format(version))
self._invfile.data[pkg]["version"] = version
def set_key(self, pkg, key, value):
"""
Set an arbitrary key
"""
if key == 'state':
return self.set_state(pkg, value)
if key == 'version':
return self.set_version(pkg, value)
if not self.has(pkg):
self._invfile.data[pkg] = {}
self.log.trace("Setting key {k} on package {p} to {v}.".format(
k=key, p=pkg, v=value))
self._invfile.data[pkg][key] = value
def get_key(self, pkg, key):
"""
Get an arbitrary key
"""
if key == 'state':
return self.get_state(pkg)
if key == 'version':
return self.get_version(pkg)
return self._invfile.data[pkg].get(key)
def get_valid_states(self):
"""
Returns a list of valid arguments for set_state()
"""
return [x[0] for x in self._states.values()]
def get_state_name(self, state):
"""
Return name for a state
"""
return self._state_names[state]
def get_packages(self):
"""
Return a list of package names installed to this inventory.
"""
return self._invfile.data.keys()
class Inventory(object):
"""
Inventory Manager.
Every prefix has an inventory, a list of packages that are
installed and in which state they current are.
Except for save(), none of the methods actually writes to the
inventory file.
"""
_states = {
'none': (0, 'Package is not installed or fetched',),
'fetched': (10, 'Package source is in prefix, but not installed.',),
'configured': (20, 'Package is downloaded and configured.',),
'built': (30, 'Package is compiled.',),
'installed': (40, 'Package is installed into current prefix.'),
}
def __init__(
self,
inventory_file=None,
):
self._filename = inventory_file
self.log = pb_logging.logger.getChild("Inventory")
self._state_names = {}
for state in self._states.keys():
setattr(self, "STATE_{0}".format(state.upper()), self._states[state][0])
self._state_names[self._states[state][0]] = state
self.load()
def load(self):
"""
Load the inventory file.
If the file does not exist, initialize the internal content
with an empty dictionary.
This will override any internal state.
"""
self.log.debug("Trying to load inventory file {0}...".format(self._filename))
self._invfile = PBConfigFile(self._filename)
def save(self):
"""
Save current state of inventory to the inventory file.
This will overwrite any existing file with the same name
and without warning. If the file didn't exist, it will be
created.
"""
self.log.debug("Saving inventory to file {0}...".format(self._filename))
if not os.path.isdir(os.path.split(self._filename)[0]):
os.mkdir(os.path.split(self._filename)[0])
self._invfile.save()
def has(self, pkg):
"""
Returns true if the package pkg is in the inventory.
"""
return pkg in self._invfile.data
def remove(self, pkg):
"""
Remove package pkg from the inventory.
"""
if self.has(pkg):
del self._invfile.data[pkg]
self._invfile.save()
def get_state(self, pkg):
"""
Return a package's state.
See the documentation for Inventory for a list of valid states.
If pkg does not exist, returns None.
"""
try:
return self._invfile.data[pkg]["state"]
except KeyError:
return self.STATE_NONE
def set_state(self, pkg, state):
"""
Sets the state of pkg to state.
If pkg does not exist, add that package to the list.
"""
if isinstance(state, str):
try:
state = self._states[state][0]
except KeyError:
try:
state = int(state)
except ValueError:
pass
if not state in self.get_valid_states():
raise ValueError("Invalid state: {0}".format(state))
if not self.has(pkg):
self._invfile.data[pkg] = {}
self.log.debug("Setting state to `{0}'".format(self._state_names[state]))
self._invfile.update({pkg: {'state': state}})
def get_version(self, pkg, default_version=None):
"""
Return a package's version.
This throws a PBException if the package doesn't exist.
If no version was set, return default_version (defaults to None).
"""
if not self.has(pkg):
raise PBException("Cannot get version for package {0} if it's not in the inventory!".format(pkg))
try:
return self._invfile.data[pkg]["version"]
except KeyError:
return default_version
def set_version(self, pkg, version):
"""
Sets the version of pkg to version.
This throws a PBException if the package doesn't exist.
"""
if not self.has(pkg):
raise PBException("Cannot set version for package {0} if it's not in the inventory!".format(pkg))
self.log.debug("Setting version to {0}".format(version))
self._invfile.data[pkg]["version"] = version
def set_key(self, pkg, key, value):
"""
Set an arbitrary key
"""
if key == 'state':
return self.set_state(pkg, value)
if key == 'version':
return self.set_version(pkg, value)
if not self.has(pkg):
self._invfile.data[pkg] = {}
self.log.obnoxious("Setting key {k} on package {p} to {v}.".format(k=key, p=pkg, v=value))
self._invfile.data[pkg][key] = value
def get_key(self, pkg, key):
"""
Get an arbitrary key
"""
if key == 'state':
return self.get_state(pkg)
if key == 'version':
return self.get_version(pkg)
return self._invfile.data[pkg].get(key)
def get_valid_states(self):
"""
Returns a list of valid arguments for set_state()
"""
return [x[0] for x in self._states.values()]
def get_state_name(self, state):
"""
Return name for a state
"""
return self._state_names[state]
def get_packages(self):
"""
Return a list of package names installed to this inventory.
"""
return self._invfile.data.keys()
def __init__(self, args, cfg_list, select_prefix=None):
self.log = pb_logging.logger.getChild("ConfigManager.PrefixInfo")
self.prefix_dir = None
self.prefix_cfg_dir = None
self.prefix_src = None
self.alias = None
self.src_dir = None
self.cfg_file = None
self.inv_file = None
self.inventory = None
self.recipe_dir = None
self.target_dir = None
self.env = os.environ.copy()
self.is_virtualenv = False
self._cfg_info = OrderedDict(self.default_config_info)
if select_prefix is not None:
args.prefix = select_prefix
# 1) Load the config info
for cfg_file in reversed(cfg_list):
self._cfg_info = \
self._merge_config_info_from_file(cfg_file, self._cfg_info)
# 2) Find the prefix directory
self._find_prefix_dir(args)
if self.prefix_dir is None:
self.log.debug("Cannot establish a prefix directory. This may cause issues down the line.")
self._set_attrs()
return
assert self.prefix_dir is not None
if self.alias is not None and self.alias in self._cfg_info['prefix_config_dir']:
self.prefix_cfg_dir = npath(self._cfg_info['prefix_config_dir'][self.alias])
self.log.debug("Choosing prefix config dir from alias: {0}".format(self.prefix_cfg_dir))
elif self.prefix_dir in self._cfg_info['prefix_config_dir']:
self.prefix_cfg_dir = npath(self._cfg_info['prefix_config_dir'][self.prefix_dir])
self.log.debug("Choosing prefix config dir from path lookup in prefix_config_dir: {0}".format(self.prefix_cfg_dir))
else:
self.prefix_cfg_dir = npath(os.path.join(self.prefix_dir, self.prefix_conf_dir))
self.log.debug("Choosing default prefix config dir: {0}".format(self.prefix_cfg_dir))
if not os.path.isdir(self.prefix_cfg_dir):
self.log.debug("Config dir does not yet exist.")
self.is_virtualenv = sysutils.is_virtualenv(self.prefix_dir)
if self.is_virtualenv:
self.log.info("Prefix is a Python virtualenv.")
# 3) Find the config file
self.cfg_file = npath(os.path.join(self.prefix_cfg_dir, ConfigManager.cfg_file_name))
config_section = {}
if not os.path.isfile(self.cfg_file):
self.log.debug("Prefix configuration file not found: {0}, assuming empty.".format(self.cfg_file))
else:
config_section = PBConfigFile(self.cfg_file).get('config')
self._cfg_info = self._merge_config_info_from_file(self.cfg_file, self._cfg_info)
# 4) Find the src dir
self.src_dir = npath(config_section.get('srcdir', os.path.join(self.prefix_dir, self.src_dir_name)))
self.log.debug("Prefix source dir is: {0}".format(self.src_dir))
if not os.path.isdir(self.src_dir):
self.log.debug("Source dir does not exist.")
# 5) Find the inventory file
self.inv_file = npath(os.path.join(self.prefix_cfg_dir, self.inv_file_name))
if not os.path.isfile(self.inv_file):
self.log.debug("Prefix inventory file not found: {0}".format(self.inv_file))
self.inventory = inventory.Inventory(inventory_file=self.inv_file)
# 6) Prefix-specific recipes. There's two places for these:
# - A 'recipes/' subdirectory
# - Anything declared in the config.yml file inside the prefix
self.recipe_dir = npath(config_section.get('recipes', os.path.join(self.prefix_cfg_dir, 'recipes')))
if os.path.isdir(self.recipe_dir):
self.log.debug("Prefix-local recipe dir is: {0}".format(self.recipe_dir))
else:
self.recipe_dir = None
# 7) Load environment
# If there's a setup_env option in the current config file, we use that
if self.setup_env_key in config_section:
self.log.debug('Loading environment from shell script: {0}'.format(config_section[self.setup_env_key]))
self.env = self._load_environ_from_script(config_section[self.setup_env_key])
else:
self.env = self._load_default_env(self.env)
# Set env vars that we always need
self.env[self.env_prefix_var] = self.prefix_dir
self.env[self.env_srcdir_var] = self.src_dir
# env: sections are always respected:
OLD_ENV = os.environ # Bit of an ugly hack to allow use of
os.environ = self.env # os.path.expandvars() on self.env
for k, v in iteritems(self._cfg_info['env']):
self.env[k.upper()] = os.path.expandvars(v.strip())
os.environ = OLD_ENV
# 8) Keep relevant config sections as attributes
self._set_attrs()
def load(self, select_prefix=None):
"""
Load the actual configuration. We put this outside the ctor so we can
reload on the same object. In that case, anything unsaved is reset!
"""
## Get command line args:
parser = argparse.ArgumentParser(add_help=False)
self.setup_parser(parser)
args = parser.parse_known_args()[0]
cfg_files = []
## Set verbosity level:
verb_offset = args.verbose - args.quiet
verb_level = pb_logging.default_log_level - 10 * verb_offset
if verb_level < pb_logging.OBNOXIOUS:
verb_level = pb_logging.OBNOXIOUS
pb_logging.logger.setLevel(verb_level)
self.yes = args.yes
## Set up logger:
self.log = pb_logging.logger.getChild("ConfigManager")
## Setup cfg_cascade:
# self.cfg_cascade is a list of dicts. The higher the index,
# the more important the dict.
# Zeroth layer: The default values.
self.cfg_cascade = [{k: v[0] for k, v in iteritems(self.defaults)},]
# Global defaults
global_cfg = os.path.join(self.global_base_dir, self.cfg_file_name)
if self._append_cfg_from_file(global_cfg):
cfg_files.insert(0, global_cfg)
# Home directory:
self.local_cfg_dir = self.get_pybombs_dir()
if not os.path.isdir(self.local_cfg_dir):
try:
self.log.debug("Creating local config dir {0}".format(self.local_cfg_dir))
os.mkdir(self.local_cfg_dir)
except (IOError, OSError):
self.log.debug("Failed.")
self.local_cfg = os.path.join(self.local_cfg_dir, self.cfg_file_name)
if self._append_cfg_from_file(self.local_cfg):
cfg_files.insert(0, self.local_cfg)
# Current prefix (don't know that yet -- so skip for now)
self.cfg_cascade.append({})
# Config file specified on command line:
if args.config_file is not None:
self._append_cfg_from_file(npath(args.config_file))
cfg_files.insert(0, npath(args.config_file))
else:
self.cfg_cascade.append({})
# Config args specified on command line:
cmd_line_opts = {}
for opt in args.config:
k, v = opt.split('=', 1)
cmd_line_opts[k] = v
self.cfg_cascade.append(cmd_line_opts)
# Append an empty one. This is what we use when set() is called
# to change settings at runtime.
self.cfg_cascade.append({})
# After this, no more dicts should be appended to cfg_cascade.
assert len(self.cfg_cascade) == self.LAYER_VOLATILE + 1
# Find recipe templates:
self._template_dir = os.path.join(self.module_dir, 'templates')
self.log.debug("Template directory: {0}".format(self._template_dir))
## Init prefix:
self._prefix_info = PrefixInfo(args, cfg_files, select_prefix)
# Add the prefix config file (if it exists)
prefix_config = self._prefix_info.cfg_file
if prefix_config is not None and os.path.exists(prefix_config):
cfg_files.insert(0, prefix_config)
self._append_cfg_from_file(prefix_config, self.LAYER_PREFIX)
## Init recipe-lists:
# Go through cfg files, then env variable, then command line args
self._recipe_locations = []
self._named_recipe_dirs = {}
self._named_recipe_sources = {}
self._named_recipe_cfg_files = {}
# From command line:
for r_loc in args.recipes:
if r_loc:
self._recipe_locations.append(npath(r_loc))
# From environment variable:
if len(os.environ.get("PYBOMBS_RECIPE_DIR", "").strip()):
self._recipe_locations += [
npath(x) \
for x in os.environ.get("PYBOMBS_RECIPE_DIR").split(os.pathsep) \
if len(x.strip())
]
# From prefix info:
if self._prefix_info.recipe_dir is not None:
self._recipe_locations.append(self._prefix_info.recipe_dir)
# From config files (from here, recipe locations are named):
for cfg_file in cfg_files:
recipe_locations = PBConfigFile(cfg_file).get('recipes')
for name, uri in reversed(recipe_locations.items()):
local_recipe_dir = self.resolve_recipe_uri(
uri, name, os.path.join(os.path.split(cfg_file)[0], 'recipes')
)
self._recipe_locations.append(local_recipe_dir)
self._named_recipe_dirs[name] = local_recipe_dir
self._named_recipe_sources[name] = uri
self._named_recipe_cfg_files[name] = cfg_file
# Internal recipe list:
self._recipe_locations.append(os.path.join(self.module_dir, 'recipes'))
self.log.debug("Full list of recipe locations: {0}".format(self._recipe_locations))
self.log.debug("Named recipe locations: {0}".format(self._named_recipe_sources))
def __init__(self, args, cfg_list, select_prefix=None):
self.log = pb_logging.logger.getChild("ConfigManager.PrefixInfo")
self.prefix_dir = None
self.prefix_cfg_dir = None
self.prefix_src = None
self.alias = None
self.src_dir = None
self.cfg_file = None
self.inv_file = None
self.inventory = None
self.recipe_dir = None
self.target_dir = None
self.env = os.environ.copy()
self.is_virtualenv = False
self._cfg_info = OrderedDict(self.default_config_info)
if select_prefix is not None:
args.prefix = select_prefix
# 1) Load the config info
for cfg_file in reversed(cfg_list):
self._cfg_info = \
self._merge_config_info_from_file(cfg_file, self._cfg_info)
# 2) Find the prefix directory
self._find_prefix_dir(args)
if self.prefix_dir is None:
self.log.debug("Cannot establish a prefix directory. This may cause issues down the line.")
self._set_attrs()
return
assert self.prefix_dir is not None
if self.alias is not None and self.alias in self._cfg_info['prefix_config_dir']:
self.prefix_cfg_dir = npath(self._cfg_info['prefix_config_dir'][self.alias])
self.log.debug("Choosing prefix config dir from alias: {0}".format(self.prefix_cfg_dir))
elif self.prefix_dir in self._cfg_info['prefix_config_dir']:
self.prefix_cfg_dir = npath(self._cfg_info['prefix_config_dir'][self.prefix_dir])
self.log.debug("Choosing prefix config dir from path lookup in prefix_config_dir: {0}".format(self.prefix_cfg_dir))
else:
self.prefix_cfg_dir = npath(os.path.join(self.prefix_dir, self.prefix_conf_dir))
self.log.debug("Choosing default prefix config dir: {0}".format(self.prefix_cfg_dir))
if not os.path.isdir(self.prefix_cfg_dir):
self.log.debug("Config dir does not yet exist.")
self.is_virtualenv = sysutils.is_virtualenv(self.prefix_dir)
if self.is_virtualenv:
self.log.info("Prefix is a Python virtualenv.")
# 3) Find the config file
self.cfg_file = npath(os.path.join(self.prefix_cfg_dir, ConfigManager.cfg_file_name))
config_section = {}
if not os.path.isfile(self.cfg_file):
self.log.debug("Prefix configuration file not found: {0}, assuming empty.".format(self.cfg_file))
else:
config_section = PBConfigFile(self.cfg_file).get('config')
self._cfg_info = self._merge_config_info_from_file(self.cfg_file, self._cfg_info)
# 4) Find the src dir
self.src_dir = npath(config_section.get('srcdir', os.path.join(self.prefix_dir, self.src_dir_name)))
self.log.debug("Prefix source dir is: {0}".format(self.src_dir))
if not os.path.isdir(self.src_dir):
self.log.debug("Source dir does not exist.")
# 5) Find the inventory file
self.inv_file = npath(os.path.join(self.prefix_cfg_dir, self.inv_file_name))
if not os.path.isfile(self.inv_file):
self.log.debug("Prefix inventory file not found: {0}".format(self.inv_file))
self.inventory = inventory.Inventory(inventory_file=self.inv_file)
# 6) Prefix-specific recipes. There's two places for these:
# - A 'recipes/' subdirectory
# - Anything declared in the config.yml file inside the prefix
self.recipe_dir = npath(config_section.get('recipes', os.path.join(self.prefix_cfg_dir, 'recipes')))
if os.path.isdir(self.recipe_dir):
self.log.debug("Prefix-local recipe dir is: {0}".format(self.recipe_dir))
else:
self.recipe_dir = None
# 7) Load environment
# If there's a setup_env option in the current config file, we use that
if self.setup_env_key in config_section:
self.log.debug('Loading environment from shell script: {0}'.format(config_section[self.setup_env_key]))
self.env = self._load_environ_from_script(config_section[self.setup_env_key])
else:
self.env = self._load_default_env(self.env)
# Set env vars that we always need
self.env[self.env_prefix_var] = self.prefix_dir
self.env[self.env_srcdir_var] = self.src_dir
# env: sections are always respected:
OLD_ENV = os.environ # Bit of an ugly hack to allow use of
os.environ = self.env # os.path.expandvars() on self.env
for k, v in iteritems(self._cfg_info['env']):
self.env[k.upper()] = os.path.expandvars(v.strip())
os.environ = OLD_ENV
# 8) Keep relevant config sections as attributes
self._set_attrs()
