def __init__(
self,
cmd,
args,
load_recipes=False,
require_prefix=True,
):
self.cmd = cmd
self.args = args
self.log = pb_logging.logger.getChild(cmd)
self.log.debug("Initializing command class for command {}".format(cmd))
self.cfg = config_manager
if not cmd in self.cmds.keys():
raise PBException(
"{} is not a valid name for this command.".format(cmd))
if load_recipes:
from pybombs import recipe_manager
self.recipe_manager = recipe_manager.recipe_manager
self.prefix = None
if self.cfg.get_active_prefix().prefix_dir is not None:
self.prefix = self.cfg.get_active_prefix()
elif require_prefix:
self.log.error("No prefix specified. Aborting.")
raise PBException("No prefix specified.")
if self.prefix is not None:
self.inventory = self.prefix.inventory
def pl_pkg(self, pkg_name):
" Called in a package requirements list, when a package name is found "
if self.preq is None:
self.log.obnoxious("Adding package with name {}".format(pkg_name))
self.preq = PBPackageRequirement(pkg_name)
elif isinstance(self.preq, PBPackageRequirement):
if self.preq.compare is None:
self.preq.name = " ".join((self.preq.name, pkg_name))
self.log.obnoxious("Extending package name {}".format(
self.preq.name))
else:
raise PBException(
"Parsing Error. Did not expect package name here.")
elif isinstance(self.preq, PBPackageRequirementPair):
if self.preq.second is None:
self.log.obnoxious(
"Adding package with name {}".format(pkg_name))
self.preq.second = PBPackageRequirement(pkg_name)
else:
if self.preq.second.compare is None:
self.preq.second.name = " ".join(
(self.preq.second.name, pkg_name))
self.log.obnoxious("Extending package name {}".format(
self.preq.second.name))
else:
raise PBException(
"Parsing Error. Did not expect package name here ({0})."
.format(self.preq.second))
else:
raise PBException("Random Foobar Parsing Error.")
def update(self, recipe):
"""
Try to softly update the source directory.
This means the build dir might actually survive.
"""
self.log.debug("Updating source for recipe: {0}".format(recipe.id))
if not self.check_fetched(recipe):
self.log.error(
"Cannot update recipe {r}, it is not yet fetched.".format(
r=recipe.id))
return False
# Jump to the src directory
cwd = os.getcwd()
self.log.debug("Switching to src directory: {0}".format(self.src_dir))
os.chdir(self.src_dir)
if not os.path.isdir(os.path.join(self.src_dir, recipe.id)):
raise PBException("Source directory {d} does not exist!!".format(
d=os.path.join(self.src_dir, recipe.id)))
# Figure out which source was used before
src = self.inventory.get_key(recipe.id, 'source')
if not src:
raise PBException(
"Cannot establish prior source for package {p}".format(
p=recipe.id))
# Do the update
self.log.obnoxious("Trying to update from {0}".format(src))
try:
if self.update_src(src, self.src_dir, recipe.id,
recipe.get_dict()):
self.log.obnoxious("Update successful.")
if self.inventory.get_state(
recipe.id) >= self.inventory.STATE_CONFIGURED:
self.log.obnoxious(
"Setting package state to 'configured'.")
self.inventory.set_state(recipe.id,
self.inventory.STATE_CONFIGURED)
else:
self.log.obnoxious("Setting package state to 'fetched'.")
self.inventory.set_state(recipe.id,
self.inventory.STATE_FETCHED)
self.inventory.save()
os.chdir(cwd)
self.log.obnoxious("Update completed.")
return True
except PBException as ex:
self.log.debug("That didn't work.")
self.log.debug(str(ex))
except Exception as ex:
self.log.error("Unexpected error while fetching {0}.".format(src))
self.log.error(ex)
# Ideally, we've left the function at this point.
raise PBException("Unable to update recipe {0}".format(recipe.id))
def get_git_version():
"""
Return the currently installed git version as a string.
"""
try:
return re.search(r'[0-9.]+',
subproc.check_output(['git', '--version'])).group(0)
except OSError:
raise PBException("Unable to execute git!")
except subproc.CalledProcessError:
raise PBException("Error executing 'git --version'!")
except AttributeError:
raise PBException("Unexpected output from 'git --version'!")
def pl_cmb(self, cmb):
" Called in a package requirements list, when a logical combiner (||, &&) is found "
self.log.obnoxious("Found package combiner {}".format(cmb))
if self.preq is None:
raise PBException("Parsing Error. Did not expect combiner here.")
self.preq = PBPackageRequirementPair(self.preq)
self.preq.combiner = cmb
def get_prefix_cfg_dir_writable(self):
" Returns self.prefix_cfg_dir, but if that doesn't exist, create it first. "
if self.prefix_dir is None:
raise PBException(
"Can't access prefix config dir if prefix does not exist.")
sysutils.mkdir_writable(self.prefix_cfg_dir, self.log)
return self.prefix_cfg_dir
def _load_environ_from_script(self, setup_env_file):
"""
Run setup_env_file, return the new env
FIXME make this portable!
"""
self.log.debug('Loading environment from shell script: {0}'.format(setup_env_file))
# It would be nice if we could do os.path.expandvars() with a custom
# env, wouldn't it
setup_env_file = setup_env_file.replace('${0}'.format(self.env_prefix_var), self.prefix_dir)
setup_env_file = setup_env_file.replace('${{{0}}}'.format(self.env_prefix_var), self.prefix_dir)
# TODO add some checks this is a legit script
# Damn, I hate just running stuff :/
# TODO unportable command:
separator = '<<<<<<<<<<<<<<<<<<<<<<>>>>>>>>>>>>>>>>>>>>>>'
get_env_cmd = "source {env_file} && echo '{sep}' && env".format(env_file=setup_env_file, sep=separator)
from pybombs.utils import subproc
try:
script_output = subproc.check_output(get_env_cmd, shell=True)
except subproc.CalledProcessError:
self.log.error("Trouble sourcing file {env_file}".format(env_file=setup_env_file))
raise PBException("Could not source env file.")
env_output = script_output.split(separator)[-1]
# TODO assumption is that env_output now just holds the env output
env_output = env_output.split('\n')
env = {}
for env_line in env_output:
env_line = env_line.strip()
if len(env_line) == 0:
continue
k, v = env_line.split('=', 1)
env[k] = v
return env
def get_recipe(pkgname, target='package', fail_easy=False):
"""
Return a recipe object by its package name.
"""
cache_key = pkgname
if cache_key in recipe_cache:
pb_logging.logger.getChild("get_recipe").obnoxious(
"Woohoo, this one's already cached ({})".format(pkgname))
return recipe_cache[cache_key]
try:
r = Recipe(recipe_manager.recipe_manager.get_recipe_filename(pkgname))
except PBException as ex:
pb_logging.logger.getChild("get_recipe").error(
"Error fetching recipe `{0}':\n{1}".format(pkgname, str(ex)))
if fail_easy:
return None
else:
raise ex
recipe_cache[cache_key] = r
if target is not None and r.target != target:
if fail_easy:
return None
pb_logging.logger.getChild("get_recipe").error(
"Recipe for `{pkg}' found, but does not match request target type `{req}' (is `{actual}')."
.format(pkg=pkgname, req=target, actual=r.target))
raise PBException("Recipe has wrong target type.")
return r
def make(self, recipe, try_again=False):
"""
Build this recipe.
If try_again is set, it will assume the build failed before
and we're trying to run it again. In this case, reduce the
makewidth to 1 and show the build output.
"""
self.log.debug("Building recipe {}".format(recipe.id))
self.log.debug("In cwd - {}".format(os.getcwd()))
o_proc = None
if self.log.getEffectiveLevel(
) >= pb_logging.DEBUG and not try_again and not recipe.make_interactive:
o_proc = output_proc.OutputProcessorMake(preamble="Building: ")
cmd = recipe.var_replace_all(self.get_command('make', recipe))
cmd = self.filter_cmd(cmd, recipe, 'make_filter')
if subproc.monitor_process(cmd, shell=True, o_proc=o_proc) == 0:
self.log.debug("Make successful")
return True
# OK, something bad happened.
if try_again == False:
recipe.vars['makewidth'] = '1'
self.make(recipe, try_again=True)
else:
self.log.error(
"Build failed. See output above for error messages.")
raise PBException("Build failed.")
def uninstall(self, recipe):
"""
"""
cwd = os.getcwd()
pkg_src_dir = os.path.normpath(os.path.join(self.prefix.src_dir, recipe.id))
builddir = os.path.normpath(os.path.join(pkg_src_dir, recipe.installdir))
os.chdir(builddir)
self.log.debug("Using build directory: {0}".format(builddir))
if not os.path.isdir(pkg_src_dir):
os.chdir(cwd)
raise PBException("There should be a source dir in {0}, but there isn't.".format(pkg_src_dir))
get_state = lambda: (self.inventory.get_state(recipe.id) or 0)
set_state = lambda state: self.inventory.set_state(recipe.id, state) or self.inventory.save()
if get_state() >= self.inventory.STATE_INSTALLED:
try:
self.make_clean(recipe)
except PBException as ex:
self.log.warn("Uninstall failed: {0}.".format(str(ex)))
return False
set_state(self.inventory.STATE_CONFIGURED)
else:
self.log.debug("Package {0} is not yet installed.".format(recipe.id))
os.chdir(cwd)
self.log.obnoxious("Uninstall complete.")
return True
def __init__(self,):
# Set up logger:
self.log = pb_logging.logger.getChild("PackageManager")
self.cfg = config_manager
self.prefix_available = self.cfg.get_active_prefix().prefix_dir is not None
# Create a source package manager
if self.prefix_available:
self.src = packagers.Source()
self.prefix = self.cfg.get_active_prefix()
else:
self.log.debug("No prefix specified. Skipping source package manager.")
self.src = None
# Create sorted list of binary package managers
requested_packagers = [x.strip() for x in self.cfg.get('packagers').split(',') if x]
binary_pkgrs = []
for pkgr in requested_packagers:
self.log.debug("Attempting to add binary package manager {}".format(pkgr))
p = packagers.get_by_name(pkgr, packagers.__dict__.values())
if p is None:
self.log.warn("This binary package manager can't be instantiated: {}".format(pkgr))
continue
if p.supported():
self.log.debug("{} is supported!".format(pkgr))
binary_pkgrs.append(p)
self._packagers = []
for satisfy in self.cfg.get('satisfy_order').split(','):
satisfy = satisfy.strip()
if satisfy == 'src':
if self.prefix_available:
self._packagers += [self.src,]
elif satisfy == 'native':
self._packagers += binary_pkgrs
else:
raise PBException("Invalid satisfy_order value: {}".format(satisfy))
self.log.debug("Using packagers: {}".format([x.name for x in self._packagers]))
def configure(self, recipe, try_again=False):
"""
Run the configuration step for this recipe.
If try_again is set, it will assume the configuration failed before
and we're trying to run it again.
"""
self.log.debug("Configuring recipe {}".format(recipe.id))
self.log.debug("Using vars - {}".format(recipe.vars))
self.log.debug("In cwd - {}".format(os.getcwd()))
pre_cmd = recipe.var_replace_all(self.get_command('configure', recipe))
cmd = self.filter_cmd(pre_cmd, recipe, 'config_filter')
o_proc = None
if self.log.getEffectiveLevel() >= pb_logging.DEBUG and not try_again:
o_proc = output_proc.OutputProcessorMake(preamble="Configuring: ")
if subproc.monitor_process(cmd, shell=True, o_proc=o_proc) == 0:
self.log.debug("Configure successful.")
return True
# OK, something went wrong.
if try_again == False:
self.log.warning(
"Configuration failed. Re-trying with higher verbosity.")
return self.configure(recipe, try_again=True)
else:
self.log.error(
"Configuration failed after running at least twice.")
raise PBException("Configuration failed")
def choose_deployer(ttype, target):
"""
Return the best match for the deployer class.
throws KeyError if class can't be determined.
"""
if ttype is not None:
return {
'tar': TarfileDeployer,
'gzip': GZipDeployer,
'bzip2': BZip2Deployer,
'xz': XZDeployer,
#'ssh': SSHDeployer
}[ttype]
if re.match(r'.*\.tar\.gz$', target):
return GZipDeployer
if re.match(r'.*\.tar\.bz2$', target):
return BZip2Deployer
if re.match(r'.*\.tar\.xz$', target):
return XZDeployer
if re.match(r'.*\.tar$', target):
return TarfileDeployer
if target.find('@') != -1:
return SSHDeployer
raise PBException(
"Cannot determine deployment type for target `{0}'".format(target))
def __init__(self):
PackagerBase.__init__(self)
if self.cfg.get_active_prefix().prefix_dir is None:
self.log.error("No prefix specified. Aborting.")
raise PBException("No prefix specified.")
self.prefix = self.cfg.get_active_prefix()
self.inventory = self.prefix.inventory
self.static = False
def get(self, key, default=None):
""" Return the value for a given key. """
for set_of_vals in reversed(self.cfg_cascade):
if key in set_of_vals.keys():
return set_of_vals[key]
if default is not None:
return default
raise PBException("Invalid configuration key: {0}".format(key))
def register_alias(alias, path):
" Write the prefix alias to the config file "
if self.prefix is not None and \
self.prefix.prefix_aliases.get(alias) is not None \
and not confirm("Alias `{0}' already exists, overwrite?".format(alias)):
self.log.warn('Aborting.')
raise PBException("Could not create alias.")
self.cfg.update_cfg_file({'prefix_aliases': {alias: path}})
def fetch(self, recipe):
"""
Fetch a package identified by its recipe into the current prefix.
If a package was already fetched, do nothing.
Contract:
- Will either raise PBException or the fetch was successful.
"""
self.log.debug("Fetching source for recipe: {0}".format(recipe.id))
if self.check_fetched(recipe):
self.log.info("Already fetched: {0}".format(recipe.id))
return True
# Jump to the src directory
cwd = os.getcwd()
self.log.debug("Switching to src directory: {0}".format(self.src_dir))
if not os.path.isdir(self.src_dir):
os.mkdir(self.src_dir)
os.chdir(self.src_dir)
if os.path.exists(os.path.join(self.src_dir, recipe.id)):
raise PBException("Directory {d} already exists!".format(
d=os.path.join(self.src_dir, recipe.id)))
# Do the fetch
for src in recipe.source:
src = recipe.var_replace_all(src)
self.log.obnoxious("Trying to fetch {0}".format(src))
try:
if self.fetch_url(src, self.src_dir, recipe.id,
recipe.get_dict()):
self.log.obnoxious("Success.")
self.inventory.set_key(recipe.id, 'source', src)
if self.inventory.get_state(
recipe.id) < self.inventory.STATE_FETCHED:
self.inventory.set_state(recipe.id,
self.inventory.STATE_FETCHED)
self.inventory.save()
os.chdir(cwd)
return True
except PBException as ex:
self.log.debug("That didn't work.")
self.log.debug(str(ex))
except Exception as ex:
self.log.error(
"Unexpected error while fetching {0}.".format(src))
self.log.error(ex)
# Ideally, we've left the function at this point.
raise PBException("Unable to fetch recipe {0}".format(recipe.id))
def set_version(self, pkg, version):
"""
Sets the version of pkg to version.
This throws a PBException if the package doesn't exist.
"""
if not self.has(pkg):
raise PBException("Cannot set version for package {} if it's not in the inventory!".format(pkg))
self.log.debug("Setting version to {}".format(version))
self._invfile.data[pkg]["version"] = version
def get_recipe_filename(self, name):
"""
Return the filename of the .lwr file that contains the recipe
for package called 'name'.
"""
try:
return self._recipe_list[name]
except KeyError:
raise PBException("Package {0} has no recipe file!".format(name))
def run(self):
"""
Override this.
The return value is passed to exit(), so don't return True on success.
Either return None (i.e., return nothing), or return an exit code (zero
on success, non-zero otherwise).
"""
raise PBException(
"run() method not implemented for command {1}!".format(self.cmd))
def get_token_functor(self, token):
" Return a functor for a given token or throw "
for (match, functor) in self.lexicon.items():
if isinstance(match, str) and token == match:
return functor
if isinstance(match, tuple) and token in match:
return functor
if hasattr(match, 'match') and match.match(token):
return functor
raise PBException("Invalid token: {tok}".format(token))
def write_file_in_subdir(base_path, file_path, content):
"""
Write 'content' to a file. The absolute path to the file comes from
joining base_path and file_path. However, if file_path tries to go
outside base_path, an exception is raised.
"""
abs_file_path = os.path.join(base_path, file_path)
if not op.normpath(base_path) in op.normpath(abs_file_path):
raise PBException("Attempting write to file outside base_path")
open(abs_file_path, 'w').write(content)
def _check_if_pkg_goes_into_tree(pkg):
" Return True if pkg has a legitimate right to be in the tree. "
self.log.obnoxious(
"Checking if package `{pkg}' goes into tree...".format(
pkg=pkg))
if fail_if_not_exists and not bool(self.pm.installed(pkg)):
self.log.obnoxious(
"Only installed packages need to into tree, and this one is not."
)
return False
if no_deps and pkg not in packages:
self.log.obnoxious(
"Not installing, because it's not in the list.")
return False
if not self.pm.exists(pkg):
self.log.error(
"Package has no install method: {0}".format(pkg))
raise PBException("Unresolved install path.")
if not self.pm.installed(pkg):
# If it's not installed, we'll try a binary install...
self.log.debug(
"Testing binary install for package {pkg}.".format(
pkg=pkg))
if self.pm.install(pkg,
install_type="binary",
static=static,
verify=verify,
fail_silently=True):
self.log.obnoxious(
"Binary install successful, so we don't put it into tree."
)
# ...and if that worked, it doesn't have to go into the tree.
return False
self.log.obnoxious("Not installed: It goes into tree.")
# Now it's still not installed, so it has to go into the tree:
return True
else:
# If a package is already installed, but not flagged for
# updating, it does not go into the tree:
if not update_if_exists or pkg not in packages:
self.log.obnoxious(
"Installed, but no update requested. Does not go into tree."
)
return False
# OK, so it needs updating. But only if it's a source package:
if self.pm.installed(pkg, install_type="source"):
self.log.obnoxious(
"Package was source-installed, and needs update.")
return True
# Otherwise, we should give it a shot:
self.log.obnoxious(
"Doesn't go into tree, but we'll try a packager update.")
self.pm.update(pkg, install_type="binary")
return False
assert False # Should never reach this line
def _download_with_wget(url):
" Use the wget tool itself "
def get_md5(filename):
" Return MD5 sum of filename using the md5sum tool "
md5_exe = sysutils.which('md5sum')
if md5_exe is not None:
return subproc.check_output([md5_exe, filename])[0:32]
return None
from pybombs.utils import sysutils
from pybombs.utils import subproc
wget = sysutils.which('wget')
if wget is None:
raise PBException("wget executable not found")
filename = os.path.split(url)[1]
retval = subproc.monitor_process([wget, url], throw=True)
if retval:
raise PBException("wget failed to wget")
return filename, get_md5(filename)
def _find_prefix_dir(self, args):
"""
Find the current prefix' directory.
Order is:
1) From the command line (-p switch; either an alias, or a directory)
2) Environment variable (see env_prefix_var)
3) CWD (if it has a .pybombs subdir and is not the home directory)
4) The config option called 'default_prefix'
If all of these fail, we have no prefix.
"""
if args.prefix is not None:
if args.prefix in self._cfg_info['prefix_aliases']:
self.log.debug("Resolving prefix alias {0}.".format(
args.prefix))
self.alias = args.prefix
args.prefix = self._cfg_info['prefix_aliases'][args.prefix]
if not os.path.isdir(npath(args.prefix)):
self.log.error("Not a prefix: {0}".format(args.prefix))
raise PBException("Can't open prefix: {0}".format(args.prefix))
self.prefix_dir = npath(args.prefix)
self.prefix_src = 'cli'
self.log.debug("Choosing prefix dir from command line: {0}".format(
self.prefix_dir))
return
if self.env_prefix_var in os.environ and os.path.isdir(
os.environ[self.env_prefix_var]):
self.prefix_dir = npath(os.environ[self.env_prefix_var])
self.prefix_src = 'env'
self.log.debug(
'Using environment variable {0} as prefix ({1})'.format(
self.env_prefix_var, self.prefix_dir))
return
if os.getcwd() != os.path.expanduser('~') and os.path.isdir(
os.path.join('.', self.prefix_conf_dir)):
self.prefix_dir = os.getcwd()
self.prefix_src = 'cwd'
self.log.debug('Using CWD as prefix ({0})'.format(self.prefix_dir))
return
if self._cfg_info.get('config', {}).get('default_prefix'):
default_prefix = self._cfg_info['config']['default_prefix']
if default_prefix in self._cfg_info['prefix_aliases']:
self.log.debug(
"Resolving prefix alias `{0}'.".format(default_prefix))
self.prefix_dir = npath(
self._cfg_info['prefix_aliases'][default_prefix])
else:
self.prefix_dir = npath(default_prefix)
self.log.debug('Using default_prefix as prefix ({0})'.format(
self.prefix_dir))
self.prefix_src = 'default'
return
self.prefix_src = None
self.prefix_dir = None
def install(self, recipe, static=False, update=False):
"""
Run the source installation process for package 'recipe'.
May raise an exception if things go terribly wrong.
Otherwise, return True on success and False if installing failed.
"""
if not os.path.isdir(self.prefix.src_dir):
os.makedirs(self.prefix.src_dir)
self.static = static
recipe.set_static(static)
cwd = os.getcwd()
get_state = lambda: (self.inventory.get_state(recipe.id) or 0)
set_state = lambda state: self.inventory.set_state(
recipe.id, state) or self.inventory.save()
if not hasattr(recipe, 'source') or len(recipe.source) == 0:
self.log.warn("Cannot find a source URI for package {0}".format(
recipe.id))
return False
from pybombs.fetcher import Fetcher
try:
if update:
if get_state() < self.inventory.STATE_CONFIGURED:
raise PBException(
"Can't update package {0}, it's not yet configured.".
format(recipe.id))
Fetcher().update(recipe)
set_state(self.inventory.STATE_CONFIGURED)
self.log.debug("State on package {0} is {1}".format(
recipe.id, get_state()))
# First, make sure we have the sources
if not update and get_state() < self.inventory.STATE_FETCHED:
Fetcher().fetch(recipe)
else:
self.log.debug("Package {0} is already fetched.".format(
recipe.id))
# If we know the package is fetched, we can attempt to build:
self.run_build(
recipe,
nuke_builddir=False,
warn_if_builddir_exists=not bool(update),
fail_if_builddir_missing=update,
)
except PBException as err:
os.chdir(cwd)
self.log.error(
"Problem occurred while building package {0}:\n{1}".format(
recipe.id,
str(err).strip()))
return False
### Housekeeping
os.chdir(cwd)
return True
def get_version(self, pkg, default_version=None):
"""
Return a package's version.
This throws a PBException if the package doesn't exist.
If no version was set, return default_version (defaults to None).
"""
if not self.has(pkg):
raise PBException("Cannot get version for package {} if it's not in the inventory!".format(pkg))
try:
return self._invfile.data[pkg]["version"]
except KeyError:
return default_version
def get_fetcher(self, src):
"""
Return scrubbed URL and fetcher for src.
"""
(url_type, url) = self.parse_uri(src)
self.log.debug("Getting fetcher for {url}, type {t}".format(
url=url, t=url_type))
fetcher = self.available.get(url_type)
if not fetcher:
raise PBException("No fetcher for type {t}".format(t=url_type))
self.log.debug("Using fetcher {0}".format(fetcher))
return (fetcher, url)
def __init__(self, filename):
# Store normalized path, in case someone chdirs after calling the ctor
self._filename = os.path.abspath(
os.path.expanduser(os.path.normpath(filename)))
self.data = None
try:
self.data = yaml.safe_load(open(filename).read()) or {}
except (IOError, OSError):
self.data = {}
except Exception as e:
raise PBException("Error loading {0}: {1}".format(
filename, str(e)))
assert isinstance(self.data, dict)
def __init__(self, cmd, args):
CommandBase.__init__(
self,
cmd,
args,
load_recipes=True,
require_prefix=True,
)
self.args.packages = args.packages[0]
if len(self.args.packages) == 0 and not args.all:
self.log.error("No packages specified.")
raise PBException("No packages specified.")
if self.args.all:
self.args.deps = False
