def update_src(self, url, dest, dirname, args=None):
"""
- src: URL, without the <type>+ prefix.
- dest: Store the fetched stuff into here
- dirname: Put the result into a dir with this name, it'll be a subdir of dest
- args: Additional args to pass to the actual fetcher
"""
args = args or {}
self.log.debug("Using url {0}".format(url))
cwd = os.getcwd()
src_dir = os.path.join(dest, dirname)
self.log.obnoxious("Switching cwd to: {0}".format(src_dir))
os.chdir(src_dir)
svn_cmd = ['svn', 'up', '--force']
if args.get('svnrev'):
svn_cmd.append('--revision')
svn_cmd.append(args.get('svnrev'))
subproc.monitor_process(
args=svn_cmd,
throw_ex=True,
#o_proc=foo #FIXME
)
self.log.obnoxious("Switching cwd back to: {0}".format(cwd))
os.chdir(cwd)
return True
def fetch_url(self, url, dest, dirname, args=None):
"""
git clone
"""
args = args or {}
self.log.debug("Using url - {}".format(url))
git_cmd = ["git", "clone", url, dirname]
if args.get("gitargs"):
for arg in args.get("gitargs").split():
git_cmd.append(arg)
if self.cfg.get("git-cache", False):
git_cmd.append("--reference")
git_cmd.append(self.cfg.get("git-cache"))
if args.get("gitbranch"):
git_cmd.append("-b")
git_cmd.append(args.get("gitbranch"))
o_proc = None
if self.log.getEffectiveLevel() >= pb_logging.DEBUG:
o_proc = output_proc.OutputProcessorMake(preamble="Cloning: ")
subproc.monitor_process(args=git_cmd, o_proc=o_proc, throw_ex=True)
# If we have a specific revision, checkout that
if args.get("gitrev"):
cwd = os.getcwd()
src_dir = os.path.join(dest, dirname)
self.log.obnoxious("Switching cwd to: {}".format(src_dir))
os.chdir(src_dir)
git_co_cmd = ["git", "checkout", "--force", args.get("gitrev")]
subproc.monitor_process(args=git_co_cmd, o_proc=o_proc, throw_ex=True)
self.log.obnoxious("Switching cwd to: {}".format(cwd))
os.chdir(cwd)
return True
def update_src(self, url, dest, dirname, args=None):
"""
- src: URL, without the <type>+ prefix.
- dest: Store the fetched stuff into here
- dirname: Put the result into a dir with this name, it'll be a subdir of dest
- args: Additional args to pass to the actual fetcher
"""
args = args or {}
self.log.debug("Using url {0}".format(url))
cwd = os.getcwd()
src_dir = os.path.join(dest, dirname)
self.log.obnoxious("Switching cwd to: {0}".format(src_dir))
os.chdir(src_dir)
svn_cmd = ['svn', 'up', '--force']
if args.get('svnrev'):
svn_cmd.append('--revision')
svn_cmd.append(args.get('svnrev'))
subproc.monitor_process(
args=svn_cmd,
throw_ex=True,
#o_proc=foo #FIXME
)
self.log.obnoxious("Switching cwd back to: {0}".format(cwd))
os.chdir(cwd)
return True
def graphviz(self, packages, dotfile, pngfile):
"""
Create the graphviz file
"""
self.log.info("Creating digraph file {0}".format(dotfile))
f = open(dotfile, "w")
f.write("digraph g {\n")
for pkg in packages:
pkg_safe = pkg.replace("-", "_")
f.write('{pkg} [label="{pkg}"]\n'.format(pkg=pkg_safe))
rec = recipe.get_recipe(pkg, fail_easy=True)
if rec is None:
continue
for dep in rec.depends:
if dep in packages:
f.write(" {pkg} -> {dep}\n".format(pkg=pkg_safe,
dep=dep.replace(
"-", "_")))
f.write("}\n")
f.close()
self.log.debug("{0} written".format(dotfile))
if pngfile is None:
return
self.log.info("Creating png file {0}".format(pngfile))
subproc.monitor_process(
['dot', dotfile, '-Tpng', '-o{0}'.format(pngfile)],
env=os.environ,
)
def graphviz(self, packages, dotfile, pngfile):
"""
Create the graphviz file
"""
self.log.info("Creating digraph file {0}".format(dotfile))
f = open(dotfile, "w")
f.write("digraph g {\n")
for pkg in packages:
pkg_safe = pkg.replace("-", "_")
f.write('{pkg} [label="{pkg}"]\n'.format(pkg=pkg_safe))
rec = recipe.get_recipe(pkg, fail_easy=True)
if rec is None:
continue
for dep in rec.depends:
if dep in packages:
f.write(" {pkg} -> {dep}\n".format(
pkg=pkg_safe,
dep=dep.replace("-", "_")
))
f.write("}\n")
f.close()
self.log.debug("{0} written".format(dotfile))
if pngfile is None:
return
self.log.info("Creating png file {0}".format(pngfile))
subproc.monitor_process(
['dot', dotfile, '-Tpng', '-o{0}'.format(pngfile)],
env=os.environ,
)
def update_src(self, url, dest, dirname, args=None):
"""
git pull / git checkout
"""
args = args or {}
self.log.debug("Using url {0}".format(url))
cwd = os.getcwd()
src_dir = os.path.join(dest, dirname)
self.log.obnoxious("Switching cwd to: {}".format(src_dir))
os.chdir(src_dir)
if args.get("gitrev"):
# If we have a rev or tag specified, fetch, then checkout.
git_cmds = [
["git", "fetch", "--tags", "--all", "--prune"],
["git", "checkout", "--force", args.get("gitrev")],
]
elif args.get("gitbranch"):
# Branch is similar, only we make sure we're up to date
# with the remote branch
git_cmds = [
["git", "fetch", "--tags", "--all", "--prune"],
["git", "checkout", "--force", args.get("gitbranch")],
["git", "reset", "--hard", "@{u}"],
]
else:
# Without a git rev, all we can do is try and pull
git_cmds = [["git", "pull", "--rebase"]]
o_proc = None
if self.log.getEffectiveLevel() >= pb_logging.DEBUG:
o_proc = output_proc.OutputProcessorMake(preamble="Updating: ")
for cmd in git_cmds:
subproc.monitor_process(args=cmd, o_proc=o_proc)
self.log.obnoxious("Switching cwd back to: {0}".format(cwd))
os.chdir(cwd)
return True
def _package_install(self,
pkgname,
comparator=">=",
required_version=None,
update=False):
"""
Call 'pip install pkgname' if we can satisfy the version requirements.
"""
try:
self.log.debug("Calling `pip install {pkg}'".format(pkg=pkgname))
command = [sysutils.which('pip'), "install"]
if update:
command.append('--upgrade')
command.append(pkgname)
subproc.monitor_process(command, elevate=True)
self.load_install_cache()
installed_version = PIP_INSTALLED_CACHE.get(pkgname)
self.log.debug("Installed version for {pkg} is: {ver}.".format(
pkg=pkgname, ver=installed_version))
if installed_version is None:
return False
if required_version is None:
return True
print required_version, comparator, installed_version
return vcompare(comparator, installed_version, required_version)
except Exception as e:
self.log.error("Running pip install failed.")
self.log.error(str(e))
return False
def _run_cmd(self, pkgname, cmd):
try:
subproc.monitor_process(['emerge',"--quiet-build y","--ask n", cmd, '"'+pkgname+'"'], elevate=True, shell=True )
return True
except Exception as e:
self.log.error("Running `emerge {}` failed.".format(cmd))
self.log.obnoxious(str(e))
return False
def update(self, pkgname):
"""
update package with 'port upgrade'
"""
try:
subproc.monitor_process(["port", "upgrade", pkgname], elevate=True, throw=True)
return True
except Exception as ex:
self.log.error("Running port upgrade failed.")
self.log.obnoxious(str(ex))
def update(self, pkgname):
"""
update package with 'port upgrade'
"""
try:
subproc.monitor_process(["port", "upgrade", pkgname], elevate=True, throw=True)
return True
except Exception as ex:
self.log.error("Running port upgrade failed.")
self.log.trace(str(ex))
def install(self, pkgname):
"""
apt(-get) -y install pkgname
"""
try:
subproc.monitor_process([self.getcmd, "-y", "install", pkgname], elevate=True, throw=True)
return True
except Exception as ex:
self.log.error("Running {0} install failed.".format(self.getcmd))
self.log.obnoxious(str(ex))
return False
def install(self, pkgname):
"""
Install package with 'port install'
"""
try:
subproc.monitor_process(["port", "install", pkgname], elevate=True, throw=True)
return True
except Exception as ex:
self.log.error("Running port install failed.")
self.log.trace(str(ex))
return False
def _run_cmd(self, pkgname, cmd):
try:
if cmd:
subproc.monitor_process(["emerge","--quiet-build","y","--ask","n",cmd,pkgname], elevate=True )
else:
subproc.monitor_process(["emerge","--quiet-build","y","--ask","n",pkgname], elevate=True )
return True
except Exception as e:
self.log.error("Running `emerge {0}` failed.".format(cmd))
self.log.obnoxious(str(e))
return False
def _run_cmd(self, pkgname, cmd):
try:
if cmd:
subproc.monitor_process(["emerge","--quiet-build","y","--ask","n",cmd,pkgname], elevate=True )
else:
subproc.monitor_process(["emerge","--quiet-build","y","--ask","n",pkgname], elevate=True )
return True
except Exception as e:
self.log.error("Running `emerge {0}` failed.".format(cmd))
self.log.obnoxious(str(e))
return False
def _run_cmd(self, pkgname, cmd):
"""
Call pacman with cmd.
"""
try:
subproc.monitor_process([self.command, "--noconfirm", cmd, pkgname], elevate=True)
return True
except Exception as ex:
self.log.error("Running `{0} {1}' failed.".format(self.command, cmd))
self.log.trace(str(ex))
return False
def deploy(self, target, prefix_dir):
"""
docstring for deploy
"""
prefix_content = [
x for x in os.listdir(prefix_dir) \
if not os.path.join(prefix_dir, x) in self.skip_names \
]
os.chdir(prefix_dir)
cmd = ['scp', '-r', '-q'] + prefix_content + [target]
subproc.monitor_process(cmd, throw_ex=True)
def deploy(self, target, prefix_dir):
"""
docstring for deploy
"""
prefix_content = [
x for x in os.listdir(prefix_dir) \
if not os.path.join(prefix_dir, x) in self.skip_names \
]
os.chdir(prefix_dir)
cmd = ['scp', '-r', '-q'] + prefix_content + [target]
subproc.monitor_process(cmd, throw_ex=True)
def install(self, pkgname):
"""
apt-get -y install pkgname
"""
try:
subproc.monitor_process(["apt-get", "-y", "install", pkgname], elevate=True, throw=True)
return True
except Exception as ex:
self.log.error("Running apt-get install failed.")
self.log.obnoxious(str(ex))
return False
def install(self, pkgname):
"""
Install package with 'port install'
"""
try:
subproc.monitor_process(["port", "install", pkgname], elevate=True, throw=True)
return True
except Exception as ex:
self.log.error("Running port install failed.")
self.log.obnoxious(str(ex))
return False
def _run_cmd(self, pkgname, cmd):
"""
Call yum or dnf with cmd.
"""
try:
subproc.monitor_process([self.command, "-y", cmd, pkgname], elevate=True)
return True
except Exception as ex:
self.log.error("Running `{0} install' failed.".format(self.command))
self.log.obnoxious(str(ex))
return False
def _run_cmd(self, pkgname, cmd):
"""
Call pacman with cmd.
"""
try:
subproc.monitor_process(
[self.command, "--noconfirm", cmd, pkgname], elevate=True)
return True
except Exception as ex:
self.log.error("Running `{} {}' failed.".format(self.command, cmd))
self.log.obnoxious(str(ex))
return False
def install(self, pkgname):
"""
apt-get -y install pkgname
"""
try:
subproc.monitor_process(["apt-get", "-y", "install", pkgname],
elevate=True,
throw=True)
return True
except Exception as ex:
self.log.error("Running apt-get install failed.")
self.log.obnoxious(str(ex))
return False
def install(self, pkgname):
"""
apt(-get) -y install pkgname
"""
try:
subproc.monitor_process([self.getcmd, "-y", "install", pkgname],
elevate=True,
throw=True)
return True
except Exception as ex:
self.log.error("Running {0} install failed.".format(self.getcmd))
self.log.obnoxious(str(ex))
return False
def install(self, pkgname):
"""
Call 'brew install pkgname' if we can satisfy the version requirements.
"""
try:
# Need to do some better checking here. Brew does not necessarily need sudo
#sysutils.monitor_process(["sudo", "brew", "", "install", pkg_name])
subproc.monitor_process(["brew", "install", pkgname])
return True
except Exception as e:
#self.log.obnoxious(e)
self.log.error("Running brew install failed.")
return False
def _package_install(self, pkg_name, comparator=">=", required_version=None):
"""
Call 'apt-get install pkgname' if we can satisfy the version requirements.
"""
available_version = self.get_version_from_apt_cache(pkg_name)
if required_version is not None and not vcompare(comparator, available_version, required_version):
return False
try:
subproc.monitor_process(["apt-get", "-y", "install", pkg_name], elevate=True)
return True
except:
self.log.error("Running apt-get install failed.")
return False
def install(self, pkgname):
"""
Call 'brew install pkgname' if we can satisfy the version requirements.
"""
try:
# Need to do some better checking here. Brew does not necessarily need sudo
#sysutils.monitor_process(["sudo", "brew", "", "install", pkg_name])
subproc.monitor_process(["brew", "install", pkgname])
return True
except Exception as e:
#self.log.obnoxious(e)
self.log.error("Running brew install failed.")
return False
def _package_install(self, pkgname, comparator=">=", required_version=None, cmd="install"):
"""
Call 'COMMAND install pkgname' if we can satisfy the version requirements.
"""
available_version = self.get_available_version_from_pkgr(pkgname)
if required_version is not None and not vcompare(comparator, available_version, required_version):
return False
try:
subproc.monitor_process([self.command, "-y", cmd, pkgname], elevate=True)
return True
except Exception as ex:
self.log.error("Running `{0} install' failed.".format(self.command))
self.log.obnoxious(str(ex))
return False
def deploy(self, target, prefix_dir):
"""
Call TarfileDeployer to make a .tar file, then use xz
to zip that
"""
tarfilename = target
if os.path.splitext(target)[1] == '.xz':
tarfilename = os.path.splitext(target)[0]
elif os.path.splitext(target)[1] == '.txz':
tarfilename = os.path.splitext(target)[0] + '.tar'
result_filename = tarfilename + '.xz'
self.tarfiledeployer.deploy(tarfilename, prefix_dir)
subproc.monitor_process(['xz', tarfilename], throw_ex=True)
if result_filename != target:
os.rename(result_filename, target)
def deploy(self, target, prefix_dir):
"""
Call TarfileDeployer to make a .tar file, then use xz
to zip that
"""
tarfilename = target
if os.path.splitext(target)[1] == '.xz':
tarfilename = os.path.splitext(target)[0]
elif os.path.splitext(target)[1] == '.txz':
tarfilename = os.path.splitext(target)[0] + '.tar'
result_filename = tarfilename + '.xz'
self.tarfiledeployer.deploy(tarfilename, prefix_dir)
subproc.monitor_process(['xz', tarfilename], throw_ex=True)
if result_filename != target:
os.rename(result_filename, target)
def configure(self, recipe, try_again=False):
"""
Run the configuration step for this recipe.
If try_again is set, it will assume the configuration failed before
and we're trying to run it again.
"""
self.log.debug("Configuring recipe {}".format(recipe.id))
self.log.debug("Using vars - {}".format(recipe.vars))
self.log.debug("In cwd - {}".format(os.getcwd()))
pre_cmd = recipe.var_replace_all(self.get_command('configure', recipe))
cmd = self.filter_cmd(pre_cmd, recipe, 'config_filter')
o_proc = None
if self.log.getEffectiveLevel() >= pb_logging.DEBUG and not try_again:
o_proc = output_proc.OutputProcessorMake(preamble="Configuring: ")
if subproc.monitor_process(cmd, shell=True, o_proc=o_proc) == 0:
self.log.debug("Configure successful.")
return True
# OK, something went wrong.
if try_again == False:
self.log.warning(
"Configuration failed. Re-trying with higher verbosity.")
return self.configure(recipe, try_again=True)
else:
self.log.error(
"Configuration failed after running at least twice.")
raise PBException("Configuration failed")
def make(self, recipe, try_again=False):
"""
Build this recipe.
If try_again is set, it will assume the build failed before
and we're trying to run it again. In this case, reduce the
makewidth to 1 and show the build output.
"""
self.log.debug("Building recipe {}".format(recipe.id))
self.log.debug("In cwd - {}".format(os.getcwd()))
o_proc = None
if self.log.getEffectiveLevel(
) >= pb_logging.DEBUG and not try_again and not recipe.make_interactive:
o_proc = output_proc.OutputProcessorMake(preamble="Building: ")
cmd = recipe.var_replace_all(self.get_command('make', recipe))
cmd = self.filter_cmd(cmd, recipe, 'make_filter')
if subproc.monitor_process(cmd, shell=True, o_proc=o_proc) == 0:
self.log.debug("Make successful")
return True
# OK, something bad happened.
if try_again == False:
recipe.vars['makewidth'] = '1'
self.make(recipe, try_again=True)
else:
self.log.error(
"Build failed. See output above for error messages.")
raise PBException("Build failed.")
def fetch_url(self, url, dest, dirname, args=None):
"""
git clone
"""
git_version = get_git_version()
self.log.debug("We have git version %s", git_version)
url, args = parse_git_url(url, args or {})
self.log.debug("Using url - {0}".format(url))
git_cmd = ['git', 'clone', url, dirname]
if args.get('gitargs'):
for arg in args.get('gitargs').split():
git_cmd.append(arg)
if self.cfg.get("git-cache", False):
from pybombs import gitcache_manager
gcm = gitcache_manager.GitCacheManager(self.cfg.get("git-cache"))
self.log.debug("Adding remote into git ref")
gcm.add_remote(dirname, url, True)
git_cmd.append(
'--reference-if-able'
if vcompare(">=", git_version, "2.11") else '--reference'
)
git_cmd.append(self.cfg.get("git-cache"))
if args.get('gitbranch'):
git_cmd.append('-b')
git_cmd.append(args.get('gitbranch'))
o_proc = None
subproc.monitor_process(
args=git_cmd,
o_proc=o_proc,
throw_ex=True,
throw=True,
)
# If we have a specific revision, checkout that
if args.get('gitrev'):
cwd = os.getcwd()
src_dir = os.path.join(dest, dirname)
self.log.trace("Switching cwd to: {0}".format(src_dir))
os.chdir(src_dir)
git_co_cmd = ["git", "checkout", "--force", args.get('gitrev')]
subproc.monitor_process(
args=git_co_cmd,
o_proc=o_proc,
throw_ex=True,
)
self.log.trace("Switching cwd to: {0}".format(cwd))
os.chdir(cwd)
return True
def fetch_url(self, url, dest, dirname, args=None):
"""
git clone
"""
git_version = get_git_version()
self.log.debug("We have git version %s", git_version)
url, args = parse_git_url(url, args or {})
self.log.debug("Using url - {0}".format(url))
git_cmd = ['git', 'clone', url, dirname]
if args.get('gitargs'):
for arg in args.get('gitargs').split():
git_cmd.append(arg)
if self.cfg.get("git-cache", False):
from pybombs import gitcache_manager
gcm = gitcache_manager.GitCacheManager(self.cfg.get("git-cache"))
self.log.debug("Adding remote into git ref")
gcm.add_remote(dirname, url, True)
git_cmd.append(
'--reference-if-able'
if vcompare(">=", git_version, "2.11") else '--reference'
)
git_cmd.append(self.cfg.get("git-cache"))
if args.get('gitbranch'):
git_cmd.append('-b')
git_cmd.append(args.get('gitbranch'))
o_proc = None
subproc.monitor_process(
args=git_cmd,
o_proc=o_proc,
throw_ex=True,
throw=True,
)
# If we have a specific revision, checkout that
if args.get('gitrev'):
cwd = os.getcwd()
src_dir = os.path.join(dest, dirname)
self.log.obnoxious("Switching cwd to: {0}".format(src_dir))
os.chdir(src_dir)
git_co_cmd = ["git", "checkout", "--force", args.get('gitrev')]
subproc.monitor_process(
args=git_co_cmd,
o_proc=o_proc,
throw_ex=True,
)
self.log.obnoxious("Switching cwd to: {0}".format(cwd))
os.chdir(cwd)
return True
def _install_sdk_to_prefix(self, sdkname):
"""
Read recipe for sdkname, and install the SDK to the prefix.
"""
from pybombs import recipe
src_dir = self.prefix.src_dir
cfg_file = self.prefix.cfg_file
### Get the recipe
r = recipe.get_recipe(sdkname, target='sdk')
try:
self.log.trace("Switching CWD to {0}".format(src_dir))
if not op.isdir(src_dir):
os.mkdir(src_dir)
os.chdir(src_dir)
except:
self.log.error("Source dir required to install SDK.")
return False
### Install the actual SDK file
self.log.debug("Fetching SDK `{sdk}'".format(sdk=sdkname))
fetcher.Fetcher().fetch(r)
self.log.info("Installing SDK `{sdk}'".format(sdk=sdkname))
# Install command
cmd = r.var_replace_all(r.get_command('install'))
if subproc.monitor_process(cmd, shell=True, env=os.environ) == 0:
self.log.debug("Installation successful")
else:
self.log.error("Error installing SDK. Aborting.")
return False
# Clean up
files_to_delete = [op.normpath(op.join(src_dir, r.var_replace_all(x))) for x in r.clean]
if len(files_to_delete):
self.log.info("Cleaning up files...")
for ftd in files_to_delete:
if op.commonprefix((src_dir, ftd)) != src_dir:
self.log.warn("Not removing {ftd} -- outside source dir!".format(ftd=ftd))
continue
self.log.debug("Removing {ftd}...".format(ftd=ftd))
if op.isdir(ftd):
shutil.rmtree(ftd)
elif op.isfile(ftd):
os.remove(ftd)
else:
self.log.error("Not sure what this is: {ftd}".format(ftd=ftd))
return False
### Update the prefix-local config file
self.log.debug("Updating config file with SDK recipe info.")
try:
old_cfg_data = PBConfigFile(cfg_file).get()
except IOError:
self.log.debug("There doesn't seem to be a config file yet for this prefix.")
old_cfg_data = {}
# Filter out keys we don't care about:
sdk_recipe_keys_for_config = ('config', 'packages', 'categories', 'env')
sdk_cfg_data = {k: v for k, v in iteritems(r.get_dict()) if k in sdk_recipe_keys_for_config}
self.log.trace("New data: {new}".format(new=sdk_cfg_data))
cfg_data = dict_merge(old_cfg_data, sdk_cfg_data)
self.log.debug("Writing updated prefix config to `{0}'".format(cfg_file))
PBConfigFile(cfg_file).save(cfg_data)
return True
def _run_pip_install(self, pkgname, update=False):
"""
Run pip install [--upgrade]
"""
try:
command = [sysutils.which('pip'), "install"]
if update:
command.append('--upgrade')
command.append(pkgname)
self.log.debug("Calling `{cmd}'".format(cmd=" ".join(command)))
subproc.monitor_process(command, elevate=True)
self.load_install_cache()
return True
except Exception as e:
self.log.error("Running pip install failed.")
self.log.debug(str(e))
return None
def _run_pip_install(self, pkgname, update=False):
"""
Run pip install [--upgrade]
"""
try:
command = [sysutils.which('pip'), "install"]
if update:
command.append('--upgrade')
command.append(pkgname)
self.log.debug("Calling `{cmd}'".format(cmd=" ".join(command)))
subproc.monitor_process(command, elevate=True)
self.load_install_cache()
return True
except Exception as e:
self.log.error("Running pip install failed.")
self.log.debug(str(e))
return None
def _package_install(self,
pkg_name,
comparator=">=",
required_version=None):
"""
Call 'apt-get install pkgname' if we can satisfy the version requirements.
"""
available_version = self.get_version_from_apt_cache(pkg_name)
if required_version is not None and not vcompare(
comparator, available_version, required_version):
return False
try:
subproc.monitor_process(["apt-get", "-y", "install", pkg_name],
elevate=True)
return True
except:
self.log.error("Running apt-get install failed.")
return False
def fetch_url(self, url, dest, dirname, args=None):
"""
git clone
"""
url, args = parse_git_url(url, args or {})
self.log.debug("Using url - {0}".format(url))
git_cmd = ['git', 'clone', url, dirname]
if args.get('gitargs'):
for arg in args.get('gitargs').split():
git_cmd.append(arg)
if self.cfg.get("git-cache", False):
from pybombs import gitcache_manager
gcm = gitcache_manager.GitCacheManager(self.cfg.get("git-cache"))
self.log.debug("Adding remote into git ref")
gcm.add_remote(dirname, url, True)
git_cmd.append('--reference')
git_cmd.append(self.cfg.get("git-cache"))
if args.get('gitbranch'):
git_cmd.append('-b')
git_cmd.append(args.get('gitbranch'))
o_proc = None
if self.log.getEffectiveLevel() >= pb_logging.DEBUG:
o_proc = output_proc.OutputProcessorMake(preamble="Cloning: ")
subproc.monitor_process(
args=git_cmd,
o_proc=o_proc,
throw_ex=True,
throw=True,
)
# If we have a specific revision, checkout that
if args.get('gitrev'):
cwd = os.getcwd()
src_dir = os.path.join(dest, dirname)
self.log.obnoxious("Switching cwd to: {0}".format(src_dir))
os.chdir(src_dir)
git_co_cmd = ["git", "checkout", "--force", args.get('gitrev')]
subproc.monitor_process(
args=git_co_cmd,
o_proc=o_proc,
throw_ex=True,
)
self.log.obnoxious("Switching cwd to: {0}".format(cwd))
os.chdir(cwd)
return True
def fetch_url(self, url, dest, dirname, args=None):
"""
git clone
"""
url, args = parse_git_url(url, args or {})
self.log.debug("Using url - {0}".format(url))
git_cmd = ['git', 'clone', url, dirname]
if args.get('gitargs'):
for arg in args.get('gitargs').split():
git_cmd.append(arg)
if self.cfg.get("git-cache", False):
from pybombs import gitcache_manager
gcm = gitcache_manager.GitCacheManager(self.cfg.get("git-cache"))
self.log.debug("Adding remote into git ref")
gcm.add_remote(dirname, url, True)
git_cmd.append('--reference')
git_cmd.append(self.cfg.get("git-cache"))
if args.get('gitbranch'):
git_cmd.append('-b')
git_cmd.append(args.get('gitbranch'))
o_proc = None
if self.log.getEffectiveLevel() >= pb_logging.DEBUG:
o_proc = output_proc.OutputProcessorMake(preamble="Cloning: ")
subproc.monitor_process(
args=git_cmd,
o_proc=o_proc,
throw_ex=True,
throw=True,
)
# If we have a specific revision, checkout that
if args.get('gitrev'):
cwd = os.getcwd()
src_dir = os.path.join(dest, dirname)
self.log.obnoxious("Switching cwd to: {0}".format(src_dir))
os.chdir(src_dir)
git_co_cmd = ["git", "checkout", "--force", args.get('gitrev')]
subproc.monitor_process(
args=git_co_cmd,
o_proc=o_proc,
throw_ex=True,
)
self.log.obnoxious("Switching cwd to: {0}".format(cwd))
os.chdir(cwd)
return True
def update_src(self, url, dest, dirname, args=None):
"""
git pull / git checkout
"""
args = args or {}
self.log.debug("Using url {0}".format(url))
cwd = os.getcwd()
src_dir = os.path.join(dest, dirname)
self.log.obnoxious("Switching cwd to: {}".format(src_dir))
os.chdir(src_dir)
if args.get('gitrev'):
# If we have a rev or tag specified, fetch, then checkout.
git_cmds = [
['git', 'fetch', '--tags', '--all', '--prune'],
['git', 'checkout', '--force',
args.get('gitrev')],
]
elif args.get('gitbranch'):
# Branch is similar, only we make sure we're up to date
# with the remote branch
git_cmds = [
['git', 'fetch', '--tags', '--all', '--prune'],
['git', 'checkout', '--force',
args.get('gitbranch')],
['git', 'reset', '--hard', '@{u}'],
]
else:
# Without a git rev, all we can do is try and pull
git_cmds = [
['git', 'pull', '--rebase'],
]
o_proc = None
if self.log.getEffectiveLevel() >= pb_logging.DEBUG:
o_proc = output_proc.OutputProcessorMake(preamble="Updating: ")
for cmd in git_cmds:
subproc.monitor_process(
args=cmd,
o_proc=o_proc,
)
self.log.obnoxious("Switching cwd back to: {0}".format(cwd))
os.chdir(cwd)
return True
def fetch_url(self, url, dest, dirname, args=None):
"""
- url: SVN repo url
- dest: src dir
- dirname: Put the result into a dir with this name, it'll be a subdir of dest
- args: Additional args to pass to the actual fetcher
"""
args = args or {}
svn_cmd = ['svn', 'co']
if args.get('svnrev') is not None:
svn_cmd.append('-r')
svn_cmd.append(args.get('svnrev'))
svn_cmd.append(url)
svn_cmd.append(dirname)
subproc.monitor_process(
args=svn_cmd,
#o_proc=foo, # FIXME
throw_ex=True,
)
return True
def fetch_url(self, url, dest, dirname, args=None):
"""
- url: SVN repo url
- dest: src dir
- dirname: Put the result into a dir with this name, it'll be a subdir of dest
- args: Additional args to pass to the actual fetcher
"""
args = args or {}
svn_cmd = ['svn', 'co']
if args.get('svnrev') is not None:
svn_cmd.append('-r')
svn_cmd.append(args.get('svnrev'))
svn_cmd.append(url)
svn_cmd.append(dirname)
subproc.monitor_process(
args=svn_cmd,
#o_proc=foo, # FIXME
throw_ex=True,
)
return True
def _package_install(self,
pkgname,
comparator=">=",
required_version=None,
cmd='install'):
"""
Call 'COMMAND install pkgname' if we can satisfy the version requirements.
"""
available_version = self.get_available_version_from_pkgr(pkgname)
if required_version is not None and not vcompare(
comparator, available_version, required_version):
return False
try:
subproc.monitor_process([self.command, "-y", cmd, pkgname],
elevate=True)
return True
except Exception as ex:
self.log.error("Running `{0} install' failed.".format(
self.command))
self.log.obnoxious(str(ex))
return False
def update_src(self, url, dest, dirname, args=None):
"""
git pull / git checkout
"""
args = args or {}
self.log.debug("Using url {0}".format(url))
cwd = os.getcwd()
src_dir = os.path.join(dest, dirname)
self.log.obnoxious("Switching cwd to: {0}".format(src_dir))
os.chdir(src_dir)
if args.get('gitrev'):
# If we have a rev or tag specified, fetch, then checkout.
git_cmds = [
['git', 'fetch', '--tags', '--all', '--prune'],
['git', 'checkout', '--force',
args.get('gitrev')],
]
elif args.get('gitbranch'):
# Branch is similar, only we make sure we're up to date
# with the remote branch
git_cmds = [
['git', 'fetch', '--tags', '--all', '--prune'],
['git', 'checkout', '--force',
args.get('gitbranch')],
['git', 'reset', '--hard', '@{u}'],
]
else:
# Without a git rev, all we can do is try and pull
git_cmds = [
['git', 'pull', '--rebase'],
]
git_cmds.append(['git', 'submodule', 'update', '--recursive'])
o_proc = None
if self.log.getEffectiveLevel() >= pb_logging.DEBUG:
o_proc = output_proc.OutputProcessorMake(preamble="Updating: ")
for cmd in git_cmds:
try:
if subproc.monitor_process(
args=cmd, o_proc=o_proc, throw_ex=True) != 0:
self.log.error("Could not run command `{0}`".format(
" ".join(cmd)))
return False
except Exception:
self.log.error("Could not run command `{0}`".format(
" ".join(cmd)))
raise PBException("git commands failed.")
self.log.obnoxious("Switching cwd back to: {0}".format(cwd))
os.chdir(cwd)
return True
def make_install(self, recipe):
"""
Run 'make install' or whatever copies the files to the right place.
"""
self.log.debug("Installing package {}".format(recipe.id))
self.log.debug("In cwd - {}".format(os.getcwd()))
pre_cmd = recipe.var_replace_all(self.get_command('install', recipe))
cmd = self.filter_cmd(pre_cmd, recipe, 'install_filter')
o_proc = None
if self.log.getEffectiveLevel() >= pb_logging.DEBUG:
o_proc = output_proc.OutputProcessorMake(preamble="Installing: ")
if subproc.monitor_process(cmd, shell=True, o_proc=o_proc) == 0:
self.log.debug("Installation successful")
return True
raise PBException("Installation failed")
def make_install(self, recipe):
"""
Run 'make install' or whatever copies the files to the right place.
"""
self.log.debug("Installing package {0}".format(recipe.id))
self.log.debug("In cwd - {0}".format(os.getcwd()))
pre_cmd = recipe.var_replace_all(self.get_command('install', recipe))
cmd = self.filter_cmd(pre_cmd, recipe, 'install_filter')
o_proc = None
if self.log.getEffectiveLevel() >= pb_logging.DEBUG:
o_proc = output_proc.OutputProcessorMake(preamble="Installing: ")
if subproc.monitor_process(cmd, shell=True, o_proc=o_proc) == 0:
self.log.debug("Installation successful")
return True
raise PBException("Installation failed")
def _download_with_wget(url):
" Use the wget tool itself "
def get_md5(filename):
" Return MD5 sum of filename using the md5sum tool "
md5_exe = sysutils.which('md5sum')
if md5_exe is not None:
return subproc.check_output([md5_exe, filename])[0:32]
return None
from pybombs.utils import sysutils
from pybombs.utils import subproc
wget = sysutils.which('wget')
if wget is None:
raise PBException("wget executable not found")
filename = os.path.split(url)[1]
retval = subproc.monitor_process([wget, url], throw=True)
if retval:
raise PBException("wget failed to wget")
return filename, get_md5(filename)
def update_src(self, url, dest, dirname, args=None):
"""
git pull / git checkout
"""
args = args or {}
self.log.debug("Using url {0}".format(url))
cwd = os.getcwd()
src_dir = os.path.join(dest, dirname)
self.log.obnoxious("Switching cwd to: {0}".format(src_dir))
os.chdir(src_dir)
if args.get('gitrev'):
# If we have a rev or tag specified, fetch, then checkout.
git_cmds = [
['git', 'fetch', '--tags', '--all', '--prune'],
['git', 'checkout', '--force', args.get('gitrev')],
]
elif args.get('gitbranch'):
# Branch is similar, only we make sure we're up to date
# with the remote branch
git_cmds = [
['git', 'fetch', '--tags', '--all', '--prune'],
['git', 'checkout', '--force', args.get('gitbranch')],
['git', 'reset', '--hard', '@{u}'],
]
else:
# Without a git rev, all we can do is try and pull
git_cmds = [
['git', 'pull', '--rebase'],
]
git_cmds.append(['git', 'submodule', 'update', '--recursive'])
o_proc = None
if self.log.getEffectiveLevel() >= pb_logging.DEBUG:
o_proc = output_proc.OutputProcessorMake(preamble="Updating: ")
for cmd in git_cmds:
try:
if subproc.monitor_process(args=cmd, o_proc=o_proc, throw_ex=True) != 0:
self.log.error("Could not run command `{0}`".format(" ".join(cmd)))
return False
except Exception:
self.log.error("Could not run command `{0}`".format(" ".join(cmd)))
raise PBException("git commands failed.")
self.log.obnoxious("Switching cwd back to: {0}".format(cwd))
os.chdir(cwd)
return True
def _download_with_wget(url):
" Use the wget tool itself "
def get_md5(filename):
" Return MD5 sum of filename using the md5sum tool "
md5_exe = sysutils.which('md5sum')
if md5_exe is not None:
return subproc.check_output([md5_exe, filename])[0:32]
return None
from pybombs.utils import sysutils
from pybombs.utils import subproc
wget = sysutils.which('wget')
if wget is None:
raise PBException("wget executable not found")
filename = os.path.split(url)[1]
retval = subproc.monitor_process([wget, url], throw=True)
if retval:
raise PBException("wget failed to wget")
return filename, get_md5(filename)
def make_clean(self, recipe, try_again=False):
"""
Run 'make clean' or whatever clears a build before recompiling
"""
self.log.debug("Uninstalling from recipe {0}".format(recipe.id))
self.log.debug("In cwd - {0}".format(os.getcwd()))
o_proc = None
if self.log.getEffectiveLevel() >= pb_logging.DEBUG and not try_again:
o_proc = output_proc.OutputProcessorMake(preamble="Uninstalling: ")
cmd = recipe.var_replace_all(self.get_command('uninstall', recipe))
cmd = self.filter_cmd(cmd, recipe, 'uninstall_filter')
if subproc.monitor_process(cmd, shell=True, o_proc=o_proc) == 0:
self.log.debug("Uninstall successful")
return True
# OK, something bad happened.
if try_again == False:
recipe.vars['makewidth'] = '1'
self.make_clean(recipe, try_again=True)
else:
self.log.error("Uninstall failed. See output above for error messages.")
raise PBException("Uninstall failed.")
def make_clean(self, recipe, try_again=False):
"""
Run 'make clean' or whatever clears a build before recompiling
"""
self.log.debug("Uninstalling from recipe {}".format(recipe.id))
self.log.debug("In cwd - {}".format(os.getcwd()))
o_proc = None
if self.log.getEffectiveLevel() >= pb_logging.DEBUG and not try_again:
o_proc = output_proc.OutputProcessorMake(preamble="Uninstalling: ")
cmd = recipe.var_replace_all(self.get_command('uninstall', recipe))
cmd = self.filter_cmd(cmd, recipe, 'uninstall_filter')
if subproc.monitor_process(cmd, shell=True, o_proc=o_proc) == 0:
self.log.debug("Uninstall successful")
return True
# OK, something bad happened.
if try_again == False:
recipe.vars['makewidth'] = '1'
self.make_clean(recipe, try_again=True)
else:
self.log.error("Uninstall failed. See output above for error messages.")
raise PBException("Uninstall failed.")
def verify(self, recipe, try_again=False):
"""
Run 'make test' or whatever checks a build was successful
"""
self.log.debug("Verifying package {0}".format(recipe.id))
self.log.debug("In cwd - {0}".format(os.getcwd()))
o_proc = None
if self.log.getEffectiveLevel() >= pb_logging.DEBUG and not try_again:
o_proc = output_proc.OutputProcessorMake(preamble="Verifying: ")
cmd = recipe.var_replace_all(self.get_command('verify', recipe))
cmd = self.filter_cmd(cmd, recipe, 'make_filter')
if subproc.monitor_process(cmd, shell=True, o_proc=o_proc) == 0:
self.log.debug("Verification successful")
return True
# OK, something bad happened.
if try_again == False:
self.log.warning("Verification failed. Retrying...")
recipe.vars['makewidth'] = '1'
self.verify(recipe, try_again=True)
else:
self.log.error("Verification failed. See output above for error messages.")
raise PBException("Verification failed.")
def verify(self, recipe, try_again=False):
"""
Run 'make test' or whatever checks a build was successful
"""
self.log.debug("Verifying package {}".format(recipe.id))
self.log.debug("In cwd - {}".format(os.getcwd()))
o_proc = None
if self.log.getEffectiveLevel() >= pb_logging.DEBUG and not try_again:
o_proc = output_proc.OutputProcessorMake(preamble="Verifying: ")
cmd = recipe.var_replace_all(self.get_command('verify', recipe))
cmd = self.filter_cmd(cmd, recipe, 'make_filter')
if subproc.monitor_process(cmd, shell=True, o_proc=o_proc) == 0:
self.log.debug("Verification successful")
return True
# OK, something bad happened.
if try_again == False:
self.log.warning("Verification failed. Retrying...")
recipe.vars['makewidth'] = '1'
self.verify(recipe, try_again=True)
else:
self.log.error("Verification failed. See output above for error messages.")
raise PBException("Verification failed.")
def make(self, recipe, try_again=False):
"""
Build this recipe.
If try_again is set, it will assume the build failed before
and we're trying to run it again. In this case, reduce the
makewidth to 1 and show the build output.
"""
self.log.debug("Building recipe {0}".format(recipe.id))
self.log.debug("In cwd - {0}".format(os.getcwd()))
o_proc = None
if self.log.getEffectiveLevel() >= pb_logging.DEBUG and not try_again and not recipe.make_interactive:
o_proc = output_proc.OutputProcessorMake(preamble="Building: ")
cmd = recipe.var_replace_all(self.get_command('make', recipe))
cmd = self.filter_cmd(cmd, recipe, 'make_filter')
if subproc.monitor_process(cmd, shell=True, o_proc=o_proc) == 0:
self.log.debug("Make successful")
return True
# OK, something bad happened.
if try_again == False:
recipe.vars['makewidth'] = '1'
self.make(recipe, try_again=True)
else:
self.log.error("Build failed. See output above for error messages.")
raise PBException("Build failed.")
def configure(self, recipe, try_again=False):
"""
Run the configuration step for this recipe.
If try_again is set, it will assume the configuration failed before
and we're trying to run it again.
"""
self.log.debug("Configuring recipe {0}".format(recipe.id))
self.log.debug("Using vars - {0}".format(recipe.vars))
self.log.debug("In cwd - {0}".format(os.getcwd()))
pre_cmd = recipe.var_replace_all(self.get_command('configure', recipe))
cmd = self.filter_cmd(pre_cmd, recipe, 'config_filter')
o_proc = None
if self.log.getEffectiveLevel() >= pb_logging.DEBUG and not try_again:
o_proc = output_proc.OutputProcessorMake(preamble="Configuring: ")
if subproc.monitor_process(cmd, shell=True, o_proc=o_proc) == 0:
self.log.debug("Configure successful.")
return True
# OK, something went wrong.
if try_again == False:
self.log.warning("Configuration failed. Re-trying with higher verbosity.")
return self.configure(recipe, try_again=True)
else:
self.log.error("Configuration failed after running at least twice.")
raise PBException("Configuration failed")
def create_virtualenv(path):
" Create a Python virtualenv "
self.log.info("Creating Python virtualenv in prefix...")
venv_args = ['virtualenv']
venv_args.append(path)
subproc.monitor_process(args=venv_args)
def _run_init(self):
"""
pybombs prefix init
"""
def register_alias(alias):
if alias is not None:
if self.prefix is not None and \
self.prefix.prefix_aliases.get(alias) is not None \
and not confirm("Alias `{0}' already exists, overwrite?".format(alias)):
self.log.warn('Aborting.')
raise PBException("Could not create alias.")
self.cfg.update_cfg_file(
{'prefix_aliases': {
self.args.alias: path
}})
# Go, go, go!
try:
prefix_recipe = get_prefix_recipe(self.args.recipe)
except PBException as ex:
self.log.error(str(ex))
return -1
if prefix_recipe is None:
self.log.error("Could not find recipe for `{0}'".format(
self.args.recipe))
return -1
# Make sure the directory is writable
path = op.abspath(op.normpath(self.args.path))
if not sysutils.mkdir_writable(path, self.log):
self.log.error("Cannot write to prefix path `{0}'.".format(path))
return -1
# Make sure that a pybombs directory doesn't already exist
from pybombs import config_manager
if op.exists(op.join(path, config_manager.PrefixInfo.prefix_conf_dir)):
self.log.error(
"Ignoring. A prefix already exists in `{0}'".format(path))
return -1
# Add subdirs
sysutils.require_subdirs(
path, [k for k, v in prefix_recipe.dirs.items() if v])
self.cfg.load(select_prefix=path)
self.prefix = self.cfg.get_active_prefix()
# Create files
for fname, content in prefix_recipe.files.items():
sysutils.write_file_in_subdir(
path, fname, prefix_recipe.var_replace_all(content))
# Register alias
if self.args.alias is not None:
register_alias(self.args.alias)
# If there is no default prefix, make this the default
if len(self.cfg.get('default_prefix')) == 0:
if self.args.alias is not None:
new_default_prefix = self.args.alias
else:
new_default_prefix = path
self.cfg.update_cfg_file(
{'config': {
'default_prefix': new_default_prefix
}})
# Create virtualenv if so desired
if self.args.virtualenv:
self.log.info("Creating Python virtualenv in prefix...")
venv_args = ['virtualenv']
venv_args.append(path)
subproc.monitor_process(args=venv_args)
# Install SDK if so desired
sdk = self.args.sdkname or prefix_recipe.sdk
if sdk is not None:
self.log.info("Installing SDK recipe {0}.".format(sdk))
self.log.info("Reloading configuration...")
self.cfg.load(select_prefix=path)
self.prefix = self.cfg.get_active_prefix()
self.inventory = self.prefix.inventory
self._install_sdk_to_prefix(sdk)
# Update config section
if len(prefix_recipe.config):
self.cfg.update_cfg_file(prefix_recipe.config,
self.prefix.cfg_file)
self.cfg.load(select_prefix=path)
self.prefix = self.cfg.get_active_prefix()
# Install dependencies
if len(prefix_recipe.depends):
self.log.info("Installing default packages for prefix...")
self.log.info("".join(
["\n - {0}".format(x) for x in prefix_recipe.depends]))
from pybombs import install_manager
install_manager.InstallManager().install(
prefix_recipe.depends,
'install', # install / update
fail_if_not_exists=False,
update_if_exists=False,
print_tree=True,
)