def execute_template(*cmds, **kargs):
if(not cmds or len(cmds) == 0):
return
params_replacements = kargs.pop('params')
ignore_missing = kargs.pop('ignore_missing', False)
outs = dict()
for cmdinfo in cmds:
cmd_to_run_templ = cmdinfo.get("cmd")
cmd_to_run = list()
for piece in cmd_to_run_templ:
if(params_replacements and len(params_replacements)):
cmd_to_run.append(param_replace(piece, params_replacements,
ignore_missing=ignore_missing))
else:
cmd_to_run.append(piece)
stdin_templ = cmdinfo.get('stdin')
stdin = None
if(stdin_templ and len(stdin_templ)):
stdin_full = list()
for piece in stdin_templ:
if(params_replacements and len(params_replacements)):
stdin_full.append(param_replace(piece, params_replacements,
ignore_missing=ignore_missing))
else:
stdin_full.append(piece)
stdin = joinlinesep(*stdin_full)
root_run = cmdinfo.get('run_as_root', False)
execute(*cmd_to_run, run_as_root=root_run, process_input=stdin, **kargs)
def _run_cmd(self, cmd):
if(self.distro == UBUNTU11):
with TemporaryFile() as f:
execute(*cmd, run_as_root=True,
stdout_fh=f, stderr_fh=f)
else:
execute(*cmd, run_as_root=True)
def install(self):
pres = PkgInstallComponent.install(self)
#extra actions to ensure we are granted access
dbtype = self.cfg.get("db", "type")
dbactions = DB_ACTIONS.get(dbtype)
if (dbactions and dbactions.get('grant_all')):
#update the DB to give user 'USER'@'%' full control of the all databases:
grant_cmd = dbactions.get('grant_all')
params = self._get_param_map()
cmds = list()
cmds.append({
'cmd': grant_cmd,
'run_as_root': False,
})
#shell seems to be needed here
#since python escapes this to much...
execute_template(*cmds, params=params, shell=True)
#special mysql actions
if (dbactions and dbtype == MYSQL):
cmd = dbactions.get('host_adjust')
if (cmd):
execute(*cmd, run_as_root=True, shell=True)
#restart it to make sure all good
self.runtime.restart()
return pres
def install(self):
pres = PkgInstallComponent.install(self)
#extra actions to ensure we are granted access
dbtype = self.cfg.get("db", "type")
dbactions = DB_ACTIONS.get(dbtype)
if(dbactions and dbactions.get('grant_all')):
#update the DB to give user 'USER'@'%' full control of the all databases:
grant_cmd = dbactions.get('grant_all')
params = self._get_param_map()
cmds = list()
cmds.append({
'cmd': grant_cmd,
'run_as_root': False,
})
#shell seems to be needed here
#since python escapes this to much...
execute_template(*cmds, params=params, shell=True)
#special mysql actions
if(dbactions and dbtype == MYSQL):
cmd = dbactions.get('host_adjust')
if(cmd):
execute(*cmd, run_as_root=True, shell=True)
#restart it to make sure all good
self.runtime.restart()
return pres
def execute_template(*cmds, **kargs):
if (not cmds or len(cmds) == 0):
return
params_replacements = kargs.pop('params')
ignore_missing = kargs.pop('ignore_missing', False)
outs = dict()
for cmdinfo in cmds:
cmd_to_run_templ = cmdinfo.get("cmd")
cmd_to_run = list()
for piece in cmd_to_run_templ:
if (params_replacements and len(params_replacements)):
cmd_to_run.append(
param_replace(piece,
params_replacements,
ignore_missing=ignore_missing))
else:
cmd_to_run.append(piece)
stdin_templ = cmdinfo.get('stdin')
stdin = None
if (stdin_templ and len(stdin_templ)):
stdin_full = list()
for piece in stdin_templ:
if (params_replacements and len(params_replacements)):
stdin_full.append(
param_replace(piece,
params_replacements,
ignore_missing=ignore_missing))
else:
stdin_full.append(piece)
stdin = joinlinesep(*stdin_full)
root_run = cmdinfo.get('run_as_root', False)
execute(*cmd_to_run,
run_as_root=root_run,
process_input=stdin,
**kargs)
def _gitdownload(storewhere, uri, branch=None):
dirsmade = mkdirslist(storewhere)
LOG.info("Downloading from %s to %s" % (uri, storewhere))
cmd = ["git", "clone"] + [uri, storewhere]
execute(*cmd)
if(branch and branch != MASTER_BRANCH):
LOG.info("Adjusting git branch to %s" % (branch))
cmd = ['git', 'checkout'] + [branch]
execute(*cmd, cwd=storewhere)
return dirsmade
def _gitdownload(storewhere, uri, branch=None):
dirsmade = mkdirslist(storewhere)
LOG.info("Downloading from %s to %s" % (uri, storewhere))
cmd = ["git", "clone"] + [uri, storewhere]
execute(*cmd)
if (branch and branch != MASTER_BRANCH):
LOG.info("Adjusting git branch to %s" % (branch))
cmd = ['git', 'checkout'] + [branch]
execute(*cmd, cwd=storewhere)
return dirsmade
def install_batch(self, pkgs):
pkgnames = sorted(pkgs.keys())
#form the needed commands
cmds = []
for name in pkgnames:
info = pkgs.get(name) or {}
if (_pkg_install_special(name, info)):
#handled by the special install
continue
torun = self._form_cmd(name, info.get("version"))
cmds.append(torun)
#install them
if (len(cmds)):
cmd = APT_GET + APT_INSTALL + cmds
execute(*cmd, run_as_root=True, env_overrides=ENV_ADDITIONS)
def _do_cmd(self, base_cmd, pkgs):
pkgnames = pkgs.keys()
pkgnames.sort()
cmds = []
for name in pkgnames:
version = None
info = pkgs.get(name)
if (info):
version = info.get("version")
torun = self._form_cmd(name, version)
cmds.append(torun)
if (len(cmds)):
cmd = YUM_CMD + base_cmd + cmds
LOG.debug("About to run:%s" % (cmd))
execute(*cmd, run_as_root=True)
def _do_cmd(self, base_cmd, pkgs):
pkgnames = pkgs.keys()
pkgnames.sort()
cmds = []
for name in pkgnames:
version = None
info = pkgs.get(name)
if(info):
version = info.get("version")
torun = self._form_cmd(name, version)
cmds.append(torun)
if(len(cmds)):
cmd = YUM_CMD + base_cmd + cmds
LOG.debug("About to run:%s" % (cmd))
execute(*cmd, run_as_root=True)
def install_batch(self, pkgs):
pkgnames = sorted(pkgs.keys())
#form the needed commands
cmds = []
for name in pkgnames:
info = pkgs.get(name) or {}
if(_pkg_install_special(name, info)):
#handled by the special install
continue
torun = self._form_cmd(name, info.get("version"))
cmds.append(torun)
#install them
if(len(cmds)):
cmd = APT_GET + APT_INSTALL + cmds
execute(*cmd, run_as_root=True,
env_overrides=ENV_ADDITIONS)
def status(self):
pkgsinstalled = self.tracereader.packages_installed()
if(len(pkgsinstalled) == 0):
msg = "Can not check the status of %s since it was not installed" % (TYPE)
raise StatusException(msg)
(sysout, stderr) = execute(*STATUS_CMD, run_as_root=True)
return sysout.strip().lower()
def uninstall(pips):
if(not pips or len(pips) == 0):
return
actions = list()
pipnames = sorted(pips.keys())
LOG.info("Uninstalling python packages [%s]" % (", ".join(pipnames)))
for name in pipnames:
pipinfo = pips.get(name, dict())
skip_errors = pipinfo.get('skip_uninstall_errors', False)
try:
cmd = UNINSTALL_CMD + [name]
execute(*cmd, run_as_root=True)
except ProcessExecutionError, e:
if(skip_errors):
pass
else:
raise
def install(pips):
if(not pips or len(pips) == 0):
return
actions = list()
pipnames = sorted(pips.keys())
for name in pipnames:
pipfull = name
pipinfo = pips.get(name)
if(pipinfo and pipinfo.get('version')):
version = str(pipinfo.get('version'))
if(len(version)):
pipfull = pipfull + "==" + version
actions.append(pipfull)
if(len(actions)):
LOG.info("Installing python packages [%s]" % (", ".join(actions)))
cmd = INSTALL_CMD + actions
execute(*cmd, run_as_root=True)
def install(pips):
if (not pips or len(pips) == 0):
return
actions = list()
pipnames = sorted(pips.keys())
for name in pipnames:
pipfull = name
pipinfo = pips.get(name)
if (pipinfo and pipinfo.get('version')):
version = str(pipinfo.get('version'))
if (len(version)):
pipfull = pipfull + "==" + version
actions.append(pipfull)
if (len(actions)):
LOG.info("Installing python packages [%s]" % (", ".join(actions)))
cmd = INSTALL_CMD + actions
execute(*cmd, run_as_root=True)
def uninstall(pips):
if (not pips or len(pips) == 0):
return
actions = list()
pipnames = sorted(pips.keys())
LOG.info("Uninstalling python packages [%s]" % (", ".join(pipnames)))
for name in pipnames:
pipinfo = pips.get(name, dict())
skip_errors = pipinfo.get('skip_uninstall_errors', False)
try:
cmd = UNINSTALL_CMD + [name]
execute(*cmd, run_as_root=True)
except ProcessExecutionError, e:
if (skip_errors):
pass
else:
raise
def status(self):
pkgsinstalled = self.tracereader.packages_installed()
if (len(pkgsinstalled) == 0):
msg = "Can not check the status of %s since it was not installed" % (
TYPE)
raise StatusException(msg)
(sysout, stderr) = execute(*STATUS_CMD, run_as_root=True)
return sysout.strip().lower()
def _pkg_install_special(name, pkginfo):
#https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878597
#https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878600
if(name == 'rabbitmq-server'):
version = pkginfo.get('version')
version_info = _extract_version(version)
if(len(version_info)):
if(version_info.get('type') == 'ubuntu' and
version_info.get('major') <= 2 and
version_info.get('minor') < 6):
LOG.info("Handling special install of %s v%s" % (name, version))
#this seems to be a temporary fix for that bug
with TemporaryFile() as f:
cmd = APT_GET + APT_INSTALL + [name + "=" + version]
execute(*cmd, run_as_root=True,
stdout_fh=f, stderr_fh=f,
env_overrides=ENV_ADDITIONS)
return True
return False
def remove_batch(self, pkgs):
pkgnames = sorted(pkgs.keys())
#form the needed commands
cmds = []
for name in pkgnames:
info = pkgs.get(name) or {}
removable = info.get('removable', True)
if (not removable):
continue
if (_pkg_remove_special(name, info)):
#handled by the special remove
continue
torun = self._form_cmd(name, info.get("version"))
cmds.append(torun)
if (len(cmds)):
cmd = APT_GET + APT_DO_REMOVE + cmds
execute(*cmd, run_as_root=True, env_overrides=ENV_ADDITIONS)
#clean them out
cmd = APT_GET + APT_AUTOREMOVE
execute(*cmd, run_as_root=True, env_overrides=ENV_ADDITIONS)
def stop(self, name, *args, **kargs):
real_name = name + NAME_POSTFIX
list_cmd = ['screen', '-list']
(sysout, stderr) = execute(*list_cmd)
lines = sysout.splitlines()
entries = list()
lookfor = r"^(\d+\." + re.escape(real_name) + r")\s+(.*)$"
for line in lines:
cleaned_line = line.strip()
if (len(cleaned_line) == 0):
continue
mtch = re.match(lookfor, cleaned_line)
if (not mtch):
continue
kill_entry = mtch.group(1)
entries.append(kill_entry)
for entry in entries:
kill_cmd = ['screen', '-r', entry, '-X', 'kill']
execute(*kill_cmd)
time.sleep(2)
quit_cmd = ['screen', '-r', entry, '-X', 'quit']
execute(*quit_cmd)
def _pkg_install_special(name, pkginfo):
#https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878597
#https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878600
if (name == 'rabbitmq-server'):
version = pkginfo.get('version')
version_info = _extract_version(version)
if (len(version_info)):
if (version_info.get('type') == 'ubuntu'
and version_info.get('major') <= 2
and version_info.get('minor') < 6):
LOG.info("Handling special install of %s v%s" %
(name, version))
#this seems to be a temporary fix for that bug
with TemporaryFile() as f:
cmd = APT_GET + APT_INSTALL + [name + "=" + version]
execute(*cmd,
run_as_root=True,
stdout_fh=f,
stderr_fh=f,
env_overrides=ENV_ADDITIONS)
return True
return False
def remove_batch(self, pkgs):
pkgnames = sorted(pkgs.keys())
#form the needed commands
cmds = []
for name in pkgnames:
info = pkgs.get(name) or {}
removable = info.get('removable', True)
if(not removable):
continue
if(_pkg_remove_special(name, info)):
#handled by the special remove
continue
torun = self._form_cmd(name, info.get("version"))
cmds.append(torun)
if(len(cmds)):
cmd = APT_GET + APT_DO_REMOVE + cmds
execute(*cmd, run_as_root=True,
env_overrides=ENV_ADDITIONS)
#clean them out
cmd = APT_GET + APT_AUTOREMOVE
execute(*cmd, run_as_root=True,
env_overrides=ENV_ADDITIONS)
def stop(self, name, *args, **kargs):
real_name = name + NAME_POSTFIX
list_cmd = ['screen', '-list']
(sysout, stderr) = execute(*list_cmd)
lines = sysout.splitlines()
entries = list()
lookfor = r"^(\d+\." + re.escape(real_name) + r")\s+(.*)$"
for line in lines:
cleaned_line = line.strip()
if(len(cleaned_line) == 0):
continue
mtch = re.match(lookfor, cleaned_line)
if(not mtch):
continue
kill_entry = mtch.group(1)
entries.append(kill_entry)
for entry in entries:
kill_cmd = ['screen', '-r', entry, '-X', 'kill']
execute(*kill_cmd)
time.sleep(2)
quit_cmd = ['screen', '-r', entry, '-X', 'quit']
execute(*quit_cmd)
def _python_install(self):
pips = get_pip_list(self.distro, self.component_name)
#install any need pip items
if(len(pips)):
Pip.install(pips)
for name in pips.keys():
self.tracewriter.pip_install(name, pips.get(name))
#do the actual python install
dirsmade = mkdirslist(self.tracedir)
self.tracewriter.dir_made(*dirsmade)
recordwhere = touch_trace(self.tracedir, PY_TRACE)
self.tracewriter.py_install(recordwhere)
(sysout, stderr) = execute(*PY_INSTALL, cwd=self.appdir, run_as_root=True)
write_file(recordwhere, sysout)
def _pkg_remove_special(name, pkginfo):
#https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878597
#https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878600
if(name == 'rabbitmq-server'):
version = pkginfo.get('version')
version_info = _extract_version(version)
if(len(version_info)):
if(version_info.get('type') == 'ubuntu' and
version_info.get('major') <= 2 and
version_info.get('minor') < 6):
LOG.info("Handling special remove of %s v%s" % (name, version))
#the first time seems to fail with exit code 100 but the second
#time seems to not fail, pretty weird, most likely the above bugs
cmd = APT_GET + APT_REMOVE + [name + "=" + version]
execute(*cmd, run_as_root=True,
check_exit_code=False, env_overrides=ENV_ADDITIONS)
#probably useful to do this
time.sleep(1)
cmd = APT_GET + APT_PURGE + [name + "=" + version]
execute(*cmd, run_as_root=True,
env_overrides=ENV_ADDITIONS)
return True
return False
def _pkg_remove_special(name, pkginfo):
#https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878597
#https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878600
if (name == 'rabbitmq-server'):
version = pkginfo.get('version')
version_info = _extract_version(version)
if (len(version_info)):
if (version_info.get('type') == 'ubuntu'
and version_info.get('major') <= 2
and version_info.get('minor') < 6):
LOG.info("Handling special remove of %s v%s" % (name, version))
#the first time seems to fail with exit code 100 but the second
#time seems to not fail, pretty weird, most likely the above bugs
cmd = APT_GET + APT_REMOVE + [name + "=" + version]
execute(*cmd,
run_as_root=True,
check_exit_code=False,
env_overrides=ENV_ADDITIONS)
#probably useful to do this
time.sleep(1)
cmd = APT_GET + APT_PURGE + [name + "=" + version]
execute(*cmd, run_as_root=True, env_overrides=ENV_ADDITIONS)
return True
return False
def _python_install(self):
pips = get_pip_list(self.distro, self.component_name)
#install any need pip items
if (len(pips)):
Pip.install(pips)
for name in pips.keys():
self.tracewriter.pip_install(name, pips.get(name))
#do the actual python install
dirsmade = mkdirslist(self.tracedir)
self.tracewriter.dir_made(*dirsmade)
recordwhere = touch_trace(self.tracedir, PY_TRACE)
self.tracewriter.py_install(recordwhere)
(sysout, stderr) = execute(*PY_INSTALL,
cwd=self.appdir,
run_as_root=True)
write_file(recordwhere, sysout)
def stop(self):
if (self.status().find('stop') == -1):
stopcmd = self._gettypeactions('stop', StopException)
execute(*stopcmd, run_as_root=True)
return None
def _run_cmd(self, cmd):
if (self.distro == UBUNTU11):
with TemporaryFile() as f:
execute(*cmd, run_as_root=True, stdout_fh=f, stderr_fh=f)
else:
execute(*cmd, run_as_root=True)
def _uninstall_python(self):
pylisting = self.tracereader.py_listing()
if (pylisting and len(pylisting)):
execute(*PY_UNINSTALL, cwd=self.appdir, run_as_root=True)
def status(self):
statuscmd = self._gettypeactions('status', StatusException)
(sysout, stderr) = execute(*statuscmd, run_as_root=True)
return sysout.strip()
def restart(self):
restartcmd = self._gettypeactions('restart', RestartException)
execute(*restartcmd, run_as_root=True)
return None
def stop(self):
if(self.status().find('stop') == -1):
stopcmd = self._gettypeactions('stop', StopException)
execute(*stopcmd, run_as_root=True)
return None
def _uninstall_python(self):
pylisting = self.tracereader.py_listing()
if(pylisting and len(pylisting)):
execute(*PY_UNINSTALL, cwd=self.appdir, run_as_root=True)
def status(self):
statuscmd = self._gettypeactions('status', StatusException)
(sysout, stderr) = execute(*statuscmd, run_as_root=True)
return sysout.strip()
def _setup_pw(self):
passwd = self.cfg.getpw("passwords", "rabbit")
cmd = PWD_CMD + [passwd]
execute(*cmd, run_as_root=True)
def restart(self):
restartcmd = self._gettypeactions('restart', RestartException)
execute(*restartcmd, run_as_root=True)
return None
def _setup_pw(self):
passwd = self.cfg.getpw("passwords", "rabbit")
cmd = PWD_CMD + [passwd]
execute(*cmd, run_as_root=True)
