def get_installed_plugins(plugins_base_dir=None):
_assert_plugins_base_initialized(plugins_base_dir)
plugins_base_dir = _get_plugins_base_dir(plugins_base_dir)
frmt = "%{name}~~~%{version}~~~%{release}\\n"
cmd = _get_rpm_cmd('-qa',
'--qf "%s"' % frmt,
plugins_base_dir=plugins_base_dir)
x = BashWrapperOrRaise(cmd)
tmp = x.stdout.split('\n')
result = []
for line in tmp:
tmp2 = line.split('~~~')
if len(tmp2) == 3:
result.append({
'name': tmp2[0],
'version': tmp2[1],
'release': tmp2[2]
})
for tmp in os.listdir(plugins_base_dir):
directory_name = tmp.strip()
if directory_name == 'base':
continue
directory = os.path.join(plugins_base_dir, directory_name)
if os.path.islink(directory):
result.append({
'name': directory_name,
'version': 'dev_link',
'release': 'dev_link'
})
return result
def make_new_circus_conf():
new_circus_conf = "%s/tmp/tmp_circus_conf2" % MFMODULE_RUNTIME_HOME
cmd = "_make_circus_conf >%s" % new_circus_conf
BashWrapperOrRaise(cmd)
return (new_circus_conf,
md5sumfile(new_circus_conf,
ignore_lines_starting_with=["autostart"]))
def uninstall_plugin(name, plugins_base_dir=None):
_assert_plugins_base_initialized(plugins_base_dir)
plugins_base_dir = _get_plugins_base_dir(plugins_base_dir)
infos = get_plugin_info(name,
mode="name",
plugins_base_dir=plugins_base_dir)
if infos is None:
raise MFUtilPluginNotInstalled("plugin %s is not installed" % name)
version = infos['metadatas']['version']
release = infos['metadatas']['release']
if release == 'dev_link':
_postuninstall_plugin(name, version, release)
os.unlink(os.path.join(plugins_base_dir, name))
return
cmd = _get_rpm_cmd('-e --noscripts %s' % name,
plugins_base_dir=plugins_base_dir,
add_prefix=False)
_postuninstall_plugin(name, version, release)
x = BashWrapperOrRaise(cmd, MFUtilPluginCantUninstall,
"can't uninstall %s" % name)
infos = get_plugin_info(name,
mode="name",
plugins_base_dir=plugins_base_dir)
if infos is not None:
raise MFUtilPluginCantUninstall("can't uninstall plugin %s" % name,
bash_wrapper=x)
def install_plugin(plugin_filepath, plugins_base_dir=None):
_assert_plugins_base_initialized(plugins_base_dir)
if not os.path.isfile(plugin_filepath):
raise MFUtilPluginFileNotFound("plugin file %s not found" %
plugin_filepath)
infos = get_plugin_info(plugin_filepath,
mode="file",
plugins_base_dir=plugins_base_dir)
if infos is None:
raise MFUtilPluginInvalid("invalid %s plugin" % plugin_filepath)
name = infos['metadatas']['name']
version = infos['metadatas']['version']
release = infos['metadatas']['release']
installed_infos = get_plugin_info(name,
mode="name",
plugins_base_dir=plugins_base_dir)
if installed_infos is not None:
raise MFUtilPluginAlreadyInstalled("plugin %s already installed" %
name)
cmd = _get_rpm_cmd('-Uvh --noscripts --force %s' % plugin_filepath,
plugins_base_dir=plugins_base_dir,
add_prefix=True)
x = BashWrapperOrRaise(cmd, MFUtilPluginCantInstall,
"can't install plugin %s" % name)
infos = get_plugin_info(name,
mode="name",
plugins_base_dir=plugins_base_dir)
if infos is None:
raise MFUtilPluginCantInstall("can't install plugin %s" % name,
bash_wrapper=x)
_postinstall_plugin(name, version, release)
def make_new_directory_observer_conf():
new_directory_observer_conf = \
"%s/tmp/tmp_directory_observer_conf2" % MODULE_RUNTIME_HOME
cmd = "_make_directory_observer_conf >%s" % new_directory_observer_conf
BashWrapperOrRaise(cmd)
return (new_directory_observer_conf,
md5sumfile(new_directory_observer_conf))
def _preuninstall_plugin(self, plugin):
if shutil.which("_plugins.preuninstall"):
env_context = {"MFMODULE_PLUGINS_BASE_DIR": self.plugins_base_dir}
# FIXME: should be python methods and not shell
with PluginEnvContextManager(env_context):
x = BashWrapperOrRaise(
"_plugins.preuninstall %s %s %s" %
(plugin.name, plugin.version, plugin.release))
if len(x.stderr) != 0:
print(x.stderr, file=sys.stderr)
def build_plugin(plugin_path, plugins_base_dir=None):
plugin_path = os.path.abspath(plugin_path)
plugins_base_dir = _get_plugins_base_dir(plugins_base_dir)
base = os.path.join(plugins_base_dir, "base")
pwd = os.getcwd()
parser = ExtendedConfigParser(config=os.environ.get('MFCONFIG', 'GENERIC'),
strict=False,
inheritance="im")
with open(os.path.join(plugin_path, "config.ini"), "r") as f:
config_content = f.read()
if six.PY2:
parser.read_string(config_content.decode('utf-8'))
else:
parser.read_string(config_content)
name = parser['general']['name']
version = parser['general']['version']
summary = parser['general']['summary']
license = parser['general']['license']
try:
packager = parser['general']['packager']
except Exception:
packager = parser['general']['maintainer']
vendor = parser['general']['vendor']
url = parser['general']['url']
tmpdir = os.path.join(RUNTIME_HOME, "tmp",
"plugin_%s" % get_unique_hexa_identifier())
mkdir_p_or_die(os.path.join(tmpdir, "BUILD"))
mkdir_p_or_die(os.path.join(tmpdir, "RPMS"))
mkdir_p_or_die(os.path.join(tmpdir, "SRPMS"))
_make_plugin_spec(os.path.join(tmpdir, "specfile.spec"), name, version,
summary, license, packager, vendor, url)
cmd = "source %s/lib/bash_utils.sh ; " % MFEXT_HOME
cmd = cmd + "layer_load rpm@mfext ; "
cmd = cmd + 'rpmbuild --define "_topdir %s" --define "pwd %s" ' \
'--define "prefix %s" --dbpath %s ' \
'-bb %s/specfile.spec' % (tmpdir, plugin_path, tmpdir,
base, tmpdir)
x = BashWrapperOrRaise(cmd, MFUtilPluginCantBuild,
"can't build plugin %s" % plugin_path)
tmp = glob.glob(os.path.join(tmpdir, "RPMS", "x86_64", "*.rpm"))
if len(tmp) == 0:
raise MFUtilPluginCantBuild("can't find generated plugin" %
plugin_path,
bash_wrapper=x)
plugin_path = tmp[0]
new_basename = \
os.path.basename(plugin_path).replace("x86_64.rpm",
"metwork.%s.plugin" %
MODULE_LOWERCASE)
new_plugin_path = os.path.join(pwd, new_basename)
shutil.move(plugin_path, new_plugin_path)
shutil.rmtree(tmpdir, True)
os.chdir(pwd)
return new_plugin_path
def get_installed_plugins(plugins_base_dir=None):
"""Get a detailed list (formatted text) of installed plugins.
Args:
plugins_base_dir (string): (optional) the plugin base directory path.
If not set, the default plugins base directory path is used.
Returns:
(string): detailed list (formatted text) of installed plugins.
Raises:
MFUtilPluginBaseNotInitialized: if the plugins base is not initialized.
"""
_assert_plugins_base_initialized(plugins_base_dir)
plugins_base_dir = _get_plugins_base_dir(plugins_base_dir)
frmt = "%{name}~~~%{version}~~~%{release}\\n"
cmd = _get_rpm_cmd('-qa', '--qf "%s"' % frmt,
plugins_base_dir=plugins_base_dir)
x = BashWrapperOrRaise(cmd)
tmp = x.stdout.split('\n')
result = []
for line in tmp:
tmp2 = line.split('~~~')
if len(tmp2) == 3:
home = get_layer_home_from_plugin_name(
tmp2[0], plugins_base_dir=plugins_base_dir)
if home:
result.append({'name': tmp2[0],
'version': tmp2[1],
'release': tmp2[2],
'home': home})
else:
result.append({'name': tmp2[0],
'version': 'ERROR',
'release': 'ERROR',
'home': 'ERROR'})
for tmp in os.listdir(plugins_base_dir):
directory_name = tmp.strip()
if directory_name == 'base':
continue
llf = os.path.join(plugins_base_dir, directory_name,
".layerapi2_label")
if not os.path.isfile(llf):
__get_logger().warning("missing %s file for installed "
"plugin directory" % llf)
continue
name = layerapi2_label_file_to_plugin_name(llf)
directory = os.path.join(plugins_base_dir, directory_name)
if os.path.islink(directory):
result.append({'name': name,
'version': 'dev_link',
'release': 'dev_link',
'home': directory})
return result
def uninstall_plugin(name,
plugins_base_dir=None,
ignore_errors=False,
quiet=False):
_assert_plugins_base_initialized(plugins_base_dir)
plugins_base_dir = _get_plugins_base_dir(plugins_base_dir)
infos = get_plugin_info(name,
mode="name",
plugins_base_dir=plugins_base_dir)
if infos is None:
raise MFUtilPluginNotInstalled("plugin %s is not installed" % name)
version = infos['metadatas']['version']
release = infos['metadatas']['release']
home = infos.get('home', None)
if release == 'dev_link':
preuninstall_status = _preuninstall_plugin(name,
version,
release,
quiet=quiet)
if not preuninstall_status and not ignore_errors:
raise MFUtilPluginCantUninstall("can't uninstall plugin %s" % name)
if home:
os.unlink(home)
return
preuninstall_status = _preuninstall_plugin(name, version, release)
if not preuninstall_status and not ignore_errors:
raise MFUtilPluginCantUninstall("can't uninstall plugin %s" % name)
cmd = _get_rpm_cmd('-e --noscripts %s' % name,
plugins_base_dir=plugins_base_dir,
add_prefix=False)
try:
x = BashWrapperOrRaise(cmd, MFUtilPluginCantUninstall,
"can't uninstall %s" % name)
except MFUtilPluginCantUninstall:
if not ignore_errors:
raise
if home:
shutil.rmtree(home, ignore_errors=True)
infos = get_plugin_info(name,
mode="name",
plugins_base_dir=plugins_base_dir)
if infos is not None:
raise MFUtilPluginCantUninstall("can't uninstall plugin %s" % name,
bash_wrapper=x)
if home and os.path.exists(home):
raise MFUtilPluginCantUninstall("can't uninstall plugin %s "
"(directory still here)" % name)
def init_plugins_base(plugins_base_dir=None):
"""Initialize the plugins base.
Args:
plugins_base_dir (string): alternate plugins base directory
(useful for unit tests).
Raises:
MFUtilPluginCantInit: if we can't init the plugin base.
"""
plugins_base_dir = _get_plugins_base_dir(plugins_base_dir)
shutil.rmtree(plugins_base_dir, ignore_errors=True)
mkdir_p_or_die(plugins_base_dir)
mkdir_p_or_die(os.path.join(plugins_base_dir, "base"))
cmd = _get_rpm_cmd("--initdb", plugins_base_dir=plugins_base_dir)
BashWrapperOrRaise(cmd, MFUtilPluginCantInit,
"can't init %s" % plugins_base_dir)
def get_installed_plugins(plugins_base_dir=None):
_assert_plugins_base_initialized(plugins_base_dir)
plugins_base_dir = _get_plugins_base_dir(plugins_base_dir)
frmt = "%{name}~~~%{version}~~~%{release}\\n"
cmd = _get_rpm_cmd('-qa',
'--qf "%s"' % frmt,
plugins_base_dir=plugins_base_dir)
x = BashWrapperOrRaise(cmd)
tmp = x.stdout.split('\n')
result = []
for line in tmp:
tmp2 = line.split('~~~')
if len(tmp2) == 3:
home = get_layer_home_from_plugin_name(tmp2[0])
if home:
result.append({
'name': tmp2[0],
'version': tmp2[1],
'release': tmp2[2],
'home': home
})
for tmp in os.listdir(plugins_base_dir):
directory_name = tmp.strip()
if directory_name == 'base':
continue
llf = os.path.join(plugins_base_dir, directory_name,
".layerapi2_label")
if not os.path.isfile(llf):
__get_logger().warning("missing %s file for installed "
"plugin directory" % llf)
continue
name = layerapi2_label_file_to_plugin_name(llf)
directory = os.path.join(plugins_base_dir, directory_name)
if os.path.islink(directory):
result.append({
'name': name,
'version': 'dev_link',
'release': 'dev_link',
'home': directory
})
return result
#!/usr/bin/env python3
import datetime
import sys
import requests
import os
import time
from mfutil import BashWrapperOrRaise
NGINX_PORT = int(os.environ['MFSERV_NGINX_PORT'])
BashWrapperOrRaise("rm -Rf foobar")
BashWrapperOrRaise("plugins.uninstall foobar || true")
BashWrapperOrRaise("rm -Rf foobar2")
BashWrapperOrRaise("plugins.uninstall foobar2 || true")
print(
BashWrapperOrRaise("bootstrap_plugin.py create --template=static "
"--no-input foobar2"))
print(BashWrapperOrRaise("cd foobar2 && make release"))
print(BashWrapperOrRaise('cd foobar2 && plugins.install "$(ls *.plugin)"'))
print(
BashWrapperOrRaise("bootstrap_plugin.py create --template=mediation "
"--no-input foobar"))
with open("foobar/main/application.py", "r") as f:
c = f.read()
c2 = c.replace(
'mybackend%s" % url_path_qs',
'127.0.0.1:%i/foobar2/index.html"' % int(os.environ['MFSERV_NGINX_PORT']))
with open("foobar/main/application.py", "w") as f:
f.write(c2)
#!/usr/bin/env python3
import datetime
import sys
import requests
import os
import time
from mfutil import BashWrapperOrRaise
NGINX_PORT = int(os.environ['MFSERV_NGINX_PORT'])
BashWrapperOrRaise("rm -Rf foobar")
BashWrapperOrRaise("plugins.uninstall foobar || true")
print(
BashWrapperOrRaise("bootstrap_plugin.py create --no-input "
"--template=python3_raw_asgi foobar"))
print(BashWrapperOrRaise("cd foobar && make release"))
print(BashWrapperOrRaise('cd foobar && plugins.install "$(ls *.plugin)"'))
now_fn = datetime.datetime.now
before = now_fn()
code = 1
while (now_fn() - before).total_seconds() <= 30:
time.sleep(1)
url = "http://127.0.0.1:%i/foobar" % NGINX_PORT
print("trying GET %s..." % url)
try:
x = requests.get(url, timeout=3)
except Exception:
continue
parser.exit(1)
extra_context = {
"name": args.plugin,
"MFMODULE_VERSION": os.environ["MFMODULE_VERSION"]
}
res = cookiecutter(template_path,
extra_context=extra_context,
no_input=args.no_input,
search_paths=[PLUGIN_TEMPLATES_PATH])
if not os.path.isdir(res):
print("ERROR : cookiecutter result is not a valid directory")
parser.exit(1)
BashWrapperOrRaise("cd %s && remove_empty.sh" % args.plugin)
BashWrapperOrRaise("cd %s && bootstrap_plugin.post" % args.plugin)
if args.make:
print("Make plugin on directory %s" % args.plugin)
os.chdir("%s" % args.plugin)
b = bash("make release")
print("%s" % b.stdout)
if b.code == 0:
if args.install:
for fic in glob.glob("*.plugin"):
print("Installing plugin %s" % fic)
b = bash("plugins.install %s" % fic)
print("%s" % b.stdout)
def make_new_switch_confs():
cmd = "_make_and_write_switch_confs"
BashWrapperOrRaise(cmd)
#!/usr/bin/env python3
import datetime
import sys
import requests
import os
import time
from mfutil import BashWrapperOrRaise
NGINX_PORT = int(os.environ['MFSERV_NGINX_PORT'])
#python2@mfext may not be installed
bash_wrapper = BashWrapperOrRaise("is_layer_installed python2@mfext")
if bash_wrapper.stdout != "1":
VERSIONS = [3]
else:
VERSIONS = [3, 2]
for VERSION in VERSIONS:
BashWrapperOrRaise("rm -Rf foobar")
BashWrapperOrRaise("plugins.uninstall foobar || true")
print(
BashWrapperOrRaise("bootstrap_plugin.py create --no-input "
"--template=python%i_raw_wsgi "
"foobar" % VERSION))
with open(
"foobar/python%i_virtualenv_sources/"
"requirements-to-freeze.txt" % VERSION, "w") as f:
f.write("falcon\n")
BashWrapperOrRaise('echo "import falcon" >foobar/main/wsgi2.py')
def test_bash_wrapper3(self):
try:
BashWrapperOrRaise("ls /foo/bar")
raise Exception("this exception must not be raised")
except BashWrapperException as e:
self.assertTrue(len("%s" % e) > 0)
#!/usr/bin/env python3
import datetime
import sys
import requests
import os
import time
from mfutil import BashWrapperOrRaise
NGINX_PORT = int(os.environ['MFSERV_NGINX_PORT'])
BashWrapperOrRaise("rm -Rf foobar")
BashWrapperOrRaise("plugins.uninstall foobar || true")
print(
BashWrapperOrRaise("bootstrap_plugin.py create --template=default "
"foobar <stdin"))
print(BashWrapperOrRaise("cd foobar && make release"))
print(BashWrapperOrRaise('cd foobar && plugins.install "$(ls *.plugin)"'))
now_fn = datetime.datetime.now
before = now_fn()
code = 1
while (now_fn() - before).total_seconds() <= 30:
time.sleep(1)
url = "http://127.0.0.1:%i/foobar" % NGINX_PORT
print("trying GET %s..." % url)
try:
x = requests.get(url, timeout=3)
except Exception:
continue
def make_new_nginx_conf():
new_nginx_conf = "%s/tmp/tmp_nginx_conf2" % MFMODULE_RUNTIME_HOME
cmd = "_make_nginx_conf >%s" % new_nginx_conf
BashWrapperOrRaise(cmd)
return (new_nginx_conf, md5sumfile(new_nginx_conf))
#!/usr/bin/env python3
import datetime
import sys
import os
import time
from mfutil import BashWrapperOrRaise, get_unique_hexa_identifier
NGINX_PORT = int(os.environ['MFSERV_NGINX_PORT'])
UNIQUE = get_unique_hexa_identifier()
BashWrapperOrRaise("rm -Rf foobar")
BashWrapperOrRaise("plugins.uninstall foobar || true")
print(
BashWrapperOrRaise("bootstrap_plugin.py create --template=python3_noweb "
"--no-input foobar"))
with open("foobar/config.ini", "a") as f:
f.write("\n\n[extra_daemon_foo]\n")
f.write("_cmd_and_args = foo.sh %s\n" % UNIQUE)
f.write("numprocesses=1\n")
BashWrapperOrRaise("mkdir -p foobar/bin")
BashWrapperOrRaise("cp -f foo.sh foobar/bin/")
BashWrapperOrRaise("chmod +x foobar/bin/foo.sh")
print(BashWrapperOrRaise("cd foobar && make develop"))
now_fn = datetime.datetime.now
before = now_fn()
code = 1
while (now_fn() - before).total_seconds() <= 30:
#!/usr/bin/env python3
import os
from mfutil import BashWrapperOrRaise
NGINX_PORT = int(os.environ['MFSERV_NGINX_PORT'])
#python2@mfext may not be installed
bash_wrapper = BashWrapperOrRaise("is_layer_installed python2@mfext")
if bash_wrapper.stdout != "1":
templates = ("python3_noweb", "node_noweb")
else:
templates = ("python3_noweb", "python2_noweb", "node_noweb")
for template in templates:
BashWrapperOrRaise("rm -Rf foobar")
BashWrapperOrRaise("plugins.uninstall foobar || true")
print(
BashWrapperOrRaise("bootstrap_plugin.py create --template=%s "
"--no-input foobar" % template))
print(BashWrapperOrRaise("cd foobar && make release"))
print(BashWrapperOrRaise('cd foobar && plugins.install "$(ls *.plugin)"'))
print("ok")
def build_plugin(plugin_path, plugins_base_dir=None):
"""Build a plugin.
Args:
plugin_path (string): the plugin path to build
plugins_base_dir (string): (optional) the plugin base directory path.
If not set, the default plugins base directory path is used.
Raises:
MFUtilPluginCantBuild: if a error occurs during build
"""
plugin_path = os.path.abspath(plugin_path)
plugins_base_dir = _get_plugins_base_dir(plugins_base_dir)
base = os.path.join(plugins_base_dir, "base")
pwd = os.getcwd()
parser = OpinionatedConfigParser()
with open(os.path.join(plugin_path, "config.ini"), "r") as f:
config_content = f.read()
if six.PY2:
parser.read_string(config_content.decode('utf-8'))
else:
parser.read_string(config_content)
with open(os.path.join(plugin_path, ".layerapi2_label"), "r") as f:
name = f.read().replace('plugin_', '', 1).split('@')[0]
version = parser['general']['version']
summary = parser['general']['summary']
license = parser['general']['license']
try:
packager = parser['general']['packager']
except Exception:
packager = parser['general']['maintainer']
vendor = parser['general']['vendor']
url = parser['general']['url']
tmpdir = os.path.join(RUNTIME_HOME, "tmp",
"plugin_%s" % get_unique_hexa_identifier())
mkdir_p_or_die(os.path.join(tmpdir, "BUILD"))
mkdir_p_or_die(os.path.join(tmpdir, "RPMS"))
mkdir_p_or_die(os.path.join(tmpdir, "SRPMS"))
_make_plugin_spec(os.path.join(tmpdir, "specfile.spec"), name, version,
summary, license, packager, vendor, url)
cmd = "source %s/lib/bash_utils.sh ; " % MFEXT_HOME
cmd = cmd + "layer_load rpm@mfext ; "
cmd = cmd + 'rpmbuild --define "_topdir %s" --define "pwd %s" ' \
'--define "prefix %s" --dbpath %s ' \
'-bb %s/specfile.spec' % (tmpdir, plugin_path, tmpdir,
base, tmpdir)
x = BashWrapperOrRaise(cmd, MFUtilPluginCantBuild,
"can't build plugin %s" % plugin_path)
tmp = glob.glob(os.path.join(tmpdir, "RPMS", "x86_64", "*.rpm"))
if len(tmp) == 0:
raise MFUtilPluginCantBuild("can't find generated plugin" %
plugin_path, bash_wrapper=x)
plugin_path = tmp[0]
new_basename = \
os.path.basename(plugin_path).replace("x86_64.rpm",
"metwork.%s.plugin" %
MFMODULE_LOWERCASE)
new_plugin_path = os.path.join(pwd, new_basename)
shutil.move(plugin_path, new_plugin_path)
shutil.rmtree(tmpdir, True)
os.chdir(pwd)
return new_plugin_path
#!/usr/bin/env python3
import datetime
import sys
import requests
import os
import time
from mfutil import BashWrapperOrRaise
NGINX_PORT = int(os.environ['MFSERV_NGINX_PORT'])
VERSION = 3
BashWrapperOrRaise("rm -Rf foobar")
BashWrapperOrRaise("plugins.uninstall foobar || true")
print(
BashWrapperOrRaise("bootstrap_plugin.py create --template=default "
"foobar <stdin_python%i" % VERSION))
cmd = "cat foobar/config.ini |sed 's/^redis_service=.*$/redis_service=1/g' " \
">foobar/config.ini2"
BashWrapperOrRaise(cmd)
BashWrapperOrRaise("mv -f foobar/config.ini2 foobar/config.ini")
BashWrapperOrRaise("cp -f wsgi.py foobar/main/wsgi.py")
with open(
"foobar/python%i_virtualenv_sources/"
"requirements-to-freeze.txt" % VERSION, "w") as f:
f.write("redis\n")
print(BashWrapperOrRaise("cd foobar && make release"))
print(BashWrapperOrRaise('cd foobar && plugins.install "$(ls *.plugin)"'))
now_fn = datetime.datetime.now