class Plugin:
"""
Handle Plugins
"""
def __init__(self, **kargs):
self.path_dirs = PathDirs(**kargs)
self.manifest = join(self.path_dirs.meta_dir, "plugin_manifest.cfg")
self.p_helper = PluginHelper(**kargs)
self.d_client = docker.from_env()
self.logger = Logger(__name__)
def add(self,
repo,
tools=None,
overrides=None,
version="HEAD",
branch="master",
build=True,
user=None,
pw=None,
groups=None,
version_alias=None,
wild=None,
remove_old=True,
disable_old=True,
limit_groups=None,
core=False):
"""
Adds a plugin of tool(s)
tools is a list of tuples, where the pair is a tool name (path to
Dockerfile) and version tools are for explicitly limiting which tools
and versions (if version in tuple is '', then defaults to version)
overrides is a list of tuples, where the pair is a tool name (path to
Dockerfile) and a version overrides are for explicitly removing tools
and overriding versions of tools (if version in tuple is '', then
tool is removed, otherwise that tool is checked out at the specific
version in the tuple) if tools and overrides are left as empty lists,
then all tools in the repo are pulled down at the version and branch
specified or defaulted to version is globally set for all tools, unless
overridden in tools or overrides branch is globally set for all tools
build is a boolean of whether or not to build the tools now user is the
username for a private repo if needed pw is the password to go along
with the username for a private repo groups is globally set for all
tools version_alias is globally set for all tools and is a mapping
from a friendly version tag to the real version commit ID wild lets
you specify individual overrides for additional values in the tuple
of tools or overrides. wild is a list containing one or more
of the following: branch, build, groups, version_alias
the order of the items in the wild list will expect values to be
tacked on in the same order to the tuple for tools and overrides in
additional to the tool name and version
remove_old lets you specify whether or not to remove previously found
tools that match to ones being added currently (note does not stop
currently running instances of the older version)
disable_old lets you specify whether or not to disable previously found
tools that match to ones being added currently (note does not stop
currently running instances of the older version)
limit_groups is a list of groups to build tools for that match group
names in vent.template of each tool if exists
Examples:
- repo=fe:
(get all tools from repo 'fe' at version 'HEAD' on branch 'master')
- repo=foo, version="3d1f", branch="foo":
(get all tools from repo 'foo' at verion '3d1f' on branch 'foo')
- repo=foo, tools=[('bar', ''), ('baz', '1d32')]:
(get only 'bar' from repo 'foo' at version 'HEAD' on branch
'master' and 'baz' from repo 'foo' at version '1d32' on branch
'master', ignore all other tools in repo 'foo')
- repo=foo overrides=[('baz/bar', ''), ('.', '1c4e')], version='4fad':
(get all tools from repo 'foo' at verion '4fad' on branch 'master'
except 'baz/bar' and for tool '.' get version '1c4e')
- repo=foo tools=[('bar', '1a2d')], overrides=[('baz', 'f2a1')]:
(not a particularly useful example, but get 'bar' from 'foo' at
version '1a2d' and get 'baz' from 'foo' at version 'f2a1' on branch
'master', ignore all other tools)
"""
# initialize and store class objects
self.repo = repo.lower()
self.tools = tools
if (isinstance(self.tools, list) and len(self.tools) == 0):
self.tools = None
self.overrides = overrides
self.version = version
self.branch = branch
self.build = build
self.groups = groups
self.core = core
self.path, self.org, self.name = self.p_helper.get_path(repo,
core=core)
# TODO these need to be implemented
self.version_alias = version_alias
self.wild = wild
self.remove_old = remove_old
self.disable_old = disable_old
self.limit_groups = limit_groups
status = (True, None)
status_code, _ = self.p_helper.clone(self.repo, user=user, pw=pw)
self.p_helper.apply_path(self.repo)
status = self._build_tools(status_code)
return status
@ErrorHandler
def add_image(self,
image,
link_name,
tag=None,
registry=None,
groups=None):
"""
Add an image with a tag from a Docker registry. Defaults to the Docker
Hub if not specified. Use a Template object to write an image's
information to `plugin_manifest.cfg'
Args:
image(type): docker image
link_name(type): fill me
Kwargs:
tag(type):
registry(type):
groups(type): Group that the docker image belongs to.
Returns:
tuple(bool,str): if the function completed successfully,
(True, name of image).
If the function failed, (False, message about failure)
"""
status = (True, None)
try:
pull_name = image
org = ''
name = image
if '/' in image:
org, name = image.split('/')
else:
org = "official"
if not tag:
tag = "latest"
if not registry:
registry = "docker.io"
full_image = registry + "/" + image + ":" + tag
image = self.d_client.images.pull(full_image)
section = ':'.join([registry, org, name, '', tag])
namespace = org + '/' + name
# set template section and options for tool at version and branch
template = Template(template=self.manifest)
template.add_section(section)
template.set_option(section, "name", name)
template.set_option(section, "pull_name", pull_name)
template.set_option(section, "namespace", namespace)
template.set_option(section, "path", "")
template.set_option(section, "repo", registry + '/' + org)
template.set_option(section, "enabled", "yes")
template.set_option(section, "branch", "")
template.set_option(section, "version", tag)
template.set_option(section, "last_updated",
str(datetime.utcnow()) + " UTC")
template.set_option(section, "image_name",
image.attrs['RepoTags'][0])
template.set_option(section, "type", "registry")
template.set_option(section, "link_name", link_name)
template.set_option(section, "commit_id", "")
template.set_option(section, "built", "yes")
template.set_option(section, "image_id",
image.attrs['Id'].split(':')[1][:12])
template.set_option(section, "groups", groups)
# write out configuration to the manifest file
template.write_config()
status = (True, "Successfully added " + full_image)
except Exception as e: # pragma: no cover
self.logger.error("Couldn't add image because " + str(e))
status = (False, str(e))
return status
@ErrorHandler
def builder(self,
template,
match_path,
image_name,
section,
build=None,
branch=None,
version=None):
"""
Build tools
"""
self.logger.info("Starting: builder")
self.logger.info("install path: " + str(match_path))
self.logger.info("image name: " + str(image_name))
self.logger.info("build: " + str(build))
self.logger.info("branch: " + str(branch))
self.logger.info("version: " + str(version))
if build:
self.build = build
elif not hasattr(self, 'build'):
self.build = True
if branch:
self.branch = branch
elif not hasattr(self, 'branch'):
self.branch = 'master'
if version:
self.version = version
elif not hasattr(self, 'version'):
self.version = 'HEAD'
cwd = getcwd()
self.logger.info("current working directory: " + str(cwd))
try:
chdir(match_path)
except Exception as e: # pragma: no cover
self.logger.error("unable to change to directory: " +
str(match_path) + " because: " + str(e))
return None
template = self._build_image(template, match_path, image_name, section)
chdir(cwd)
# get untagged images
untagged = None
try:
untagged = self.d_client.images.list(filters={
"label": "vent",
"dangling": "true"
})
except Exception as e: # pragma: no cover
self.logger.error("unabled to get images to remove: " + str(e))
# remove untagged images
if untagged:
deleted_images = ""
for image in untagged:
deleted_images = '\n'.join([deleted_images, image.id])
try:
self.d_client.images.remove(image.id, force=True)
except Exception as e: # pragma: no cover
self.logger.warning("unable to remove image: " + image.id +
" because: " + str(e))
self.logger.info("removed dangling images:" + deleted_images)
self.logger.info("template of builder: " + str(template))
self.logger.info("Finished: builder")
return template
def _build_tools(self, status):
"""
Create list of tools, paths, and versions to be built and sends them to
build_manifest
Args:
status (tuple(bool, str)):
Returns:
response (tuple(bool, str)): If True, then the function performed
as expected and the str is a string
"""
response = (True, None)
# TODO implement features: wild, remove_old, disable_old, limit_groups
# check result of clone, ensure successful or that it already exists
if status:
response = self.p_helper.checkout(branch=self.branch,
version=self.version)
if response[0]:
search_groups = None
if self.core:
search_groups = 'core'
matches = []
if self.tools is None and self.overrides is None:
# get all tools
matches = self.p_helper.available_tools(
self.path, version=self.version, groups=search_groups)
elif self.tools is None:
# there's only something in overrides
# grab all the tools then apply overrides
matches = self.p_helper.available_tools(
self.path, version=self.version, groups=search_groups)
# !! TODO apply overrides to matches
elif self.overrides is None:
# there's only something in tools
# only grab the tools specified
matches = PluginHelper.tool_matches(tools=self.tools,
version=self.version)
else:
# both tools and overrides were specified
# grab only the tools specified, with the overrides applied
o_matches = PluginHelper.tool_matches(tools=self.tools,
version=self.version)
matches = o_matches
for override in self.overrides:
override_t = None
if override[0] == '.':
override_t = ('', override[1])
else:
override_t = override
for match in o_matches:
if override_t[0] == match[0]:
matches.remove(match)
matches.append(override_t)
if len(matches) > 0:
self._build_manifest(matches)
else:
response = (False, status)
return response
def _build_manifest(self, matches):
"""
Builds and writes the manifest for the tools being added
"""
# !! TODO check for pre-existing that conflict with request and
# disable and/or remove image
for match in matches:
# keep track of whether or not to write an additional manifest
# entry for multiple instances, and how many additional entries
# to write
addtl_entries = 0
# remove the .git for adding repo info to manifest
if self.repo.endswith('.git'):
self.repo = self.repo[:-4]
# remove @ in match for template setting purposes
if match[0].find('@') >= 0:
true_name = match[0].split('@')[1]
else:
true_name = match[0]
template = Template(template=self.manifest)
# TODO check for special settings here first for the specific match
self.version = match[1]
response = self.p_helper.checkout(branch=self.branch,
version=self.version)
if response[0]:
section = self.org + ":" + self.name + ":" + true_name + ":"
section += self.branch + ":" + self.version
# need to get rid of temp identifiers for tools in same repo
match_path = self.path + match[0].split('@')[0]
if not self.core:
image_name = self.org + "-" + self.name + "-"
if match[0] != '':
# if tool is in a subdir, add that to the name of the
# image
image_name += '-'.join(match[0].split('/')[1:]) + "-"
image_name += self.branch + ":" + self.version
else:
image_name = ('cyberreboot/vent-' +
match[0].split('/')[-1] + ':' + self.branch)
image_name = image_name.replace('_', '-')
# check if the section already exists
exists, options = template.section(section)
previous_commit = None
previous_commits = None
head = False
if exists:
for option in options:
# TODO check if tool name but a different version
# exists - then disable/remove if set
if option[0] == 'version' and option[1] == 'HEAD':
head = True
if option[0] == 'built' and option[1] == 'yes':
# !! TODO remove pre-existing image
pass
if option[0] == 'commit_id':
previous_commit = option[1]
if option[0] == 'previous_versions':
previous_commits = option[1]
# check if tool comes from multi directory
multi_tool = "no"
if match[0].find('@') >= 0:
multi_tool = "yes"
# !! TODO
# check if section should be removed from config i.e. all tools
# but new commit removed one that was in a previous commit
image_name = image_name.lower()
if image_name.endswith(":head"):
image_name = image_name.split(":head")[0] + ":HEAD"
# set template section & options for tool at version and branch
template.add_section(section)
template.set_option(section, "name", true_name.split('/')[-1])
template.set_option(section, "namespace",
self.org + '/' + self.name)
template.set_option(section, "path", match_path)
template.set_option(section, "repo", self.repo)
template.set_option(section, "enabled", "yes")
template.set_option(section, "multi_tool", multi_tool)
template.set_option(section, "branch", self.branch)
template.set_option(section, "version", self.version)
template.set_option(section, "last_updated",
str(datetime.utcnow()) + " UTC")
template.set_option(section, "image_name",
image_name.replace('@', '-'))
template.set_option(section, "type", "repository")
# save settings in vent.template to plugin_manifest
# watch for multiple tools in same directory
# just wanted to store match path with @ for path for use in
# other actions
tool_template = 'vent.template'
if match[0].find('@') >= 0:
tool_template = match[0].split('@')[1] + '.template'
vent_template_path = join(match_path, tool_template)
if os.path.exists(vent_template_path):
with open(vent_template_path) as f:
vent_template_val = f.read()
else:
vent_template_val = ''
settings_dict = ParsedSections(vent_template_val)
for setting in settings_dict:
template.set_option(section, setting,
json.dumps(settings_dict[setting]))
# TODO do we need this if we save as a dictionary?
vent_template = Template(vent_template_path)
vent_status, response = vent_template.option("info", "name")
if vent_status:
template.set_option(section, "link_name", response)
else:
template.set_option(section, "link_name",
true_name.split('/')[-1])
commit_id = None
if self.version == 'HEAD':
# remove @ in multi-tools
chdir(match_path)
cmd = "git rev-parse --short HEAD"
commit_id = check_output(shlex.split(cmd),
stderr=STDOUT,
close_fds=True).strip()
template.set_option(section, "commit_id", commit_id)
if head:
# no need to store previous commits if not HEAD, since
# the version will always be the same commit ID
if previous_commit and previous_commit != commit_id:
if (previous_commits
and previous_commit not in previous_commits):
previous_commits = (previous_commit + ',' +
previous_commits)
elif not previous_commits:
previous_commits = previous_commit
if previous_commits and previous_commits != commit_id:
template.set_option(section, "previous_versions",
previous_commits)
if self.version_alias:
template.set_option(section, "version_alias",
self.version_alias)
if self.groups:
template.set_option(section, "groups", self.groups)
else:
groups = vent_template.option("info", "groups")
if groups[0]:
template.set_option(section, "groups", groups[1])
# set groups to empty string if no groups defined for tool
else:
template.set_option(section, "groups", '')
template = self._build_image(template, match_path, image_name,
section)
# write additional entries for multiple instances
if addtl_entries > 0:
# add 2 for naming conventions
for i in range(2, addtl_entries + 2):
addtl_section = section.rsplit(':', 2)
addtl_section[0] += str(i)
addtl_section = ':'.join(addtl_section)
template.add_section(addtl_section)
orig_vals = template.section(section)[1]
for val in orig_vals:
template.set_option(addtl_section, val[0], val[1])
template.set_option(addtl_section, "name",
true_name.split('/')[-1] + str(i))
# write out configuration to the manifest file
template.write_config()
# reset to repo directory
chdir(self.path)
return
def _build_image(self,
template,
match_path,
image_name,
section,
build_local=False):
"""
Build docker images and store results in template
"""
def set_instances(template, section, built, image_id=None):
""" Set build information for multiple instances """
self.logger.info("entering set_instances")
i = 2
while True:
addtl_section = section.rsplit(':', 2)
addtl_section[0] += str(i)
addtl_section = ':'.join(addtl_section)
self.logger.info(addtl_section)
if template.section(addtl_section)[0]:
template.set_option(addtl_section, "built", built)
if image_id:
template.set_option(addtl_section, "image_id",
image_id)
template.set_option(addtl_section, "last_updated",
Timestamp())
else:
break
i += 1
# determine whether a tool should be considered a multi instance
try:
settings_dict = json.loads(template.option(section, 'settings')[1])
if int(settings_dict['instances']) > 1:
multi_instance = True
else:
multi_instance = False
except Exception:
multi_instance = False
# !! TODO return status of whether it built successfully or not
if self.build:
cwd = getcwd()
chdir(match_path)
try:
# currently can't use docker-py because it doesn't support
# labels on images yet
name = template.option(section, "name")
groups = template.option(section, "groups")
repo = template.option(section, "repo")
t_type = template.option(section, "type")
if groups[1] == "" or not groups[0]:
groups = (True, "none")
if not name[0]:
name = (True, image_name)
# pull if '/' in image_name, fallback to build
pull = False
if '/' in image_name and not build_local:
try:
self.logger.info("Trying to pull " + image_name)
output = check_output(shlex.split("docker pull " +
image_name),
stderr=STDOUT,
close_fds=True)
self.logger.info("Pulling " + name[1] + "\n" +
str(output))
i_attrs = self.d_client.images.get(image_name).attrs
image_id = i_attrs['Id'].split(':')[1][:12]
if image_id:
template.set_option(section, "built", "yes")
template.set_option(section, "image_id", image_id)
template.set_option(
section, "last_updated",
str(datetime.utcnow()) + " UTC")
# set other instances too
if multi_instance:
set_instances(template, section, 'yes',
image_id)
status = (True, "Pulled " + image_name)
self.logger.info(str(status))
else:
template.set_option(section, "built", "failed")
template.set_option(
section, "last_updated",
str(datetime.utcnow()) + " UTC")
# set other instances too
if multi_instace:
set_instances(template, section, 'failed')
status = (False, "Failed to pull image " +
str(output.split('\n')[-1]))
self.logger.warning(str(status))
pull = True
except Exception as e: # pragma: no cover
self.logger.warning("Failed to pull image, going to"
" build instead: " + str(e))
if not pull:
# see if additional tags needed for images tagged at HEAD
commit_tag = ""
image_name = image_name.replace('@', '-')
if image_name.endswith('HEAD'):
commit_id = template.option(section, "commit_id")
if commit_id[0]:
commit_tag = (" -t " + image_name[:-4] +
str(commit_id[1]))
# see if additional file arg needed for building multiple
# images from same directory
file_tag = " ."
multi_tool = template.option(section, 'multi_tool')
if multi_tool[0] and multi_tool[1] == 'yes':
specific_file = template.option(section, 'name')[1]
if specific_file == 'unspecified':
file_tag = " -f Dockerfile ."
else:
file_tag = " -f Dockerfile." + specific_file + " ."
# update image name with new version for update
image_name = image_name.rsplit(':',
1)[0] + ':' + self.version
output = check_output(
shlex.split("docker build --label"
" vent --label"
" vent.section=" + section + " --label"
" vent.repo=" + repo[1] + " --label"
" vent.type=" + t_type[1] + " --label"
" vent.name=" + name[1] + " --label"
" vent.groups=" + groups[1] + " -t " +
image_name + commit_tag + file_tag),
stderr=STDOUT,
close_fds=True)
self.logger.info("Building " + name[1] + "\n" +
str(output))
image_id = ""
for line in output.split("\n"):
suc_str = "Successfully built "
if line.startswith(suc_str):
image_id = line.split(suc_str)[1].strip()
template.set_option(section, "built", "yes")
template.set_option(section, "image_id", image_id)
template.set_option(section, "last_updated",
str(datetime.utcnow()) + " UTC")
# set other instances too
if multi_instance:
set_instances(template, section, 'yes', image_id)
except Exception as e: # pragma: no cover
self.logger.error("unable to build image: " + str(image_name) +
" because: " + str(e))
template.set_option(section, "built", "failed")
template.set_option(section, "last_updated",
str(datetime.utcnow()) + " UTC")
if multi_instance:
set_instances(template, section, 'failed')
chdir(cwd)
else:
template.set_option(section, "built", "no")
template.set_option(section, "last_updated",
str(datetime.utcnow()) + " UTC")
if multi_instance:
set_instances(template, section, 'no')
template.set_option(section, 'running', 'no')
return template
def list_tools(self):
"""
Return list of tuples of all tools
"""
tools = []
template = Template(template=self.manifest)
exists, sections = template.sections()
if exists:
for section in sections:
options = {
'section': section,
'enabled': None,
'built': None,
'version': None,
'repo': None,
'branch': None,
'name': None,
'groups': None,
'image_name': None
}
for option in options.keys():
exists, value = template.option(section, option)
if exists:
options[option] = value
tools.append(options)
return tools
def remove(self,
name=None,
repo=None,
namespace=None,
branch="master",
groups=None,
enabled="yes",
version="HEAD",
built="yes"):
"""
Remove tool (name) or repository, repository is the url. If no
arguments are specified, all tools will be removed for the defaults.
"""
# initialize
args = locals()
# want to remove things from manifest regardless of if built
del args['built']
status = (True, None)
# get resulting dict of sections with options that match constraints
results, template = self.p_helper.constraint_options(args, [])
for result in results:
response, image_name = template.option(result, 'image_name')
name = template.option(result, 'name')[1]
try:
settings_dict = json.loads(
template.option(result, 'settings')[1])
instances = int(settings_dict['instances'])
except Exception:
instances = 1
try:
# check for container and remove
c_name = image_name.replace(':', '-').replace('/', '-')
for i in range(1, instances + 1):
container_name = c_name + str(i) if i != 1 else c_name
container = self.d_client.containers.get(container_name)
response = container.remove(v=True, force=True)
self.logger.info(response)
self.logger.info("Removing plugin container: " +
container_name)
except Exception as e: # pragma: no cover
self.logger.warn("Unable to remove the plugin container: " +
container_name + " because: " + str(e))
# check for image and remove
try:
response = None
image_id = template.option(result, 'image_id')[1]
response = self.d_client.images.remove(image_id, force=True)
self.logger.info(response)
self.logger.info("Removing plugin image: " + image_name)
except Exception as e: # pragma: no cover
self.logger.warn("Unable to remove the plugin image: " +
image_name + " because: " + str(e))
# remove tool from the manifest
for i in range(1, instances + 1):
res = result.rsplit(':', 2)
res[0] += str(i) if i != 1 else ''
res = ':'.join(res)
if template.section(res)[0]:
status = template.del_section(res)
self.logger.info("Removing plugin tool: " + res)
# TODO if all tools from a repo have been removed, remove the repo
template.write_config()
return status
def update(self,
name=None,
repo=None,
namespace=None,
branch=None,
groups=None):
"""
Update tool (name) or repository, repository is the url. If no
arguments are specified, all tools will be updated
"""
# initialize
args = locals()
status = (False, None)
options = ['branch', 'groups', 'image_name']
# get resulting dict of sections with options that match constraints
results, template = self.p_helper.constraint_options(args, options)
for result in results:
# check for container and remove
try:
container_name = results['image_name'].replace(':', '-') \
.replace('/', '-')
container = self.d_client.containers.get(container_name)
container.remove(v=True, force=True)
except Exception as e: # pragma: no cover
self.logger.info("Error updating: " + str(result) +
" because: " + str(e))
# TODO git pull
# TODO build
# TODO docker pull
# TODO update tool in the manifest
self.logger.info("Updating plugin tool: " + result)
template.write_config()
return status
# !! TODO name or group ?
def versions(self, name, namespace=None, branch="master"):
"""
Return available versions of a tool
"""
# initialize
args = locals()
versions = []
options = ['version', 'previous_versions']
# get resulting dict of sections with options that match constraints
results, _ = self.p_helper.constraint_options(args, options)
for result in results:
version_list = [results[result]['version']]
if 'previous_versions' in results[result]:
version_list += (results[result]['previous_versions']) \
.split(',')
versions.append((result, version_list))
return versions
# !! TODO name or group ?
def current_version(self, name, namespace=None, branch="master"):
"""
Return current version for a given tool
"""
# initialize
args = locals()
versions = []
options = ['version']
# get resulting dict of sections with options that match constraints
results, _ = self.p_helper.constraint_options(args, options)
for result in results:
versions.append((result, results[result]['version']))
return versions
# !! TODO name or group ?
def state(self, name, namespace=None, branch="master"):
"""
Return state of a tool, disabled/enabled for each version
"""
# initialize
args = locals()
states = []
options = ['enabled']
# get resulting dict of sections with options that match constraints
results, _ = self.p_helper.constraint_options(args, options)
for result in results:
if results[result]['enabled'] == 'yes':
states.append((result, 'enabled'))
else:
states.append((result, 'disabled'))
return states
# !! TODO name or group ?
def enable(self, name, namespace=None, branch="master", version="HEAD"):
"""
Enable tool at a specific version, default to head
"""
# initialize
args = locals()
status = (False, None)
# get resulting dict of sections with options that match constraints
results, template = self.p_helper.constraint_options(args, [])
for result in results:
status = template.set_option(result, 'enabled', 'yes')
template.write_config()
return status
# !! TODO name or group ?
def disable(self, name, namespace=None, branch="master", version="HEAD"):
"""
Disable tool at a specific version, default to head
"""
# initialize
args = locals()
status = (False, None)
# get resulting dict of sections with options that match constraints
results, template = self.p_helper.constraint_options(args, [])
for result in results:
status = template.set_option(result, 'enabled', 'no')
template.write_config()
return status
def auto_install(self):
"""
Automatically detects images and installs them in the manifest if they
are not there already
"""
template = Template(template=self.manifest)
sections = template.sections()
images = self.d_client.images.list(filters={'label': 'vent'})
add_sections = []
status = (True, None)
for image in images:
if ('Labels' in image.attrs
and 'vent.section' in image.attrs['Config']['Labels']
and not image.attrs['Config']['Labels']['vent.section']
in sections[1]):
section = image.attrs['Config']['Labels']['vent.section']
section_str = image.attrs['Config']['Labels'][
'vent.section'].split(":")
template.add_section(section)
if 'vent.name' in image.attrs['Config']['Labels']:
template.set_option(
section, 'name',
image.attrs['Config']['Labels']['vent.name'])
if 'vent.repo' in image.attrs['Config']['Labels']:
template.set_option(
section, 'repo',
image.attrs['Config']['Labels']['vent.repo'])
git_path = join(self.path_dirs.plugins_dir,
"/".join(section_str[:2]))
if not isdir(git_path):
# clone it down
status = self.p_helper.clone(
image.attrs['Config']['Labels']['vent.repo'])
template.set_option(section, 'path',
join(git_path, section_str[-3][1:]))
# get template settings
# TODO account for template files not named vent.template
v_template = Template(template=join(
git_path, section_str[-3][1:], 'vent.template'))
tool_sections = v_template.sections()
if tool_sections[0]:
for s in tool_sections[1]:
section_dict = {}
options = v_template.options(s)
if options[0]:
for option in options[1]:
option_name = option
if option == 'name':
# get link name
template.set_option(
section, "link_name",
v_template.option(s, option)[1])
option_name = 'link_name'
opt_val = v_template.option(s, option)[1]
section_dict[option_name] = opt_val
if section_dict:
template.set_option(section, s,
json.dumps(section_dict))
if ('vent.type' in image.attrs['Config']['Labels']
and image.attrs['Config']['Labels']['vent.type']
== 'repository'):
template.set_option(section, 'namespace',
"/".join(section_str[:2]))
template.set_option(section, 'enabled', 'yes')
template.set_option(section, 'branch', section_str[-2])
template.set_option(section, 'version', section_str[-1])
template.set_option(section, 'last_updated',
str(datetime.utcnow()) + " UTC")
template.set_option(section, 'image_name',
image.attrs['RepoTags'][0])
template.set_option(section, 'type', 'repository')
if 'vent.groups' in image.attrs['Config']['Labels']:
template.set_option(
section, 'groups',
image.attrs['Config']['Labels']['vent.groups'])
template.set_option(section, 'built', 'yes')
template.set_option(section, 'image_id',
image.attrs['Id'].split(":")[1][:12])
template.set_option(section, 'running', 'no')
# check if image is running as a container
containers = self.d_client.containers.list(
filters={'label': 'vent'})
for container in containers:
if container.attrs['Image'] == image.attrs['Id']:
template.set_option(section, 'running', 'yes')
add_sections.append(section)
template.write_config()
if status[0]:
status = (True, add_sections)
return status
def tools_status(self, core, branch="master", version="HEAD", **kargs):
"""
Get tools that are currently installed/built/running and also the
number of repos that those tools come from; can toggle whether looking
for core tools or plugin tools
"""
# !! TODO this might need to store namespaces/branches/versions
all_tools = {'built': [], 'running': [], 'installed': [], 'normal': []}
core_repo = 'https://github.com/cyberreboot/vent'
repos = set()
tools = Tools(**kargs)
# get manifest file
manifest = os.path.join(self.api_action.plugin.path_dirs.meta_dir,
"plugin_manifest.cfg")
template = Template(template=manifest)
tools = template.sections()
# get repos
if core:
p_helper = PluginHelper(plugins_dir='.internals/plugins/')
repos.add(core_repo)
else:
p_helper = PluginHelper(plugins_dir='plugins/')
for tool in tools[1]:
repo = template.option(tool, 'repo')
if repo[0] and repo[1] != core_repo:
repos.add(repo[1])
# get normal tools
for repo in repos:
status, _ = p_helper.clone(repo)
if status:
p_helper.apply_path(repo)
p_helper.checkout(branch=branch, version=version)
path, _, _ = p_helper.get_path(repo, core=core)
matches = None
if core:
matches = p_helper.available_tools(path, version=version,
groups='core')
else:
matches = p_helper.available_tools(path, version=version)
for match in matches:
if core:
all_tools['normal'].append(match[0].split('/')[-1].replace('_', '-'))
else:
all_tools['normal'].append(match[0].split('/')[-1])
# get tools that have been installed
for tool in tools[1]:
repo = template.option(tool, "repo")
if repo[0] and repo[1] in repos:
name = template.option(tool, "name")
if name[0]:
all_tools['installed'].append(name[1].replace('_', '-'))
# get tools that have been built and/or are running
try:
d_client = docker.from_env()
images = d_client.images.list(filters={'label': 'vent'})
for image in images:
try:
core_check = ("vent.groups" in image.attrs['Config']['Labels'] and
'core' in image.attrs['Config']['Labels']['vent.groups'])
image_check = None
if core:
image_check = core_check
else:
image_check = not core_check
if image_check:
if ('vent.name' in image.attrs['Config']['Labels'] and
'hidden' not in image.attrs['Config']['Labels']['vent.groups']):
if core:
all_tools['built'].append(image.attrs['Config']['Labels']['vent.name'].replace('_', '-'))
else:
all_tools['built'].append(image.attrs['Config']['Labels']['vent.name'])
except Exception as err: # pragma: no cover
self.logger.error("image_check went wrong " + str(err))
containers = d_client.containers.list(filters={'label': 'vent'})
for container in containers:
try:
core_check = ("vent.groups" in container.attrs['Config']['Labels'] and
'core' in container.attrs['Config']['Labels']['vent.groups'])
container_check = None
if core:
container_check = core_check
else:
container_check = not core_check
if container_check:
if ('vent.name' in container.attrs['Config']['Labels'] and
'hidden' not in image.attrs['Config']['Labels']['vent.groups']):
if core:
all_tools['running'].append(container.attrs['Config']['Labels']['vent.name'].replace('_', '-'))
else:
all_tools['running'].append(container.attrs['Config']['Labels']['vent.name'])
except Exception as err: # pragma: no cover
self.logger.error("core_check went wrong " + str(err))
except Exception as e: # pragma: no cover
self.logger.error("Something with docker went wrong " + str(e))
return (len(repos), all_tools)
class EditorForm(npyscreen.ActionForm):
""" Form that can be used as a pseudo text editor in npyscreen """
def __init__(self,
repo='',
tool_name='',
branch='',
version='',
next_tool=None,
just_downloaded=False,
vent_cfg=False,
from_registry=False,
new_instance=False,
*args,
**keywords):
""" Initialize EditorForm objects """
# default for any editor
self.settings = locals()
self.settings.update(keywords)
del self.settings['self']
del self.settings['args']
del self.settings['keywords']
del self.settings['parentApp']
self.p_helper = PluginHelper(plugins_dir='.internals/')
self.tool_identifier = {
'name': tool_name,
'branch': branch,
'version': version
}
self.settings.update(self.tool_identifier)
del self.settings['name']
self.settings['tool_name'] = tool_name
self.settings['next_tool'] = next_tool
self.settings['repo'] = repo
# setup checks
self.just_downloaded = ('just_downloaded' in self.settings
and self.settings['just_downloaded'])
self.vent_cfg = ('vent_cfg' in self.settings
and self.settings['vent_cfg'])
self.registry_tool = ('from_registry' in self.settings
and self.settings['from_registry'])
self.instance_cfg = ('new_instance' in self.settings
and self.settings['new_instance'])
# get manifest info for tool that will be used throughout
if not self.just_downloaded and not self.vent_cfg:
result = self.p_helper.constraint_options(self.tool_identifier, [])
tool, self.manifest = result
self.section = tool.keys()[0]
# get configuration information depending on type
if self.just_downloaded:
self.config_val = '[info]\n'
self.config_val += 'name = ' + keywords['link_name'] + '\n'
self.config_val += 'groups = ' + keywords['groups'] + '\n'
elif self.vent_cfg:
self.config_val = keywords['get_configure'](main_cfg=True)[1]
self.settings['tool_name'] = 'vent configuration'
elif self.instance_cfg:
path = self.manifest.option(self.section, 'path')[1]
# defaults in .internals
path = path.replace('.vent/plugins', '.vent/.internals')
multi_tool = self.manifest.option(self.section, 'multi_tool')
if multi_tool[0] and multi_tool[1] == 'yes':
name = self.manifest.option(self.section, 'name')[1]
if name == 'unspecified':
name = 'vent'
template_path = os.path.join(path, name + '.template')
else:
template_path = os.path.join(path, 'vent.template')
# ensure instances is in the editor and that it is the right number
template = Template(template_path)
template.add_section('settings')
template.set_option('settings', 'instances',
str(self.settings['new_instances']))
template.write_config()
with open(template_path) as vent_template:
self.config_val = vent_template.read()
else:
self.config_val = keywords['get_configure'](
**self.tool_identifier)[1]
super(EditorForm, self).__init__(*args, **keywords)
def create(self):
""" Create multi-line widget for editing """
# add various pointers to those editing vent_cfg
if self.vent_cfg:
self.add(npyscreen.Textfield,
value='# when configuring external'
' services make sure to do so',
editable=False)
self.add(npyscreen.Textfield,
value='# in the form of Service = {"setting": "value"}',
editable=False)
self.add(npyscreen.Textfield,
value='# make sure to capitalize your service correctly'
' (i.e. Elasticsearch vs. elasticsearch)',
editable=False)
self.add(npyscreen.Textfield,
value='# and make sure to enclose all dict keys and'
' values in double quotes ("")',
editable=False)
self.add(npyscreen.Textfield, value='', editable=False)
elif self.instance_cfg:
self.add(npyscreen.Textfield,
value='# these settings will be used'
' to configure the new instances',
editable=False)
self.edit_space = self.add(npyscreen.MultiLineEdit,
value=self.config_val)
def change_screens(self):
""" Change to the next tool to edit or back to MAIN form """
if self.settings['next_tool']:
self.parentApp.change_form(self.settings['next_tool'])
else:
self.parentApp.change_form("MAIN")
@staticmethod
def valid_input(val):
""" Ensure the input the user gave is of a valid format """
# looks for 3 nums followed by a dot 3 times and then ending with
# 3 nums, can be proceeded by any number of spaces
ip_value = re.compile(r'(\d{1,3}\.){3}\d{1,3}$')
# looks for only numbers and commas (because priorities can have commas
# between them), can be proceeded by any number of spaces
all_num = re.compile(r'(\d,?\ *)+$')
sections_comments = re.compile(
r"""
\ *\#.* # comments (any number of whitespace, then #
# followed by anything)
| \[[\w-]+\]$ # section headers (any combination of chars, nums,
# underscores, and dashes between brackets)
""", re.VERBOSE)
# can't can be a comment on option side and value side can't have
# [, ], {, or } otherwise it is turned over to literal_eval for
# checkout
options_values = re.compile(r"[^# ]+\ *=[^[\]{}]*$")
line_num = 0
warning_str = ''
error_str = ''
trimmed_val = []
for entry in val.split('\n'):
line_num += 1
# get rid of any extraneous commas at the end of a dict and remove
# extra whitespace from input
trimmed_val.append(re.sub(r',\ *}', '}', entry).strip())
# empty line
if entry.strip() == '':
continue
# look at regular (non dictionary or list) option-value pairs
if options_values.match(entry):
value = entry.split('=', 1)[1]
# deal with potentially more equals signs
for val in value.split('='):
val = val.strip()
# empty val means malformed equals signs
if val == '':
error_str += '-You have a misplaced equals sign on' \
' line ' + str(line_num) + '\n'
# starts with a num; look for bad ip input or warn user
# about having extraneous characters in number input
if re.match('\ *\d', val):
# bad ip syntax
if val.find('.') >= 0 and not ip_value.match(val):
error_str += '-You have an incorrectly' \
' formatted ip address (bad syntax) at' \
' line ' + str(line_num) + '\n'
# possibly malformed numbers
elif val.find('.') < 0 and not all_num.match(val):
warning_str += '-Line starting with a number has' \
' characters mixed in at line ' + \
str(line_num) + '\n'
# bad ip values
elif val.find('.') >= 0:
for num in val.strip().split('.'):
num = int(num)
if num > 255 or num < 0:
error_str += '-You have an incorrectly' \
' formatted ip address (values' \
' exceeding 255 or below 0) at' \
' line ' + str(line_num) + '\n'
# ensure no lines end with a comma (most likely extraneous
# commas from groups or priorities)
if re.search(',$', val):
error_str += '-You have an incorrect comma at the' \
' end of line ' + str(line_num) + '\n'
# see if input is a header or comment, otherwise try to
# literal_eval it to ensure correct structure
elif not sections_comments.match(entry):
lit_val = ''
try:
opt_val = entry.split('=', 1)
if opt_val[0].strip() == '':
error_str += '-You have nothing preceeding an' \
' equals sign at line ' + str(line_num) + '\n'
else:
lit_val = opt_val[1].strip()
except IndexError:
lit_val = ''
error_str += '-You have an incorrectly formatted' \
' section header at line ' + str(line_num) + '\n'
if lit_val:
try:
ast.literal_eval(lit_val)
except SyntaxError:
error_str += '-You have an incorrectly formatted' \
' list/dictionary at line ' + str(line_num) + \
'\n'
if error_str:
npyscreen.notify_confirm("You have the following error(s) and"
" can't proceed until they are fixed:" +
"\n" + "-" * 50 + "\n" + error_str,
title="Error in input")
return (False, '')
elif warning_str:
res = npyscreen.notify_yes_no(
"You have may have some error(s)"
" that you want to check before"
" proceeding:" + "\n" + "-" * 50 + "\n" + warning_str + "\n" +
"-" * 50 + "\n" + "Do you want to continue?",
title="Double check")
return (res, '\n'.join(trimmed_val))
return (True, '\n'.join(trimmed_val))
def on_ok(self):
""" Save changes made to vent.template """
# ensure user didn't have any syntactical errors
input_is_good, trimmed_input = self.valid_input(self.edit_space.value)
if not input_is_good:
return
self.edit_space.value = trimmed_input
# get the number of instances and ensure user didn't malform that
if re.search(r"instances\ *=", self.edit_space.value):
try:
# split out spaces
instances_val = re.split(r"instances\ *=\ *",
self.edit_space.value)[1]
instances_val = instances_val.split('\n')[0]
new_instances = int(re.match(r"\d+$", instances_val).group())
except AttributeError:
npyscreen.notify_confirm(
"You didn't specify a valid number"
" for instances.",
title="Invalid"
" instance number")
return
# user can't change instances when configuring new instnaces
if (self.instance_cfg
and self.settings['new_instances'] != new_instances):
npyscreen.notify_confirm(
"You can't change the number of"
" instnaces while configuring new"
" instances!",
title="Illegal change")
return
# get old number of instances
try:
if 'old_instances' in self.settings:
old_instances = self.settings['old_instances']
else:
settings_dict = json.loads(
self.manifest.option(self.section, 'settings')[1])
old_instances = int(settings_dict['instances'])
except Exception:
old_instances = 1
else:
new_instances = 1
old_instances = 1
# save changes and update manifest we're looking at with changes
if self.vent_cfg:
save_args = {'main_cfg': True, 'config_val': self.edit_space.value}
self.manifest = self.settings['save_configure'](**save_args)[1]
else:
save_args = copy.deepcopy(self.tool_identifier)
save_args.update({'config_val': self.edit_space.value})
if self.registry_tool:
save_args.update({'from_registry': True})
if self.instance_cfg:
save_args.update({'instances': new_instances})
self.manifest = self.settings['save_configure'](**save_args)[1]
# restart tools, if necessary
if not self.just_downloaded and not self.instance_cfg:
restart_kargs = {
'main_cfg': self.vent_cfg,
'old_val': self.config_val,
'new_val': self.edit_space.value
}
if self.vent_cfg:
wait_str = "Restarting tools affected by changes..."
else:
wait_str = "Restarting this tool with new settings..."
restart_kargs.update(self.tool_identifier)
npyscreen.notify_wait(wait_str, title="Restarting with changes")
self.settings['restart_tools'](**restart_kargs)
# start new instances if user wanted to
if self.instance_cfg and self.settings['start_new']:
npyscreen.notify_wait("Starting new instances...", title="Start")
tool_d = {}
for i in range(self.settings['old_instances'] + 1,
self.settings['new_instances'] + 1):
# create section by scrubbing instance number out of names
# and adding new instance number
i_section = self.section.rsplit(':', 2)
i_section[0] = re.sub(r'[0-9]', '', i_section[0]) + str(i)
i_section = ':'.join(i_section)
t_name = self.manifest.option(i_section, 'name')[1]
t_branch = self.manifest.option(i_section, 'branch')[1]
t_version = self.manifest.option(i_section, 'version')[1]
t_id = {
'name': t_name,
'branch': t_branch,
'version': t_version
}
tool_d.update(self.settings['prep_start'](**t_id)[1])
if tool_d:
self.settings['start_tools'](tool_d)
# prompt user for instance changes, as necessary
if not self.instance_cfg and not self.vent_cfg:
if new_instances > old_instances:
try:
diff = str(new_instances - old_instances)
res = npyscreen.notify_yes_no("You will be creating " +
diff + " additional"
" instance(s) is that okay?",
title="Confirm new"
" instance(s)")
if res:
if self.manifest.option(self.section,
'built')[1] == 'yes':
run = npyscreen.notify_yes_no(
"Do you want to"
" start these new"
" tools upon"
" creation?",
title="Run new"
" instance(s)")
else:
run = False
# get clean name (no instance numbers in it)
new_name = self.settings['tool_name']
new_name = re.sub(r'[0-9]+$', '', new_name)
self.settings['tool_name'] = new_name
npyscreen.notify_wait(
"Pulling up default settings"
" for " + self.settings['tool_name'] + "...",
title="Gathering settings")
self.p_helper.clone(self.settings['repo'])
self.settings['new_instances'] = new_instances
self.settings['old_instances'] = old_instances
self.settings['start_new'] = run
self.settings['new_instance'] = True
self.settings['name'] = "Configure new instance(s)" + \
" for " + self.settings['tool_name']
self.parentApp.addForm(
'INSTANCEEDITOR' + self.settings['tool_name'],
EditorForm, **self.settings)
self.parentApp.change_form('INSTANCEEDITOR' +
self.settings['tool_name'])
else:
return
except Exception:
npyscreen.notify_confirm(
"Trouble finding tools to add,"
" exiting",
title="Error")
self.on_cancel()
elif new_instances < old_instances:
try:
diff = str(old_instances - new_instances)
res = npyscreen.notify_yes_no("You will be deleting " +
diff + " instance(s), is"
" that okay?",
title="Confirm delete"
" instance(s)")
if res:
form_name = 'Delete instances for ' + \
re.sub(r'\d+$', '',
self.settings['tool_name']) + '\t'*8 + \
'^E to exit configuration process'
clean_section = self.section.rsplit(':', 2)
clean_section[0] = re.sub(r'\d+$', '',
clean_section[0])
clean_section = ':'.join(clean_section)
d_args = {
'name': form_name,
'new_instances': new_instances,
'old_instances': old_instances,
'next_tool': self.settings['next_tool'],
'manifest': self.manifest,
'section': clean_section,
'clean': self.settings['clean'],
'prep_start': self.settings['prep_start'],
'start_tools': self.settings['start_tools']
}
self.parentApp.addForm(
'DELETER' + self.settings['tool_name'], DeleteForm,
**d_args)
self.parentApp.change_form('DELETER' +
self.settings['tool_name'])
except Exception:
npyscreen.notify_confirm(
"Trouble finding instances to"
" delete, exiting",
title="Error")
self.on_cancel()
if (new_instances == old_instances or self.instance_cfg
or self.vent_cfg):
npyscreen.notify_confirm("Done configuring " +
self.settings['tool_name'],
title="Configurations saved")
self.change_screens()
def on_cancel(self):
""" Don't save changes made to vent.template """
npyscreen.notify_confirm("No changes made to " +
self.settings['tool_name'],
title="Configurations not saved")
self.change_screens()