def test_current_version():
""" Test the current_version function """
instance = Plugin()
versions = instance.current_version('elasticsearch', branch='master')
assert versions == [
('cyberreboot:vent:/vent/core/elasticsearch:master:HEAD', 'HEAD')
]
def test_repo_tools():
""" Test the repo_tools function """
instance = Plugin()
status = instance.repo_tools('https://github.com/cyberreboot/vent', 'master', 'HEAD')
assert type(status) == tuple
assert status[0] == True
assert type(status[1]) == list
def test_versions():
""" Test the versions function """
instance = Plugin()
status = instance.add('https://github.com/cyberreboot/vent', build=False, branch='master')
assert status[0] == True
versions = instance.versions('elasticsearch', branch='master')
assert versions == [('cyberreboot:vent:/vent/core/elasticsearch:master:HEAD', ['HEAD'])]
def test_repo_commits():
""" Test the repo_commits function """
instance = Plugin()
status = instance.add('https://github.com/cyberreboot/vent', build=False)
assert isinstance(status, tuple)
assert status[0] == True
instance = MenuHelper()
status = instance.repo_commits('https://github.com/cyberreboot/vent')
assert isinstance(status, tuple)
def test_apply_path():
""" Test the apply_path function """
instance = Plugin()
status = instance.apply_path('https://github.com/cyberreboot/vent')
assert type(status) == tuple
assert status[0] == True
status = instance.apply_path('https://github.com/cyberreboot/vent.git')
assert type(status) == tuple
assert status[0] == True
def repo_tools(self, branch):
""" Set the appropriate repo dir and get the tools available of it """
tools = []
plugin = Plugin()
status = plugin.repo_tools(self.parentApp.repo_value['repo'], branch, self.parentApp.repo_value['versions'][branch])
if status[0]:
r_tools = status[1]
for tool in r_tools:
tools.append(tool[0])
return tools
def test_add_image():
""" Test the add_image function """
instance = Plugin()
#status = instance.add_image('quay/redis', 'redis', registry='quay.io')
#assert isinstance(status, tuple)
#assert status[0] == True
status = instance.add_image('alpine',
'alpine',
tag='latest',
groups='alpine')
assert isinstance(status, tuple)
assert status[0] == True
def test_versions():
""" Test the versions function """
instance = Plugin()
status = instance.add('https://github.com/cyberreboot/vent',
build=False,
branch='master')
assert status[0] == True
versions = instance.versions('elasticsearch', branch='master')
assert isinstance(versions, list)
assert isinstance(versions[0], tuple)
assert isinstance(versions[0][1], list)
assert versions[0][
0] == 'cyberreboot:vent:/vent/core/elasticsearch:master:HEAD'
assert 'HEAD' in versions[0][1]
def test_builder():
""" Test the builder function """
instance = Plugin()
template = Template(instance.manifest)
template = instance.builder(template,
os.getcwd() + '/plugins/cyberreboot/vent',
'image_name', 'section')
template = instance.builder(template,
'bad_path',
'image_name',
'section',
build=True,
branch='master',
version='HEAD')
def on_ok(self):
""" Add the repository """
self.parentApp.repo_value['repo'] = self.repo.value
def popup(thr, title):
"""
Start the thread and display a popup of the plugin being cloned
until the thread is finished
"""
thr.start()
tool_str = "Cloning repository..."
npyscreen.notify_wait(tool_str, title=title)
while thr.is_alive():
time.sleep(1)
return
api_plugin = Plugin()
thr = threading.Thread(target=api_plugin.clone,
args=(),
kwargs={
'repo': self.repo.value,
'user': self.user.value,
'pw': self.pw.value
})
popup(thr, 'Please wait, adding repository...')
self.parentApp.change_form('ADDOPTIONS')
def Core(branch="master", **kargs):
"""
Get the normal core tools, and the currently installed/built/running ones,
including custom core services
"""
# !! TODO this might need to store namespaces/branches/versions
core = {'built':[], 'running':[], 'installed':[], 'normal':[]}
# get normal core tools
plugins = Plugin(plugins_dir=".internals/plugins")
status, cwd = plugins.clone('https://github.com/cyberreboot/vent')
if status:
plugins.version = 'HEAD'
plugins.branch = branch
response = plugins.checkout()
matches = plugins._available_tools(groups='core')
for match in matches:
core['normal'].append(match[0].split('/')[-1])
else:
core['normal'] = 'failed'
# get core tools that have been installed
path_dirs = PathDirs(**kargs)
manifest = os.path.join(path_dirs.meta_dir, "plugin_manifest.cfg")
template = Template(template=manifest)
tools = template.sections()
if tools[0]:
for tool in tools[1]:
groups = template.option(tool, "groups")
if groups[0] and "core" in groups[1]:
name = template.option(tool, "name")
if name[0]:
core['installed'].append(name[1])
# get core tools that have been built and/or are running
try:
d_client = docker.from_env()
images = d_client.images.list()
for image in images:
try:
if "vent.groups" in image.attrs['Labels'] and 'core' in image.attrs['Labels']['vent.groups']:
if 'vent.name' in image.attrs['Labels']:
core['built'].append(image.attrs['Labels']['vent.name'])
except Exception as err: # pragma: no cover
pass
containers = d_client.containers.list()
for container in containers:
try:
if "vent.groups" in container.attrs['Config']['Labels'] and 'core' in container.attrs['Config']['Labels']['vent.groups']:
if 'vent.name' in container.attrs['Config']['Labels']:
core['running'].append(container.attrs['Config']['Labels']['vent.name'])
except Exception as err: # pragma: no cover
pass
except Exception as e: # pragma: no cover
pass
return core
def repo_values(self):
""" Set the appropriate repo dir and get the branches and commits of it """
branches = []
commits = {}
plugin = Plugin()
status = plugin.repo_branches(self.parentApp.repo_value['repo'])
# branches and commits must both be retrieved successfully
if status[0]:
branches = status[1]
status = plugin.repo_commits(self.parentApp.repo_value['repo'])
if status[0]:
r_commits = status[1]
for commit in r_commits:
commits[commit[0]] = commit[1]
else:
# if commits failed, return commit errors
return status
else:
# if branch failed, return branch errors
return status
# if everything is good, return branches with commits
return branches, commits
def repo_values(self):
""" Set the appropriate repo dir and get the branches and commits of it """
branches = []
commits = {}
plugin = Plugin()
status = plugin.repo_branches(self.parentApp.repo_value['repo'])
# branches and commits must both be retrieved successfully
if status[0]:
branches = status[1]
status = plugin.repo_commits(self.parentApp.repo_value['repo'])
if status[0]:
r_commits = status[1]
for commit in r_commits:
commits[commit[0]] = commit[1]
else:
# if commits failed, return commit errors
return status
else:
# if branch failed, return branch errors
return status
# if everything is good, return branches with commits
return branches, commits
def test_builder():
""" Test the builder function """
instance = Plugin()
template = Template(instance.manifest)
template = instance.builder(template, os.getcwd()+'/plugins/cyberreboot/vent', 'image_name', 'section')
template = instance.builder(template, 'bad_path', 'image_name', 'section', build=True, branch='master', version='HEAD')
def test_add_image():
""" Test the add_image function """
Plugin.add_image('foo')
class Action:
""" Handle actions in menu """
def __init__(self, **kargs):
self.plugin = Plugin(**kargs)
self.d_client = self.plugin.d_client
self.vent_config = os.path.join(self.plugin.path_dirs.meta_dir,
"vent.cfg")
self.logger = Logger(__name__)
def add(self, repo, tools=None, overrides=None, version="HEAD",
branch="master", build=True, user=None, pw=None, groups=None,
version_alias=None, wild=None, remove_old=True, disable_old=True):
""" Add a new set of tool(s) """
self.logger.info("Starting: add")
status = (True, None)
try:
status = self.plugin.add(repo,
tools=tools,
overrides=overrides,
version=version,
branch=branch,
build=build,
user=user,
pw=pw,
groups=groups,
version_alias=version_alias,
wild=wild,
remove_old=remove_old,
disable_old=disable_old)
except Exception as e: # pragma: no cover
self.logger.error("add failed with error: "+str(e))
status = (False, e)
self.logger.info("Status of add: "+str(status))
self.logger.info("Finished: add")
return status
def remove(self, repo=None, namespace=None, name=None, groups=None,
enabled="yes", branch="master", version="HEAD", built="yes"):
""" Remove tools or a repo """
self.logger.info("Starting: remove")
status = (True, None)
try:
status = self.plugin.remove(name=name,
repo=repo,
namespace=namespace,
groups=groups,
enabled=enabled,
branch=branch,
version=version,
built=built)
except Exception as e:
self.logger.error("remove failed with error: "+str(e))
status = (False, e)
self.logger.info("Status of remove: " + str(status))
self.logger.info("Finished: remove")
return status
def prep_start(self,
repo=None,
name=None,
groups=None,
enabled="yes",
branch="master",
version="HEAD",
run_build=False):
"""
Start a set of tools that match the parameters given, if no parameters
are given, start all installed tools on the master branch at verison
HEAD that are enabled
"""
args = locals()
self.logger.info("Starting: prep_start")
self.logger.info("Arguments: "+str(args))
status = (True, None)
tool_dict = {}
try:
del args['run_build']
options = ['name',
'namespace',
'built',
'groups',
'path',
'image_name',
'branch',
'version']
vent_config = Template(template=self.vent_config)
files = vent_config.option('main', 'files')
sections, template = self.plugin.constraint_options(args, options)
for section in sections:
# initialize needed vars
template_path = os.path.join(sections[section]['path'], 'vent.template')
container_name = sections[section]['image_name'].replace(':','-')
container_name = container_name.replace('/','-')
image_name = sections[section]['image_name']
# checkout the right version and branch of the repo
self.plugin.branch = branch
self.plugin.version = version
cwd = os.getcwd()
self.logger.info("current directory is: "+str(cwd))
os.chdir(os.path.join(sections[section]['path']))
status = self.plugin.checkout()
self.logger.info(status)
os.chdir(cwd)
if run_build:
status = self.build(name=sections[section]['name'],
groups=groups,
enabled=enabled,
branch=branch,
version=version)
self.logger.info(status)
# set docker settings for container
vent_template = Template(template_path)
status = vent_template.section('docker')
self.logger.info(status)
tool_dict[container_name] = {'image':image_name, 'name':container_name}
if status[0]:
for option in status[1]:
options = option[1]
# check for commands to evaluate
if '`' in options:
cmds = options.split('`')
# TODO this probably needs better error checking to handle mismatched ``
if len(cmds) > 2:
i = 1
while i < len(cmds):
try:
cmds[i] = subprocess.check_output(shlex.split(cmds[i]), stderr=subprocess.STDOUT, close_fds=True).strip()
except Exception as e: # pragma: no cover
self.logger.error("unable to evaluate command specified in vent.template: "+str(e))
i += 2
options = "".join(cmds)
# store options set for docker
try:
tool_dict[container_name][option[0]] = ast.literal_eval(options)
except Exception as e: # pragma: no cover
self.logger.error("unable to store the options set for docker: "+str(e))
tool_dict[container_name][option[0]] = options
# get temporary name for links, etc.
status = vent_template.section('info')
self.logger.info(status)
plugin_config = Template(template=self.plugin.manifest)
status, plugin_sections = plugin_config.sections()
self.logger.info(status)
for plugin_section in plugin_sections:
status = plugin_config.option(plugin_section, "link_name")
self.logger.info(status)
image_status = plugin_config.option(plugin_section, "image_name")
self.logger.info(image_status)
if status[0] and image_status[0]:
cont_name = image_status[1].replace(':','-')
cont_name = cont_name.replace('/','-')
if cont_name not in tool_dict:
tool_dict[cont_name] = {'image':image_status[1], 'name':cont_name, 'start':False}
tool_dict[cont_name]['tmp_name'] = status[1]
# add extra labels
if 'labels' not in tool_dict[container_name]:
tool_dict[container_name]['labels'] = {}
tool_dict[container_name]['labels']['vent'] = Version()
tool_dict[container_name]['labels']['vent.namespace'] = sections[section]['namespace']
tool_dict[container_name]['labels']['vent.branch'] = branch
tool_dict[container_name]['labels']['vent.version'] = version
tool_dict[container_name]['labels']['vent.name'] = sections[section]['name']
if 'groups' in sections[section]:
# add labels for groups
tool_dict[container_name]['labels']['vent.groups'] = sections[section]['groups']
# send logs to syslog
if 'syslog' not in sections[section]['groups'] and 'core' in sections[section]['groups']:
tool_dict[container_name]['log_config'] = {'type':'syslog', 'config': {'syslog-address':'tcp://0.0.0.0:514', 'syslog-facility':'daemon', 'tag':'core'}}
if 'syslog' not in sections[section]['groups']:
tool_dict[container_name]['log_config'] = {'type':'syslog', 'config': {'syslog-address':'tcp://0.0.0.0:514', 'syslog-facility':'daemon', 'tag':'plugin'}}
# mount necessary directories
if 'files' in sections[section]['groups']:
if 'volumes' in tool_dict[container_name]:
tool_dict[container_name]['volumes'][self.plugin.path_dirs.base_dir[:-1]] = {'bind': '/vent', 'mode': 'ro'}
else:
tool_dict[container_name]['volumes'] = {self.plugin.path_dirs.base_dir[:-1]: {'bind': '/vent', 'mode': 'ro'}}
if files[0]:
tool_dict[container_name]['volumes'][files[1]] = {'bind': '/files', 'mode': 'ro'}
else:
tool_dict[container_name]['log_config'] = {'type':'syslog', 'config': {'syslog-address':'tcp://0.0.0.0:514', 'syslog-facility':'daemon', 'tag':'plugin'}}
# add label for priority
status = vent_template.section('settings')
self.logger.info(status)
if status[0]:
for option in status[1]:
if option[0] == 'priority':
tool_dict[container_name]['labels']['vent.priority'] = option[1]
# only start tools that have been built
if sections[section]['built'] != 'yes':
del tool_dict[container_name]
# check and update links, volumes_from, network_mode
for container in tool_dict.keys():
if 'links' in tool_dict[container]:
for link in tool_dict[container]['links']:
for c in tool_dict.keys():
if 'tmp_name' in tool_dict[c] and tool_dict[c]['tmp_name'] == link:
tool_dict[container]['links'][tool_dict[c]['name']] = tool_dict[container]['links'].pop(link)
if 'volumes_from' in tool_dict[container]:
tmp_volumes_from = tool_dict[container]['volumes_from']
tool_dict[container]['volumes_from'] = []
for volumes_from in list(tmp_volumes_from):
for c in tool_dict.keys():
if 'tmp_name' in tool_dict[c] and tool_dict[c]['tmp_name'] == volumes_from:
tool_dict[container]['volumes_from'].append(tool_dict[c]['name'])
tmp_volumes_from.remove(volumes_from)
tool_dict[container]['volumes_from'] += tmp_volumes_from
if 'network_mode' in tool_dict[container]:
if tool_dict[container]['network_mode'].startswith('container:'):
network_c_name = tool_dict[container]['network_mode'].split('container:')[1]
for c in tool_dict.keys():
if 'tmp_name' in tool_dict[c] and tool_dict[c]['tmp_name'] == network_c_name:
tool_dict[container]['network_mode'] = 'container:'+tool_dict[c]['name']
# remove tmp_names
for c in tool_dict.keys():
if 'tmp_name' in tool_dict[c]:
del tool_dict[c]['tmp_name']
# remove containers that shouldn't be started
for c in tool_dict.keys():
if 'start' in tool_dict[c] and not tool_dict[c]['start']:
del tool_dict[c]
except Exception as e:
self.logger.error("prep_start failed with error: "+str(e))
status = (False, e)
status = (True, tool_dict)
self.logger.info("Status of prep_start: "+str(status))
self.logger.info("Finished: prep_start")
return status
def start(self, tool_dict):
"""
Start a set of tools that match the parameters given, if no parameters
are given, start all installed tools on the master branch at verison
HEAD that are enabled
"""
self.logger.info("Starting: start")
status = (True, None)
try:
# check start priorities (priority of groups is alphabetical for now)
group_orders = {}
groups = []
containers_remaining = []
for container in tool_dict:
containers_remaining.append(container)
if 'labels' in tool_dict[container]:
if 'vent.groups' in tool_dict[container]['labels']:
groups += tool_dict[container]['labels']['vent.groups'].split(',')
if 'vent.priority' in tool_dict[container]['labels']:
priorities = tool_dict[container]['labels']['vent.priority'].split(',')
container_groups = tool_dict[container]['labels']['vent.groups'].split(',')
for i, priority in enumerate(priorities):
if container_groups[i] not in group_orders:
group_orders[container_groups[i]] = []
group_orders[container_groups[i]].append((int(priority), container))
containers_remaining.remove(container)
# start containers based on priorities
groups = sorted(set(groups))
started_containers = []
for group in groups:
if group in group_orders:
for container_tuple in sorted(group_orders[group]):
if container_tuple[1] not in started_containers:
started_containers.append(container_tuple[1])
try:
try:
container = self.d_client.containers.get(container_tuple[1])
container.start()
self.logger.info("started "+str(container_tuple[1])+" with ID: "+str(container.short_id))
except Exception as err: # pragma: no cover
self.logger.error(str(err))
container_id = self.d_client.containers.run(detach=True, **tool_dict[container_tuple[1]])
self.logger.info("started "+str(container_tuple[1])+" with ID: "+str(container_id))
except Exception as e: # pragma: no cover
self.logger.error("failed to start "+str(container_tuple[1])+" because: "+str(e))
# start the rest of the containers that didn't have any priorities set
for container in containers_remaining:
try:
try:
c = self.d_client.containers.get(container)
c.start()
self.logger.info("started "+str(container)+" with ID: "+str(c.short_id))
except Exception as err: # pragma: no cover
self.logger.error(str(err))
container_id = self.d_client.containers.run(detach=True, **tool_dict[container])
self.logger.info("started "+str(container)+" with ID: "+str(container_id))
except Exception as e: # pragma: no cover
self.logger.error("failed to start "+str(container)+" because: "+str(e))
except Exception as e:
self.logger.error("start failed with error: "+str(e))
status = (False, e)
self.logger.info("Status of start: "+str(status))
self.logger.info("Finished: start")
return status
def update(self,
repo=None,
name=None,
groups=None,
enabled="yes",
branch="master",
version="HEAD"):
"""
Update a set of tools that match the parameters given, if no parameters
are given, updated all installed tools on the master branch at verison
HEAD that are enabled
"""
args = locals()
self.logger.info("Starting: update")
self.logger.info(args)
status = (True, None)
try:
options = ['path', 'image_name', 'image_id']
sections, template = self.plugin.constraint_options(args, options)
# get existing containers and images and states
running_containers = Containers()
built_images = Images()
self.logger.info("running docker containers: "+str(running_containers))
self.logger.info("built docker images: "+str(built_images))
# if repo, pull and build
# if registry image, pull
for section in sections:
try:
cwd = os.getcwd()
self.logger.info("current working directory: "+str(cwd))
os.chdir(sections[section]['path'])
self.plugin.version = version
self.plugin.branch = branch
c_status = self.plugin.checkout()
self.logger.info(c_status)
try:
os.chdir(cwd)
except Exception as e: # pragma: no cover
self.logger.error("unable to change directory: "+str(e))
pass
template = self.plugin.builder(template, sections[section]['path'], sections[section]['image_name'], section, build=True, branch=branch, version=version)
self.logger.info(template)
# stop and remove old containers and images if image_id updated
# !! TODO
# start containers if they were running
# !! TODO
# TODO logging
except Exception as e: # pragma: no cover
self.logger.error("unable to update: "+str(section)+" because: "+str(e))
template.write_config()
except Exception as e:
self.logger.error("update failed with error: "+str(e))
status = (False, e)
self.logger.info("Status of update: "+str(status))
self.logger.info("Finished: update")
return status
def stop(self,
repo=None,
name=None,
groups=None,
enabled="yes",
branch="master",
version="HEAD"):
"""
Stop a set of tools that match the parameters given, if no parameters
are given, stop all installed tools on the master branch at verison
HEAD that are enabled
"""
args = locals()
self.logger.info("Starting: stop")
self.logger.info(args)
status = (True, None)
try:
# !! TODO need to account for plugin containers that have random names, use labels perhaps
options = ['name',
'namespace',
'built',
'groups',
'path',
'image_name',
'branch',
'version']
sections, template = self.plugin.constraint_options(args, options)
self.logger.info(sections)
for section in sections:
container_name = sections[section]['image_name'].replace(':','-')
container_name = container_name.replace('/','-')
try:
container = self.d_client.containers.get(container_name)
container.stop()
self.logger.info("stopped "+str(container_name))
except Exception as e: # pragma: no cover
self.logger.error("failed to stop "+str(container_name)+" because: "+str(e))
except Exception as e:
self.logger.error("stop failed with error: "+str(e))
status = (False, e)
self.logger.info("Status of stop: "+str(status))
self.logger.info("Finished: stop")
return status
def clean(self,
repo=None,
name=None,
groups=None,
enabled="yes",
branch="master",
version="HEAD"):
"""
Clean (stop and remove) a set of tools that match the parameters given,
if no parameters are given, clean all installed tools on the master
branch at verison HEAD that are enabled
"""
args = locals()
self.logger.info("Starting: clean")
self.logger.info(args)
status = (True, None)
try:
# !! TODO need to account for plugin containers that have random names, use labels perhaps
options = ['name',
'namespace',
'built',
'groups',
'path',
'image_name',
'branch',
'version']
sections, template = self.plugin.constraint_options(args, options)
self.logger.info(sections)
for section in sections:
container_name = sections[section]['image_name'].replace(':','-')
container_name = container_name.replace('/','-')
try:
container = self.d_client.containers.get(container_name)
container.remove(force=True)
self.logger.info("cleaned "+str(container_name))
except Exception as e: # pragma: no cover
self.logger.error("failed to clean "+str(container_name)+" because: "+str(e))
except Exception as e:
self.logger.error("clean failed with error: "+str(e))
status = (False, e)
self.logger.info("Status of clean: "+ str(status))
self.logger.info("Finished: clean")
return status
def build(self,
repo=None,
name=None,
groups=None,
enabled="yes",
branch="master",
version="HEAD"):
""" Build a set of tools that match the parameters given """
args = locals()
self.logger.info("Starting: build")
self.logger.info(args)
status = (True, None)
try:
options = ['image_name', 'path']
sections, template = self.plugin.constraint_options(args, options)
self.logger.info(sections)
for section in sections:
self.logger.info("Building "+str(section)+" ...")
template = self.plugin.builder(template, sections[section]['path'],
sections[section]['image_name'],
section, build=True, branch=branch,
version=version)
template.write_config()
except Exception as e:
self.logger.error("build failed with error: "+str(e))
status = (False, e)
self.logger.info("Status of build: "+str(status))
self.logger.info("Finished: build")
return status
def cores(self, action, branch="master"):
""" Supply action (install, build, start, stop, clean) for core tools """
self.logger.info("Starting: cores")
status = (True, None)
try:
self.logger.info("action provided: "+str(action))
core = Core(branch=branch)
if action in ["install", "build"]:
tools = []
plugins = Plugin(plugins_dir=".internals/plugins")
plugins.version = 'HEAD'
plugins.branch = branch
plugins.apply_path('https://github.com/cyberreboot/vent')
response = plugins.checkout()
self.logger.info("status of plugin checkout "+str(response))
matches = plugins._available_tools(groups='core')
for match in matches:
tools.append((match[0], ''))
status = plugins.add('https://github.com/cyberreboot/vent', tools=tools, branch=branch, build=False)
self.logger.info("status of plugin add: "+str(status))
plugin_config = Template(template=self.plugin.manifest)
sections = plugin_config.sections()
for tool in core['normal']:
for section in sections[1]:
name = plugin_config.option(section, "name")
orig_branch = plugin_config.option(section, "branch")
namespace = plugin_config.option(section, "namespace")
version = plugin_config.option(section, "version")
if name[1] == tool and orig_branch[1] == branch and namespace[1] == "cyberreboot/vent" and version[1] == "HEAD":
plugin_config.set_option(section, "image_name", "cyberreboot/vent-"+tool+":"+branch)
plugin_config.write_config()
if action == "build":
plugin_config = Template(template=self.plugin.manifest)
sections = plugin_config.sections()
try:
for tool in core['normal']:
for section in sections[1]:
image_name = plugin_config.option(section, "image_name")
if image_name[1] == "cyberreboot/vent-"+tool+":"+branch:
try:
# currently can't use docker-py because it
# returns a 404 on pull so no way to valid if it
# worked or didn't
#image_id = self.d_client.images.pull('cyberreboot/vent-'+tool, tag=branch)
image_id = None
output = subprocess.check_output(shlex.split("docker pull cyberreboot/vent-"+tool+":"+branch), stderr=subprocess.STDOUT)
for line in output.split('\n'):
if line.startswith("Digest: sha256:"):
image_id = line.split("Digest: sha256:")[1][:12]
if image_id:
plugin_config.set_option(section, "built", "yes")
plugin_config.set_option(section, "image_id", image_id)
plugin_config.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC")
status = (True, "Pulled "+tool)
self.logger.info(str(status))
else:
plugin_config.set_option(section, "built", "failed")
plugin_config.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC")
status = (False, "Failed to pull image "+str(output.split('\n')[-1]))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
plugin_config.set_option(section, "built", "failed")
plugin_config.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC")
status = (False, "Failed to pull image "+str(e))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
status = (False, "Failed to pull images "+str(e))
self.logger.error(str(status))
plugin_config.write_config()
elif action == "start":
status = self.prep_start(groups="core", branch=branch)
if status[0]:
tool_dict = status[1]
status = self.start(tool_dict)
elif action == "stop":
status = self.stop(groups="core", branch=branch)
elif action == "clean":
status = self.clean(groups="core", branch=branch)
except Exception as e:
self.logger.info("core failed with error: "+str(e))
status = (False, e)
self.logger.info("Status of core: "+str(status))
self.logger.info("Finished: core")
return status
@staticmethod
def backup():
# TODO
return
@staticmethod
def restore():
# TODO
return
@staticmethod
def configure():
# TODO
# tools, core, etc.
return
@staticmethod
def system_commands():
# reset, upgrade, etc.
return
def logs(self, container_type=None, grep_list=None):
""" generically filter logs stored in log containers """
self.logger.info("Starting: logs")
status = (True, None)
log_entries = {}
try:
containers = self.d_client.containers.list(all=True, filters={'label':'vent'})
self.logger.info("containers found: "+str(containers))
if grep_list:
compare_containers = containers
if container_type:
try:
compare_containers = [c for c in containers if (container_type in c.attrs['Config']['Labels']['vent.groups'])]
except Exception as e: # pragma: no cover
self.logger.error("Unable to limit containers by container_type: "+str(container_type)+" because: "+str(e))
for expression in grep_list:
for container in compare_containers:
try:
# 'logs' stores each line which contains the expression
logs = [log for log in container.logs().split("\n") if expression in log]
for log in logs:
if str(container.name) in log_entries:
log_entries[str(container.name)].append(log)
else:
log_entries[str(container.name)] = [log]
except Exception as e: # pragma: no cover
self.logger.error("Unable to get logs for "+str(container.name)+" because: "+str(e))
else:
compare_containers = containers
if container_type:
try:
compare_containers = [c for c in containers if (container_type in c.attrs['Config']['Labels']['vent.groups'])]
except Exception as e: # pragma: no cover
self.logger.error("Unable to limit containers by container_type: "+str(container_type)+" because: "+str(e))
for container in compare_containers:
try:
logs = container.logs().split("\n")
for log in logs:
if str(container.name) in log_entries:
log_entries[str(container.name)].append(log)
else:
log_entries[str(container.name)] = [log]
except Exception as e: # pragma: no cover
self.logger.error("Unable to get logs for "+str(container.name)+" because: "+str(e))
status = (True, log_entries)
except Exception as e:
self.logger.error("logs failed with error: "+str(e))
status = (False, e)
self.logger.info("Status of logs: "+str(status))
self.logger.info("Finished: logs")
return status
@staticmethod
def help():
# TODO
return
def inventory(self, choices=None):
""" Return a dictionary of the inventory items and status """
self.logger.info("Starting: inventory")
status = (True, None)
self.logger.info("choices specified: "+str(choices))
try:
# choices: repos, core, tools, images, built, running, enabled
items = {'repos':[], 'core':[], 'tools':[], 'images':[],
'built':[], 'running':[], 'enabled':[]}
tools = self.plugin.tools()
self.logger.info("found tools: "+str(tools))
for choice in choices:
for tool in tools:
try:
if choice == 'repos':
if 'repo' in tool:
if tool['repo'] and tool['repo'] not in items[choice]:
items[choice].append(tool['repo'])
elif choice == 'core':
if 'groups' in tool:
if 'core' in tool['groups']:
items[choice].append((tool['section'], tool['name']))
elif choice == 'tools':
items[choice].append((tool['section'], tool['name']))
elif choice == 'images':
# TODO also check against docker
images = Images()
items[choice].append((tool['section'], tool['name'], tool['image_name']))
elif choice == 'built':
items[choice].append((tool['section'], tool['name'], tool['built']))
elif choice == 'running':
containers = Containers()
status = 'not running'
for container in containers:
image_name = tool['image_name'].rsplit(":"+tool['version'], 1)[0]
image_name = image_name.replace(':', '-')
image_name = image_name.replace('/', '-')
if container[0] == image_name:
status = container[1]
items[choice].append((tool['section'], tool['name'], status))
elif choice == 'enabled':
items[choice].append((tool['section'], tool['name'], tool['enabled']))
else:
# unknown choice
pass
except Exception as e: # pragma: no cover
self.logger.error("unable to grab information about tool: "+str(tool)+" because: "+str(e))
pass
status = (True, items)
except Exception as e:
self.logger.error("inventory failed with error: "+str(e))
status = (False, e)
self.logger.info("Status of inventory: "+str(status))
self.logger.info("Finished: inventory")
return status
def test_state():
""" Test the state function """
instance = Plugin()
states = instance.state('elasticsearch', branch='master')
assert states == [('cyberreboot:vent:/vent/core/elasticsearch:master:HEAD', 'enabled')]
def __init__(self, **kargs):
self.plugin = Plugin(**kargs)
self.d_client = self.plugin.d_client
self.vent_config = os.path.join(self.plugin.path_dirs.meta_dir,
"vent.cfg")
self.logger = Logger(__name__)
def test_tools():
""" Test the tools function """
instance = Plugin()
tools = instance.tools()
class Action:
""" Handle actions in menu """
def __init__(self, **kargs):
self.plugin = Plugin(**kargs)
self.d_client = self.plugin.d_client
self.vent_config = os.path.join(self.plugin.path_dirs.meta_dir,
"vent.cfg")
self.logger = Logger(__name__)
def add(self,
repo,
tools=None,
overrides=None,
version="HEAD",
branch="master",
build=True,
user=None,
pw=None,
groups=None,
version_alias=None,
wild=None,
remove_old=True,
disable_old=True):
""" Add a new set of tool(s) """
self.logger.info("Starting: add")
status = (True, None)
try:
status = self.plugin.add(repo,
tools=tools,
overrides=overrides,
version=version,
branch=branch,
build=build,
user=user,
pw=pw,
groups=groups,
version_alias=version_alias,
wild=wild,
remove_old=remove_old,
disable_old=disable_old)
except Exception as e: # pragma: no cover
self.logger.error("add failed with error: " + str(e))
status = (False, e)
self.logger.info("Status of add: " + str(status))
self.logger.info("Finished: add")
return status
def remove(self,
repo=None,
namespace=None,
name=None,
groups=None,
enabled="yes",
branch="master",
version="HEAD",
built="yes"):
""" Remove tools or a repo """
self.logger.info("Starting: remove")
status = (True, None)
try:
status = self.plugin.remove(name=name,
repo=repo,
namespace=namespace,
groups=groups,
enabled=enabled,
branch=branch,
version=version,
built=built)
except Exception as e:
self.logger.error("remove failed with error: " + str(e))
status = (False, e)
self.logger.info("Status of remove: " + str(status))
self.logger.info("Finished: remove")
return status
def prep_start(self,
repo=None,
name=None,
groups=None,
enabled="yes",
branch="master",
version="HEAD",
run_build=False):
"""
Start a set of tools that match the parameters given, if no parameters
are given, start all installed tools on the master branch at verison
HEAD that are enabled
"""
args = locals()
self.logger.info("Starting: prep_start")
self.logger.info("Arguments: " + str(args))
status = (True, None)
tool_dict = {}
try:
del args['run_build']
options = [
'name', 'namespace', 'built', 'groups', 'path', 'image_name',
'branch', 'version'
]
vent_config = Template(template=self.vent_config)
files = vent_config.option('main', 'files')
sections, template = self.plugin.constraint_options(args, options)
for section in sections:
# initialize needed vars
template_path = os.path.join(sections[section]['path'],
'vent.template')
container_name = sections[section]['image_name'].replace(
':', '-')
container_name = container_name.replace('/', '-')
image_name = sections[section]['image_name']
# checkout the right version and branch of the repo
self.plugin.branch = branch
self.plugin.version = version
cwd = os.getcwd()
self.logger.info("current directory is: " + str(cwd))
os.chdir(os.path.join(sections[section]['path']))
status = self.plugin.checkout()
self.logger.info(status)
os.chdir(cwd)
if run_build:
status = self.build(name=sections[section]['name'],
groups=groups,
enabled=enabled,
branch=branch,
version=version)
self.logger.info(status)
# set docker settings for container
vent_template = Template(template_path)
status = vent_template.section('docker')
self.logger.info(status)
tool_dict[container_name] = {
'image': image_name,
'name': container_name
}
if status[0]:
for option in status[1]:
options = option[1]
# check for commands to evaluate
if '`' in options:
cmds = options.split('`')
# TODO this probably needs better error checking to handle mismatched ``
if len(cmds) > 2:
i = 1
while i < len(cmds):
try:
cmds[i] = subprocess.check_output(
shlex.split(cmds[i]),
stderr=subprocess.STDOUT,
close_fds=True).strip()
except Exception as e: # pragma: no cover
self.logger.error(
"unable to evaluate command specified in vent.template: "
+ str(e))
i += 2
options = "".join(cmds)
# store options set for docker
try:
tool_dict[container_name][
option[0]] = ast.literal_eval(options)
except Exception as e: # pragma: no cover
self.logger.error(
"unable to store the options set for docker: "
+ str(e))
tool_dict[container_name][option[0]] = options
# get temporary name for links, etc.
status = vent_template.section('info')
self.logger.info(status)
plugin_config = Template(template=self.plugin.manifest)
status, plugin_sections = plugin_config.sections()
self.logger.info(status)
for plugin_section in plugin_sections:
status = plugin_config.option(plugin_section, "link_name")
self.logger.info(status)
image_status = plugin_config.option(
plugin_section, "image_name")
self.logger.info(image_status)
if status[0] and image_status[0]:
cont_name = image_status[1].replace(':', '-')
cont_name = cont_name.replace('/', '-')
if cont_name not in tool_dict:
tool_dict[cont_name] = {
'image': image_status[1],
'name': cont_name,
'start': False
}
tool_dict[cont_name]['tmp_name'] = status[1]
# add extra labels
if 'labels' not in tool_dict[container_name]:
tool_dict[container_name]['labels'] = {}
tool_dict[container_name]['labels']['vent'] = Version()
tool_dict[container_name]['labels'][
'vent.namespace'] = sections[section]['namespace']
tool_dict[container_name]['labels']['vent.branch'] = branch
tool_dict[container_name]['labels']['vent.version'] = version
tool_dict[container_name]['labels']['vent.name'] = sections[
section]['name']
if 'groups' in sections[section]:
# add labels for groups
tool_dict[container_name]['labels'][
'vent.groups'] = sections[section]['groups']
# send logs to syslog
if 'syslog' not in sections[section][
'groups'] and 'core' in sections[section]['groups']:
tool_dict[container_name]['log_config'] = {
'type': 'syslog',
'config': {
'syslog-address': 'tcp://0.0.0.0:514',
'syslog-facility': 'daemon',
'tag': 'core'
}
}
if 'syslog' not in sections[section]['groups']:
tool_dict[container_name]['log_config'] = {
'type': 'syslog',
'config': {
'syslog-address': 'tcp://0.0.0.0:514',
'syslog-facility': 'daemon',
'tag': 'plugin'
}
}
# mount necessary directories
if 'files' in sections[section]['groups']:
if 'volumes' in tool_dict[container_name]:
tool_dict[container_name]['volumes'][
self.plugin.path_dirs.base_dir[:-1]] = {
'bind': '/vent',
'mode': 'ro'
}
else:
tool_dict[container_name]['volumes'] = {
self.plugin.path_dirs.base_dir[:-1]: {
'bind': '/vent',
'mode': 'ro'
}
}
if files[0]:
tool_dict[container_name]['volumes'][files[1]] = {
'bind': '/files',
'mode': 'ro'
}
else:
tool_dict[container_name]['log_config'] = {
'type': 'syslog',
'config': {
'syslog-address': 'tcp://0.0.0.0:514',
'syslog-facility': 'daemon',
'tag': 'plugin'
}
}
# add label for priority
status = vent_template.section('settings')
self.logger.info(status)
if status[0]:
for option in status[1]:
if option[0] == 'priority':
tool_dict[container_name]['labels'][
'vent.priority'] = option[1]
# only start tools that have been built
if sections[section]['built'] != 'yes':
del tool_dict[container_name]
# check and update links, volumes_from, network_mode
for container in tool_dict.keys():
if 'links' in tool_dict[container]:
for link in tool_dict[container]['links']:
for c in tool_dict.keys():
if 'tmp_name' in tool_dict[c] and tool_dict[c][
'tmp_name'] == link:
tool_dict[container]['links'][
tool_dict[c]['name']] = tool_dict[
container]['links'].pop(link)
if 'volumes_from' in tool_dict[container]:
tmp_volumes_from = tool_dict[container]['volumes_from']
tool_dict[container]['volumes_from'] = []
for volumes_from in list(tmp_volumes_from):
for c in tool_dict.keys():
if 'tmp_name' in tool_dict[c] and tool_dict[c][
'tmp_name'] == volumes_from:
tool_dict[container]['volumes_from'].append(
tool_dict[c]['name'])
tmp_volumes_from.remove(volumes_from)
tool_dict[container]['volumes_from'] += tmp_volumes_from
if 'network_mode' in tool_dict[container]:
if tool_dict[container]['network_mode'].startswith(
'container:'):
network_c_name = tool_dict[container][
'network_mode'].split('container:')[1]
for c in tool_dict.keys():
if 'tmp_name' in tool_dict[c] and tool_dict[c][
'tmp_name'] == network_c_name:
tool_dict[container][
'network_mode'] = 'container:' + tool_dict[
c]['name']
# remove tmp_names
for c in tool_dict.keys():
if 'tmp_name' in tool_dict[c]:
del tool_dict[c]['tmp_name']
# remove containers that shouldn't be started
for c in tool_dict.keys():
if 'start' in tool_dict[c] and not tool_dict[c]['start']:
del tool_dict[c]
except Exception as e:
self.logger.error("prep_start failed with error: " + str(e))
status = (False, e)
status = (True, tool_dict)
self.logger.info("Status of prep_start: " + str(status))
self.logger.info("Finished: prep_start")
return status
def start(self, tool_dict):
"""
Start a set of tools that match the parameters given, if no parameters
are given, start all installed tools on the master branch at verison
HEAD that are enabled
"""
self.logger.info("Starting: start")
status = (True, None)
try:
# check start priorities (priority of groups is alphabetical for now)
group_orders = {}
groups = []
containers_remaining = []
for container in tool_dict:
containers_remaining.append(container)
if 'labels' in tool_dict[container]:
if 'vent.groups' in tool_dict[container]['labels']:
groups += tool_dict[container]['labels'][
'vent.groups'].split(',')
if 'vent.priority' in tool_dict[container]['labels']:
priorities = tool_dict[container]['labels'][
'vent.priority'].split(',')
container_groups = tool_dict[container]['labels'][
'vent.groups'].split(',')
for i, priority in enumerate(priorities):
if container_groups[i] not in group_orders:
group_orders[container_groups[i]] = []
group_orders[container_groups[i]].append(
(int(priority), container))
containers_remaining.remove(container)
# start containers based on priorities
groups = sorted(set(groups))
started_containers = []
for group in groups:
if group in group_orders:
for container_tuple in sorted(group_orders[group]):
if container_tuple[1] not in started_containers:
started_containers.append(container_tuple[1])
try:
try:
container = self.d_client.containers.get(
container_tuple[1])
container.start()
self.logger.info("started " +
str(container_tuple[1]) +
" with ID: " +
str(container.short_id))
except Exception as err: # pragma: no cover
self.logger.error(str(err))
container_id = self.d_client.containers.run(
detach=True,
**tool_dict[container_tuple[1]])
self.logger.info("started " +
str(container_tuple[1]) +
" with ID: " +
str(container_id))
except Exception as e: # pragma: no cover
self.logger.error("failed to start " +
str(container_tuple[1]) +
" because: " + str(e))
# start the rest of the containers that didn't have any priorities set
for container in containers_remaining:
try:
try:
c = self.d_client.containers.get(container)
c.start()
self.logger.info("started " + str(container) +
" with ID: " + str(c.short_id))
except Exception as err: # pragma: no cover
self.logger.error(str(err))
container_id = self.d_client.containers.run(
detach=True, **tool_dict[container])
self.logger.info("started " + str(container) +
" with ID: " + str(container_id))
except Exception as e: # pragma: no cover
self.logger.error("failed to start " + str(container) +
" because: " + str(e))
except Exception as e:
self.logger.error("start failed with error: " + str(e))
status = (False, e)
self.logger.info("Status of start: " + str(status))
self.logger.info("Finished: start")
return status
def update(self,
repo=None,
name=None,
groups=None,
enabled="yes",
branch="master",
version="HEAD"):
"""
Update a set of tools that match the parameters given, if no parameters
are given, updated all installed tools on the master branch at verison
HEAD that are enabled
"""
args = locals()
self.logger.info("Starting: update")
self.logger.info(args)
status = (True, None)
try:
options = ['path', 'image_name', 'image_id']
sections, template = self.plugin.constraint_options(args, options)
# get existing containers and images and states
running_containers = Containers()
built_images = Images()
self.logger.info("running docker containers: " +
str(running_containers))
self.logger.info("built docker images: " + str(built_images))
# if repo, pull and build
# if registry image, pull
for section in sections:
try:
cwd = os.getcwd()
self.logger.info("current working directory: " + str(cwd))
os.chdir(sections[section]['path'])
self.plugin.version = version
self.plugin.branch = branch
c_status = self.plugin.checkout()
self.logger.info(c_status)
try:
os.chdir(cwd)
except Exception as e: # pragma: no cover
self.logger.error("unable to change directory: " +
str(e))
pass
template = self.plugin.builder(
template,
sections[section]['path'],
sections[section]['image_name'],
section,
build=True,
branch=branch,
version=version)
self.logger.info(template)
# stop and remove old containers and images if image_id updated
# !! TODO
# start containers if they were running
# !! TODO
# TODO logging
except Exception as e: # pragma: no cover
self.logger.error("unable to update: " + str(section) +
" because: " + str(e))
template.write_config()
except Exception as e:
self.logger.error("update failed with error: " + str(e))
status = (False, e)
self.logger.info("Status of update: " + str(status))
self.logger.info("Finished: update")
return status
def stop(self,
repo=None,
name=None,
groups=None,
enabled="yes",
branch="master",
version="HEAD"):
"""
Stop a set of tools that match the parameters given, if no parameters
are given, stop all installed tools on the master branch at verison
HEAD that are enabled
"""
args = locals()
self.logger.info("Starting: stop")
self.logger.info(args)
status = (True, None)
try:
# !! TODO need to account for plugin containers that have random names, use labels perhaps
options = [
'name', 'namespace', 'built', 'groups', 'path', 'image_name',
'branch', 'version'
]
sections, template = self.plugin.constraint_options(args, options)
self.logger.info(sections)
for section in sections:
container_name = sections[section]['image_name'].replace(
':', '-')
container_name = container_name.replace('/', '-')
try:
container = self.d_client.containers.get(container_name)
container.stop()
self.logger.info("stopped " + str(container_name))
except Exception as e: # pragma: no cover
self.logger.error("failed to stop " + str(container_name) +
" because: " + str(e))
except Exception as e:
self.logger.error("stop failed with error: " + str(e))
status = (False, e)
self.logger.info("Status of stop: " + str(status))
self.logger.info("Finished: stop")
return status
def clean(self,
repo=None,
name=None,
groups=None,
enabled="yes",
branch="master",
version="HEAD"):
"""
Clean (stop and remove) a set of tools that match the parameters given,
if no parameters are given, clean all installed tools on the master
branch at verison HEAD that are enabled
"""
args = locals()
self.logger.info("Starting: clean")
self.logger.info(args)
status = (True, None)
try:
# !! TODO need to account for plugin containers that have random names, use labels perhaps
options = [
'name', 'namespace', 'built', 'groups', 'path', 'image_name',
'branch', 'version'
]
sections, template = self.plugin.constraint_options(args, options)
self.logger.info(sections)
for section in sections:
container_name = sections[section]['image_name'].replace(
':', '-')
container_name = container_name.replace('/', '-')
try:
container = self.d_client.containers.get(container_name)
container.remove(force=True)
self.logger.info("cleaned " + str(container_name))
except Exception as e: # pragma: no cover
self.logger.error("failed to clean " +
str(container_name) + " because: " +
str(e))
except Exception as e:
self.logger.error("clean failed with error: " + str(e))
status = (False, e)
self.logger.info("Status of clean: " + str(status))
self.logger.info("Finished: clean")
return status
def build(self,
repo=None,
name=None,
groups=None,
enabled="yes",
branch="master",
version="HEAD"):
""" Build a set of tools that match the parameters given """
args = locals()
self.logger.info("Starting: build")
self.logger.info(args)
status = (True, None)
try:
options = ['image_name', 'path']
sections, template = self.plugin.constraint_options(args, options)
self.logger.info(sections)
for section in sections:
self.logger.info("Building " + str(section) + " ...")
template = self.plugin.builder(template,
sections[section]['path'],
sections[section]['image_name'],
section,
build=True,
branch=branch,
version=version)
template.write_config()
except Exception as e:
self.logger.error("build failed with error: " + str(e))
status = (False, e)
self.logger.info("Status of build: " + str(status))
self.logger.info("Finished: build")
return status
def cores(self, action, branch="master"):
""" Supply action (install, build, start, stop, clean) for core tools """
self.logger.info("Starting: cores")
status = (True, None)
try:
self.logger.info("action provided: " + str(action))
core = Core(branch=branch)
if action in ["install", "build"]:
tools = []
plugins = Plugin(plugins_dir=".internals/plugins")
plugins.version = 'HEAD'
plugins.branch = branch
plugins.apply_path('https://github.com/cyberreboot/vent')
response = plugins.checkout()
self.logger.info("status of plugin checkout " + str(response))
matches = plugins._available_tools(groups='core')
for match in matches:
tools.append((match[0], ''))
status = plugins.add('https://github.com/cyberreboot/vent',
tools=tools,
branch=branch,
build=False)
self.logger.info("status of plugin add: " + str(status))
plugin_config = Template(template=self.plugin.manifest)
sections = plugin_config.sections()
for tool in core['normal']:
for section in sections[1]:
name = plugin_config.option(section, "name")
orig_branch = plugin_config.option(section, "branch")
namespace = plugin_config.option(section, "namespace")
version = plugin_config.option(section, "version")
if name[1] == tool and orig_branch[
1] == branch and namespace[
1] == "cyberreboot/vent" and version[
1] == "HEAD":
plugin_config.set_option(
section, "image_name",
"cyberreboot/vent-" + tool + ":" + branch)
plugin_config.write_config()
if action == "build":
plugin_config = Template(template=self.plugin.manifest)
sections = plugin_config.sections()
try:
for tool in core['normal']:
for section in sections[1]:
image_name = plugin_config.option(
section, "image_name")
if image_name[
1] == "cyberreboot/vent-" + tool + ":" + branch:
try:
# currently can't use docker-py because it
# returns a 404 on pull so no way to valid if it
# worked or didn't
#image_id = self.d_client.images.pull('cyberreboot/vent-'+tool, tag=branch)
image_id = None
output = subprocess.check_output(
shlex.split(
"docker pull cyberreboot/vent-" +
tool + ":" + branch),
stderr=subprocess.STDOUT)
for line in output.split('\n'):
if line.startswith("Digest: sha256:"):
image_id = line.split(
"Digest: sha256:")[1][:12]
if image_id:
plugin_config.set_option(
section, "built", "yes")
plugin_config.set_option(
section, "image_id", image_id)
plugin_config.set_option(
section, "last_updated",
str(datetime.datetime.utcnow()) +
" UTC")
status = (True, "Pulled " + tool)
self.logger.info(str(status))
else:
plugin_config.set_option(
section, "built", "failed")
plugin_config.set_option(
section, "last_updated",
str(datetime.datetime.utcnow()) +
" UTC")
status = (False,
"Failed to pull image " +
str(output.split('\n')[-1]))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
plugin_config.set_option(
section, "built", "failed")
plugin_config.set_option(
section, "last_updated",
str(datetime.datetime.utcnow()) +
" UTC")
status = (False,
"Failed to pull image " + str(e))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
status = (False, "Failed to pull images " + str(e))
self.logger.error(str(status))
plugin_config.write_config()
elif action == "start":
status = self.prep_start(groups="core", branch=branch)
if status[0]:
tool_dict = status[1]
status = self.start(tool_dict)
elif action == "stop":
status = self.stop(groups="core", branch=branch)
elif action == "clean":
status = self.clean(groups="core", branch=branch)
except Exception as e:
self.logger.info("core failed with error: " + str(e))
status = (False, e)
self.logger.info("Status of core: " + str(status))
self.logger.info("Finished: core")
return status
@staticmethod
def backup():
# TODO
return
@staticmethod
def restore():
# TODO
return
@staticmethod
def configure():
# TODO
# tools, core, etc.
return
@staticmethod
def system_commands():
# reset, upgrade, etc.
return
def logs(self, container_type=None, grep_list=None):
""" generically filter logs stored in log containers """
self.logger.info("Starting: logs")
status = (True, None)
log_entries = {}
try:
containers = self.d_client.containers.list(
all=True, filters={'label': 'vent'})
self.logger.info("containers found: " + str(containers))
if grep_list:
compare_containers = containers
if container_type:
try:
compare_containers = [
c for c in containers
if (container_type in c.attrs['Config']['Labels']
['vent.groups'])
]
except Exception as e: # pragma: no cover
self.logger.error(
"Unable to limit containers by container_type: " +
str(container_type) + " because: " + str(e))
for expression in grep_list:
for container in compare_containers:
try:
# 'logs' stores each line which contains the expression
logs = [
log for log in container.logs().split("\n")
if expression in log
]
for log in logs:
if str(container.name) in log_entries:
log_entries[str(
container.name)].append(log)
else:
log_entries[str(container.name)] = [log]
except Exception as e: # pragma: no cover
self.logger.error("Unable to get logs for " +
str(container.name) +
" because: " + str(e))
else:
compare_containers = containers
if container_type:
try:
compare_containers = [
c for c in containers
if (container_type in c.attrs['Config']['Labels']
['vent.groups'])
]
except Exception as e: # pragma: no cover
self.logger.error(
"Unable to limit containers by container_type: " +
str(container_type) + " because: " + str(e))
for container in compare_containers:
try:
logs = container.logs().split("\n")
for log in logs:
if str(container.name) in log_entries:
log_entries[str(container.name)].append(log)
else:
log_entries[str(container.name)] = [log]
except Exception as e: # pragma: no cover
self.logger.error("Unable to get logs for " +
str(container.name) + " because: " +
str(e))
status = (True, log_entries)
except Exception as e:
self.logger.error("logs failed with error: " + str(e))
status = (False, e)
self.logger.info("Status of logs: " + str(status))
self.logger.info("Finished: logs")
return status
@staticmethod
def help():
# TODO
return
def inventory(self, choices=None):
""" Return a dictionary of the inventory items and status """
self.logger.info("Starting: inventory")
status = (True, None)
self.logger.info("choices specified: " + str(choices))
try:
# choices: repos, core, tools, images, built, running, enabled
items = {
'repos': [],
'core': [],
'tools': [],
'images': [],
'built': [],
'running': [],
'enabled': []
}
tools = self.plugin.tools()
self.logger.info("found tools: " + str(tools))
for choice in choices:
for tool in tools:
try:
if choice == 'repos':
if 'repo' in tool:
if tool['repo'] and tool['repo'] not in items[
choice]:
items[choice].append(tool['repo'])
elif choice == 'core':
if 'groups' in tool:
if 'core' in tool['groups']:
items[choice].append(
(tool['section'], tool['name']))
elif choice == 'tools':
items[choice].append(
(tool['section'], tool['name']))
elif choice == 'images':
# TODO also check against docker
images = Images()
items[choice].append(
(tool['section'], tool['name'],
tool['image_name']))
elif choice == 'built':
items[choice].append(
(tool['section'], tool['name'], tool['built']))
elif choice == 'running':
containers = Containers()
status = 'not running'
for container in containers:
image_name = tool['image_name'].rsplit(
":" + tool['version'], 1)[0]
image_name = image_name.replace(':', '-')
image_name = image_name.replace('/', '-')
if container[0] == image_name:
status = container[1]
items[choice].append(
(tool['section'], tool['name'], status))
elif choice == 'enabled':
items[choice].append(
(tool['section'], tool['name'],
tool['enabled']))
else:
# unknown choice
pass
except Exception as e: # pragma: no cover
self.logger.error(
"unable to grab information about tool: " +
str(tool) + " because: " + str(e))
pass
status = (True, items)
except Exception as e:
self.logger.error("inventory failed with error: " + str(e))
status = (False, e)
self.logger.info("Status of inventory: " + str(status))
self.logger.info("Finished: inventory")
return status
def __init__(self, **kargs):
self.plugin = Plugin(**kargs)
self.d_client = self.plugin.d_client
self.vent_config = os.path.join(self.plugin.path_dirs.meta_dir,
"vent.cfg")
self.logger = Logger(__name__)
def test_add():
""" Test the add function """
instance = Plugin()
status = instance.add('https://github.com/cyberreboot/vent', build=False)
assert isinstance(status, tuple)
assert status[0] == True
status = instance.add('https://github.com/cyberreboot/vent.git',
build=False)
assert isinstance(status, tuple)
assert status[0] == True
bad_instance = Plugin()
status = bad_instance.add('https://github.com/cyberreboot/vent',
build=False)
assert isinstance(status, tuple)
assert status[0] == True
instance = Plugin()
status = instance.add('https://github.com/cyberreboot/vent',
build=False,
user='******',
pw='bar')
assert isinstance(status, tuple)
assert status[0] == True
instance = Plugin()
status = instance.add('https://github.com/cyberreboot/vent',
build=False,
overrides=[('.', 'HEAD')])
assert isinstance(status, tuple)
assert status[0] == True
instance = Plugin()
status = instance.add('https://github.com/cyberreboot/vent',
build=False,
tools=[('vent/', 'HEAD')],
overrides=[('vent', 'HEAD')])
assert isinstance(status, tuple)
assert status[0] == True
instance = Plugin()
status = instance.add('https://github.com/cyberreboot/vent',
build=False,
overrides=[('.', 'HEAD')],
user='******',
pw='foo')
assert isinstance(status, tuple)
assert status[0] == True
def cores(self, action, branch="master"):
""" Supply action (install, build, start, stop, clean) for core tools """
self.logger.info("Starting: cores")
status = (True, None)
try:
self.logger.info("action provided: " + str(action))
core = Core(branch=branch)
if action in ["install", "build"]:
tools = []
plugins = Plugin(plugins_dir=".internals/plugins")
plugins.version = 'HEAD'
plugins.branch = branch
plugins.apply_path('https://github.com/cyberreboot/vent')
response = plugins.checkout()
self.logger.info("status of plugin checkout " + str(response))
matches = plugins._available_tools(groups='core')
for match in matches:
tools.append((match[0], ''))
status = plugins.add('https://github.com/cyberreboot/vent',
tools=tools,
branch=branch,
build=False)
self.logger.info("status of plugin add: " + str(status))
plugin_config = Template(template=self.plugin.manifest)
sections = plugin_config.sections()
for tool in core['normal']:
for section in sections[1]:
name = plugin_config.option(section, "name")
orig_branch = plugin_config.option(section, "branch")
namespace = plugin_config.option(section, "namespace")
version = plugin_config.option(section, "version")
if name[1] == tool and orig_branch[
1] == branch and namespace[
1] == "cyberreboot/vent" and version[
1] == "HEAD":
plugin_config.set_option(
section, "image_name",
"cyberreboot/vent-" + tool + ":" + branch)
plugin_config.write_config()
if action == "build":
plugin_config = Template(template=self.plugin.manifest)
sections = plugin_config.sections()
try:
for tool in core['normal']:
for section in sections[1]:
image_name = plugin_config.option(
section, "image_name")
if image_name[
1] == "cyberreboot/vent-" + tool + ":" + branch:
try:
# currently can't use docker-py because it
# returns a 404 on pull so no way to valid if it
# worked or didn't
#image_id = self.d_client.images.pull('cyberreboot/vent-'+tool, tag=branch)
image_id = None
output = subprocess.check_output(
shlex.split(
"docker pull cyberreboot/vent-" +
tool + ":" + branch),
stderr=subprocess.STDOUT)
for line in output.split('\n'):
if line.startswith("Digest: sha256:"):
image_id = line.split(
"Digest: sha256:")[1][:12]
if image_id:
plugin_config.set_option(
section, "built", "yes")
plugin_config.set_option(
section, "image_id", image_id)
plugin_config.set_option(
section, "last_updated",
str(datetime.datetime.utcnow()) +
" UTC")
status = (True, "Pulled " + tool)
self.logger.info(str(status))
else:
plugin_config.set_option(
section, "built", "failed")
plugin_config.set_option(
section, "last_updated",
str(datetime.datetime.utcnow()) +
" UTC")
status = (False,
"Failed to pull image " +
str(output.split('\n')[-1]))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
plugin_config.set_option(
section, "built", "failed")
plugin_config.set_option(
section, "last_updated",
str(datetime.datetime.utcnow()) +
" UTC")
status = (False,
"Failed to pull image " + str(e))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
status = (False, "Failed to pull images " + str(e))
self.logger.error(str(status))
plugin_config.write_config()
elif action == "start":
status = self.prep_start(groups="core", branch=branch)
if status[0]:
tool_dict = status[1]
status = self.start(tool_dict)
elif action == "stop":
status = self.stop(groups="core", branch=branch)
elif action == "clean":
status = self.clean(groups="core", branch=branch)
except Exception as e:
self.logger.info("core failed with error: " + str(e))
status = (False, e)
self.logger.info("Status of core: " + str(status))
self.logger.info("Finished: core")
return status
def test_update():
""" Test the update function """
instance = Plugin()
status = instance.update()
assert isinstance(status, tuple)
assert status[0] == False
def test_disable():
""" Test the disable function """
instance = Plugin()
status = instance.disable('elasticsearch', branch='master')
assert status[0] == True
def test_state():
""" Test the state function """
instance = Plugin()
states = instance.state('elasticsearch', branch='master')
assert states == [('cyberreboot:vent:/vent/core/elasticsearch:master:HEAD',
'enabled')]
def test_build_tools():
""" Test the _build_tools function """
instance = Plugin()
status = instance._build_tools(False)
assert status[0] == False
def test_add():
""" Test the add function """
instance = Plugin()
status = instance.add('https://github.com/cyberreboot/vent', build=False)
assert type(status) == tuple
assert status[0] == True
status = instance.add('https://github.com/cyberreboot/vent.git', build=False)
assert type(status) == tuple
assert status[0] == True
bad_instance = Plugin()
status = bad_instance.add('https://github.com/cyberreboot/vent', build=False)
assert type(status) == tuple
assert status[0] == True
instance = Plugin()
status = instance.add('https://github.com/cyberreboot/vent', build=False, user='******', pw='bar')
assert type(status) == tuple
assert status[0] == True
instance = Plugin()
status = instance.add('https://github.com/cyberreboot/vent', build=False, overrides=[('.', 'HEAD')])
assert type(status) == tuple
assert status[0] == True
instance = Plugin()
status = instance.add('https://github.com/cyberreboot/vent', build=False, tools=[('vent/', 'HEAD')], overrides=[('vent', 'HEAD')])
assert type(status) == tuple
assert status[0] == True
def test_list_tools():
""" Test the tools function """
instance = Plugin()
tools = instance.list_tools()
def test_remove():
""" Test the remove function """
instance = Plugin()
status = instance.remove()
assert status[0] == True
def test_remove():
""" Test the remove function """
instance = Plugin()
status = instance.remove(groups='core', built='no')
assert status[0] == True
def test_current_version():
""" Test the current_version function """
instance = Plugin()
versions = instance.current_version('elasticsearch', branch='master')
assert versions == [('cyberreboot:vent:/vent/core/elasticsearch:master:HEAD', 'HEAD')]
def cores(self, action, branch="master"):
""" Supply action (install, build, start, stop, clean) for core tools """
self.logger.info("Starting: cores")
status = (True, None)
try:
self.logger.info("action provided: "+str(action))
core = Core(branch=branch)
if action in ["install", "build"]:
tools = []
plugins = Plugin(plugins_dir=".internals/plugins")
plugins.version = 'HEAD'
plugins.branch = branch
plugins.apply_path('https://github.com/cyberreboot/vent')
response = plugins.checkout()
self.logger.info("status of plugin checkout "+str(response))
matches = plugins._available_tools(groups='core')
for match in matches:
tools.append((match[0], ''))
status = plugins.add('https://github.com/cyberreboot/vent', tools=tools, branch=branch, build=False)
self.logger.info("status of plugin add: "+str(status))
plugin_config = Template(template=self.plugin.manifest)
sections = plugin_config.sections()
for tool in core['normal']:
for section in sections[1]:
name = plugin_config.option(section, "name")
orig_branch = plugin_config.option(section, "branch")
namespace = plugin_config.option(section, "namespace")
version = plugin_config.option(section, "version")
if name[1] == tool and orig_branch[1] == branch and namespace[1] == "cyberreboot/vent" and version[1] == "HEAD":
plugin_config.set_option(section, "image_name", "cyberreboot/vent-"+tool+":"+branch)
plugin_config.write_config()
if action == "build":
plugin_config = Template(template=self.plugin.manifest)
sections = plugin_config.sections()
try:
for tool in core['normal']:
for section in sections[1]:
image_name = plugin_config.option(section, "image_name")
if image_name[1] == "cyberreboot/vent-"+tool+":"+branch:
try:
# currently can't use docker-py because it
# returns a 404 on pull so no way to valid if it
# worked or didn't
#image_id = self.d_client.images.pull('cyberreboot/vent-'+tool, tag=branch)
image_id = None
output = subprocess.check_output(shlex.split("docker pull cyberreboot/vent-"+tool+":"+branch), stderr=subprocess.STDOUT)
for line in output.split('\n'):
if line.startswith("Digest: sha256:"):
image_id = line.split("Digest: sha256:")[1][:12]
if image_id:
plugin_config.set_option(section, "built", "yes")
plugin_config.set_option(section, "image_id", image_id)
plugin_config.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC")
status = (True, "Pulled "+tool)
self.logger.info(str(status))
else:
plugin_config.set_option(section, "built", "failed")
plugin_config.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC")
status = (False, "Failed to pull image "+str(output.split('\n')[-1]))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
plugin_config.set_option(section, "built", "failed")
plugin_config.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC")
status = (False, "Failed to pull image "+str(e))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
status = (False, "Failed to pull images "+str(e))
self.logger.error(str(status))
plugin_config.write_config()
elif action == "start":
status = self.prep_start(groups="core", branch=branch)
if status[0]:
tool_dict = status[1]
status = self.start(tool_dict)
elif action == "stop":
status = self.stop(groups="core", branch=branch)
elif action == "clean":
status = self.clean(groups="core", branch=branch)
except Exception as e:
self.logger.info("core failed with error: "+str(e))
status = (False, e)
self.logger.info("Status of core: "+str(status))
self.logger.info("Finished: core")
return status
def test_disable():
""" Test the disable function """
instance = Plugin()
status = instance.disable('elasticsearch', branch='master')
assert status[0] == True
def test_update():
""" Test the update function """
instance = Plugin()
status = instance.update()
assert type(status) == tuple
assert status[0] == False
def onStart(self):
""" Override onStart method for npyscreen """
curses.mousemask(0)
self.paths.host_config()
version = Version()
# setup initial runtime stuff
if self.first_time[0] and self.first_time[1] != 'exists':
plugins = Plugin()
actions = Action()
thr = Thread(target=MainForm.t_status,
args=(),
kwargs={'core': True})
thr.start()
while thr.is_alive():
npyscreen.notify_wait(
'Please wait while Vent initializes...1/4',
title='Setting up things...')
time.sleep(1)
thr.join()
thr = Thread(target=MainForm.t_status,
args=(),
kwargs={'core': False})
thr.start()
while thr.is_alive():
npyscreen.notify_wait(
'Please wait while Vent initializes...2/4',
title='Setting up things...')
time.sleep(1)
thr.join()
thr = Thread(target=plugins.auto_install, args=(), kwargs={})
thr.start()
while thr.is_alive():
npyscreen.notify_wait(
'Please wait while Vent initializes...3/4',
title='Setting up things...')
time.sleep(1)
thr.join()
thr = Thread(target=actions.startup, args=(), kwargs={})
thr.start()
while thr.is_alive():
npyscreen.notify_wait(
'Please wait while Vent initializes...4/4',
title='Setting up things...')
time.sleep(1)
thr.join()
quit_s = '\t' * 4 + '^Q to quit'
tab_esc = '\t' * 4 + 'TAB to close menu popup'
self.addForm('MAIN',
MainForm,
name='Vent ' + version + '\t\t\t\t\t^T for help' +
quit_s + tab_esc,
color='IMPORTANT')
self.addForm('HELP',
HelpForm,
name='Help\t\t\t\t\t\t\t\t^T to toggle previous' + quit_s,
color='DANGER')
self.addForm('TUTORIALINTRO',
TutorialIntroForm,
name='Vent Tutorial' + quit_s,
color='DANGER')
self.addForm('TUTORIALBACKGROUND',
TutorialBackgroundForm,
name='About Vent' + quit_s,
color='DANGER')
self.addForm('TUTORIALTERMINOLOGY',
TutorialTerminologyForm,
name='About Vent' + quit_s,
color='DANGER')
self.addForm('TUTORIALGETTINGSETUP',
TutorialGettingSetupForm,
name='About Vent' + quit_s,
color='DANGER')
self.addForm('TUTORIALBUILDINGCORES',
TutorialBuildingCoresForm,
name='Working with Cores' + quit_s,
color='DANGER')
self.addForm('TUTORIALSTARTINGCORES',
TutorialStartingCoresForm,
name='Working with Cores' + quit_s,
color='DANGER')
self.addForm('TUTORIALADDINGPLUGINS',
TutorialAddingPluginsForm,
name='Working with Plugins' + quit_s,
color='DANGER')
self.addForm('TUTORIALADDINGFILES',
TutorialAddingFilesForm,
name='Files' + quit_s,
color='DANGER')
self.addForm('TUTORIALTROUBLESHOOTING',
TutorialTroubleshootingForm,
name='Troubleshooting' + quit_s,
color='DANGER')
def test_get_tool_matches():
""" Test the get_tool_matches function """
instance = Plugin()
instance.tools = []
matches = instance.get_tool_matches()
assert matches == []
def test_build_tools():
""" Test the _build_tools function """
instance = Plugin()
status = instance._build_tools(False)
assert status[0] == False
def onStart(self):
""" Override onStart method for npyscreen """
curses.mousemask(0)
self.paths.host_config()
version = Version()
# setup initial runtime stuff
if self.first_time[0] and self.first_time[1] != "exists":
plugins = Plugin()
actions = Action()
thr = Thread(target=MainForm.t_status,
args=(),
kwargs={'core': True})
thr.start()
while thr.is_alive():
npyscreen.notify_wait(
"Please wait while Vent initializes...1/4",
title="Setting up things...")
time.sleep(1)
thr.join()
thr = Thread(target=MainForm.t_status,
args=(),
kwargs={'core': False})
thr.start()
while thr.is_alive():
npyscreen.notify_wait(
"Please wait while Vent initializes...2/4",
title="Setting up things...")
time.sleep(1)
thr.join()
thr = Thread(target=plugins.auto_install, args=(), kwargs={})
thr.start()
while thr.is_alive():
npyscreen.notify_wait(
"Please wait while Vent initializes...3/4",
title="Setting up things...")
time.sleep(1)
thr.join()
thr = Thread(target=actions.startup, args=(), kwargs={})
thr.start()
while thr.is_alive():
npyscreen.notify_wait(
"Please wait while Vent initializes...4/4",
title="Setting up things...")
time.sleep(1)
thr.join()
quit_s = "\t" * 4 + "^Q to quit"
tab_esc = "\t" * 4 + "TAB to close menu popup"
self.addForm("MAIN",
MainForm,
name="Vent " + version + "\t\t\t\t\t^T for help" +
quit_s + tab_esc,
color="IMPORTANT")
self.addForm("HELP",
HelpForm,
name="Help\t\t\t\t\t\t\t\t^T to toggle previous" + quit_s,
color="DANGER")
self.addForm("TUTORIALINTRO",
TutorialIntroForm,
name="Vent Tutorial" + quit_s,
color="DANGER")
self.addForm("TUTORIALBACKGROUND",
TutorialBackgroundForm,
name="About Vent" + quit_s,
color="DANGER")
self.addForm("TUTORIALTERMINOLOGY",
TutorialTerminologyForm,
name="About Vent" + quit_s,
color="DANGER")
self.addForm("TUTORIALGETTINGSETUP",
TutorialGettingSetupForm,
name="About Vent" + quit_s,
color="DANGER")
self.addForm("TUTORIALBUILDINGCORES",
TutorialBuildingCoresForm,
name="Working with Cores" + quit_s,
color="DANGER")
self.addForm("TUTORIALSTARTINGCORES",
TutorialStartingCoresForm,
name="Working with Cores" + quit_s,
color="DANGER")
self.addForm("TUTORIALADDINGPLUGINS",
TutorialAddingPluginsForm,
name="Working with Plugins" + quit_s,
color="DANGER")
self.addForm("TUTORIALADDINGFILES",
TutorialAddingFilesForm,
name="Files" + quit_s,
color="DANGER")
self.addForm("TUTORIALTROUBLESHOOTING",
TutorialTroubleshootingForm,
name="Troubleshooting" + quit_s,
color="DANGER")