def start(self):
status = (True, None)
# startup based on startup file
if exists(self.startup_file):
status = self._startup()
else:
tools = Tools()
status = tools.new('core', None)
if status[0]:
status = tools.start(
'https://github.com/cyberreboot/vent', None)
return status
def repo_tools(self, branch):
""" Set the appropriate repo dir and get the tools available of it """
tools = []
m_helper = Tools()
repo = self.parentApp.repo_value['repo']
version = self.parentApp.repo_value['versions'][branch]
status = m_helper.repo_tools(repo, branch, version)
if status[0]:
r_tools = status[1]
for tool in r_tools:
tools.append(tool[0])
return tools
def repo_tools(self, branch):
""" Set the appropriate repo dir and get the tools available of it """
tools = []
m_helper = Tools()
repo = self.parentApp.repo_value['repo']
version = self.parentApp.repo_value['versions'][branch]
status = m_helper.repo_tools(repo, branch, version)
if status[0]:
r_tools = status[1]
for tool in r_tools:
tools.append(tool[0])
return tools
def __init__(self, action_dict=None, action_name=None, *args, **keywords):
api_action = Tools()
action = {'api_action': api_action}
if action_dict:
action.update(action_dict)
logger = Logger(action_name)
InventoryForm.__init__(self, action, logger, *args, **keywords)
def start(self):
status = (True, None)
vent_bridge = None
# create vent network bridge if it doesn't already exist
try:
vent_bridge = self.d_client.networks.create(
'vent', check_duplicate=True, driver='bridge')
except docker.errors.APIError as e: # pragma: no cover
if str(e) != '409 Client Error: Conflict ("network with name vent already exists")':
self.logger.error(
'Unable to create network bridge because: {0}'.format(str(e)))
status = (False, str(e))
else:
vent_bridge = self.d_client.networks.list('vent')[0]
if status[0]:
# add vent to the vent network bridge
try:
vent_bridge.connect(environ['HOSTNAME'])
except Exception as e: # pragma: no coverr
self.logger.error(
'Unable to connect vent to the network bridge because: {0}'.format(str(e)))
status = (False, str(e))
if status[0]:
# remove vent to the default network bridge
try:
default_bridge = self.d_client.networks.list('bridge')[0]
default_bridge.disconnect(environ['HOSTNAME'])
except Exception as e: # pragma: no coverr
self.logger.error(
'Unable to disconnect vent from the default network bridge because: {0}'.format(str(e)))
status = (False, str(e))
if status[0]:
# startup based on startup file
if exists(self.startup_file):
status = self._startup()
else:
tools = Tools()
status = tools.new('core', None)
if status[0]:
status = tools.start(
'https://github.com/cyberreboot/vent', None)
return status
def on_ok(self):
"""
Take the tool selections and add them as plugins
"""
def diff(first, second):
"""
Get the elements that exist in the first list and not in the second
"""
second = set(second)
return [item for item in first if item not in second]
def popup(original_tools, branch, thr, title):
"""
Start the thread and display a popup of the tools being added until
the thread is finished
"""
thr.start()
tool_str = 'Adding tools...'
npyscreen.notify_wait(tool_str, title=title)
while thr.is_alive():
tools = diff(ManifestTools(), original_tools)
if tools:
tool_str = ''
for tool in tools:
pre_tool = 'Added: ' + branch + '/' + tool + '\n'
tool_str = pre_tool + tool_str
npyscreen.notify_wait(tool_str, title=title)
time.sleep(1)
return
original_tools = ManifestTools()
for branch in self.tools_tc:
tools = []
for tool in self.tools_tc[branch]:
if self.tools_tc[branch][tool].value:
# get rid of temporary show for multiple tools in same
# directory
if tool == '/':
tools.append(('.', ''))
else:
tools.append((tool, ''))
repo = self.parentApp.repo_value['repo']
version = self.parentApp.repo_value['versions'][branch]
api_action = Tools(version=version, branch=branch)
thr = threading.Thread(target=api_action.new,
args=(),
kwargs={
'tool_type': 'repo',
'uri': repo,
'tools': tools
})
popup(original_tools, branch, thr,
'Please wait, adding tools for the ' + branch + ' branch...')
npyscreen.notify_confirm('Done adding repository: ' +
self.parentApp.repo_value['repo'],
title='Added Repository')
self.quit()
def __init__(self, *args, **keywords):
""" Initialize tool form objects """
self.logger = Logger(__name__)
self.api_action = System()
self.tools_inst = Tools()
action = {'api_action': self.tools_inst}
self.tools_tc = {}
self.repo_widgets = {}
if keywords['action_dict']:
action.update(keywords['action_dict'])
if keywords['names']:
i = 1
for name in keywords['names']:
try:
action['action_object' + str(i)] = getattr(
self.tools_inst, name)
except AttributeError:
action['action_object' + str(i)] = getattr(
self.api_action, name)
i += 1
self.action = action
# get list of all possible group views to display
self.views = deque()
possible_groups = set()
manifest = Template(self.api_action.manifest)
tools = self.tools_inst.inventory(choices=['tools'])[1]['tools']
for tool in tools:
groups = manifest.option(tool, 'groups')[1].split(',')
for group in groups:
# don't do core because that's the purpose of all in views
if group != '' and group != 'core':
possible_groups.add(group)
self.manifest = manifest
self.views += possible_groups
self.views.append('all groups')
self.no_instance = ['remove']
super(ToolForm, self).__init__(*args, **keywords)
def repo_values(self):
"""
Set the appropriate repo dir and get the branches and commits of it
"""
branches = []
commits = {}
m_helper = Tools()
status = m_helper.repo_branches(self.parentApp.repo_value['repo'])
# branches and commits must both be retrieved successfully
if status[0]:
branches = status[1]
status = m_helper.repo_commits(self.parentApp.repo_value['repo'])
if status[0]:
r_commits = status[1]
for commit in r_commits:
commits[commit[0]] = commit[1]
else:
# if commits failed, return commit errors
return status
else:
# if branch failed, return branch errors
return status
# if everything is good, return branches with commits
return branches, commits
def repo_values(self):
"""
Set the appropriate repo dir and get the branches and commits of it
"""
branches = []
commits = {}
m_helper = Tools()
status = m_helper.repo_branches(self.parentApp.repo_value['repo'])
# branches and commits must both be retrieved successfully
if status[0]:
branches = status[1]
status = m_helper.repo_commits(self.parentApp.repo_value['repo'])
if status[0]:
r_commits = status[1]
for commit in r_commits:
commits[commit[0]] = commit[1]
else:
# if commits failed, return commit errors
return status
else:
# if branch failed, return branch errors
return status
# if everything is good, return branches with commits
return branches, commits
def __init__(self, *args, **keywords):
""" Initialize tool form objects """
self.logger = Logger(__name__)
self.api_action = System()
self.tools_inst = Tools()
action = {'api_action': self.tools_inst}
self.tools_tc = {}
self.repo_widgets = {}
if keywords['action_dict']:
action.update(keywords['action_dict'])
if keywords['names']:
i = 1
for name in keywords['names']:
try:
action['action_object' +
str(i)] = getattr(self.tools_inst, name)
except AttributeError:
action['action_object' +
str(i)] = getattr(self.api_action, name)
i += 1
self.action = action
# get list of all possible group views to display
self.views = deque()
possible_groups = set()
manifest = Template(self.api_action.manifest)
tools = self.tools_inst.inventory(choices=['tools'])[1]['tools']
for tool in tools:
groups = manifest.option(tool, 'groups')[1].split(',')
for group in groups:
# don't do core because that's the purpose of all in views
if group != '' and group != 'core':
possible_groups.add(group)
self.manifest = manifest
self.views += possible_groups
self.views.append('all groups')
self.no_instance = ['remove']
super(ToolForm, self).__init__(*args, **keywords)
class ToolForm(npyscreen.ActionForm):
""" Tools form for the Vent CLI """
def __init__(self, *args, **keywords):
""" Initialize tool form objects """
self.logger = Logger(__name__)
self.api_action = System()
self.tools_inst = Tools()
action = {'api_action': self.tools_inst}
self.tools_tc = {}
self.repo_widgets = {}
if keywords['action_dict']:
action.update(keywords['action_dict'])
if keywords['names']:
i = 1
for name in keywords['names']:
try:
action['action_object' +
str(i)] = getattr(self.tools_inst, name)
except AttributeError:
action['action_object' +
str(i)] = getattr(self.api_action, name)
i += 1
self.action = action
# get list of all possible group views to display
self.views = deque()
possible_groups = set()
manifest = Template(self.api_action.manifest)
tools = self.tools_inst.inventory(choices=['tools'])[1]['tools']
for tool in tools:
groups = manifest.option(tool, 'groups')[1].split(',')
for group in groups:
# don't do core because that's the purpose of all in views
if group != '' and group != 'core':
possible_groups.add(group)
self.manifest = manifest
self.views += possible_groups
self.views.append('all groups')
self.no_instance = ['remove']
super(ToolForm, self).__init__(*args, **keywords)
def quit(self, *args, **kwargs):
""" Overridden to switch back to MAIN form """
self.parentApp.switchForm('MAIN')
def toggle_view(self, *args, **kwargs):
""" Toggles the view between different groups """
group_to_display = self.views.popleft()
self.cur_view.value = group_to_display
for repo in self.tools_tc:
for tool in self.tools_tc[repo]:
t_groups = self.manifest.option(tool, 'groups')[1]
if group_to_display not in t_groups and \
group_to_display != 'all groups':
self.tools_tc[repo][tool].value = False
self.tools_tc[repo][tool].hidden = True
else:
self.tools_tc[repo][tool].value = True
self.tools_tc[repo][tool].hidden = False
# redraw elements
self.display()
# add view back to queue
self.views.append(group_to_display)
def create(self, group_view=False):
""" Update with current tools """
self.add_handlers({'^T': self.quit, '^Q': self.quit})
self.add(npyscreen.TitleText,
name='Select which tools to ' + self.action['action'] + ':',
editable=False)
togglable = ['remove']
if self.action['action_name'] in togglable:
self.cur_view = self.add(npyscreen.TitleText,
name='Group view:',
value='all groups', editable=False,
rely=3)
self.add_handlers({'^V': self.toggle_view})
i = 5
else:
i = 4
if self.action['action_name'] == 'start':
response = self.tools_inst.inventory(choices=['repos',
'tools',
'built',
'running'])
else:
response = self.tools_inst.inventory(choices=['repos', 'tools'])
if response[0]:
inventory = response[1]
repos = inventory['repos']
# dict has repo as key and list of core/non-core tools as values
has_core = {}
has_non_core = {}
# find all tools that are in this repo
# and list them if they are core
for repo in repos:
core_list = []
ncore_list = []
# splice the repo names for processing
if (repo.startswith('http')):
repo_name = repo.rsplit('/', 2)[1:]
else:
repo_name = repo.split('/')
for tool in inventory['tools']:
tool_repo_name = tool.split(':')
# cross reference repo names
if (repo_name[0] == tool_repo_name[0] and
repo_name[1] == tool_repo_name[1]):
# check to ensure tool not set to locally active = no
# in vent.cfg
externally_active = False
vent_cfg_file = self.api_action.vent_config
vent_cfg = Template(vent_cfg_file)
tool_pairs = vent_cfg.section('external-services')[1]
for ext_tool in tool_pairs:
if ext_tool[0].lower() == inventory['tools'][tool]:
try:
ext_tool_options = json.loads(ext_tool[1])
loc = 'locally_active'
if (loc in ext_tool_options and
ext_tool_options[loc] == 'no'):
externally_active = True
except Exception as e:
self.logger.error("Couldn't check ext"
' because: ' + str(e))
externally_active = False
manifest = Template(self.api_action.manifest)
if not externally_active:
instance_num = re.search(r'\d+$',
manifest.option(
tool, 'name')[1])
if not instance_num:
ncore_list.append(tool)
# multiple instances share same image
elif self.action['action_name'] not in self.no_instance:
ncore_list.append(tool)
has_core[repo] = core_list
has_non_core[repo] = ncore_list
for repo in repos:
self.tools_tc[repo] = {}
if self.action['cores']:
# make sure only repos with core tools are displayed
if has_core.get(repo):
self.repo_widgets[repo] = self.add(npyscreen.TitleText,
name='Plugin: '+repo,
editable=False,
rely=i, relx=5)
for tool in has_core[repo]:
tool_name = tool.split(':', 2)[2].split('/')[-1]
if tool_name == '':
tool_name = '/'
self.tools_tc[repo][tool] = self.add(
npyscreen.CheckBox, name=tool_name,
value=True, relx=10)
i += 1
i += 3
else:
# make sure only repos with non-core tools are displayed
if has_non_core.get(repo):
self.repo_widgets[repo] = self.add(npyscreen.TitleText,
name='Plugin: '+repo,
editable=False,
rely=i, relx=5)
for tool in has_non_core[repo]:
tool_name = tool.split(':', 2)[2].split('/')[-1]
if tool_name == '':
tool_name = '/'
self.tools_tc[repo][tool] = self.add(
npyscreen.CheckBox, name=tool_name,
value=True, relx=10)
i += 1
i += 3
return
def on_ok(self):
"""
Take the tool selections and perform the provided action on them
"""
def diff(first, second):
"""
Get the elements that exist in the first list and not in the second
"""
second = set(second)
return [item for item in first if item not in second]
def popup(original, orig_type, thr, title):
"""
Start the thread and display a popup of info
until the thread is finished
"""
thr.start()
info_str = ''
while thr.is_alive():
if orig_type == 'containers':
info = diff(Containers(), original)
elif orig_type == 'images':
info = diff(Images(), original)
if info:
info_str = ''
for entry in info:
info_str = entry[0] + ': ' + entry[1] + '\n' + info_str
if self.action['action_name'] != 'configure':
npyscreen.notify_wait(info_str, title=title)
time.sleep(1)
thr.join()
try:
result = self.api_action.queue.get(False)
if isinstance(result, tuple) and isinstance(result[1], tuple):
running, failed = result[1]
r_str = ''
for container in running:
r_str += container + ': successful\n'
for container in failed:
r_str += container + ': failed\n'
npyscreen.notify_confirm(r_str)
except Exception as e: # pragma: no cover
pass
return
if self.action['type'] == 'images':
originals = Images()
else:
originals = Containers()
tool_d = {}
if self.action['action_name'] in ['remove', 'stop', 'update']:
reconfirmation_str = ''
if self.action['cores']:
reconfirmation_str = 'Are you sure you want to '
reconfirmation_str += self.action['action_name']
reconfirmation_str += ' core containers?'
else:
reconfirmation_str = 'Are you sure you want to '
reconfirmation_str += self.action['action_name']
reconfirmation_str += ' plugin containers?'
perform = npyscreen.notify_ok_cancel(reconfirmation_str,
title='Confirm command')
if not perform:
return
tools_to_configure = []
for repo in self.tools_tc:
for tool in self.tools_tc[repo]:
if self.tools_tc[repo][tool].value:
t = tool.split(':', 2)[2].split('/')[-1]
if t.startswith('/:'):
t = ' '+t[1:]
t = t.split(':')
if self.action['action_name'] in ['start', 'stop']:
status = self.action['action_object1'](repo, t[0])
elif self.action['action_name'] == 'configure':
constraints = {'name': t[0],
'branch': t[1],
'version': t[2],
'repo': repo}
options = ['type']
action = self.action['api_action']
tool = self.manifest.constrain_opts(constraints,
options)[0]
# only one tool should be returned
name = list(tool.keys())[0]
if tool[name]['type'] == 'registry':
registry_image = True
else:
registry_image = False
kargs = {'name': 'Configure ' + t[0],
'tool_name': t[0],
'branch': t[1],
'version': t[2],
'repo': repo,
'next_tool': None,
'get_configure': self.api_action.get_configure,
'save_configure': self.api_action.save_configure,
'restart_tools': self.api_action.restart_tools,
'start_tools': action.start,
'from_registry': registry_image}
if tools_to_configure:
kargs['next_tool'] = tools_to_configure[-1]
self.parentApp.addForm('EDITOR' + t[0], EditorForm,
**kargs)
tools_to_configure.append('EDITOR' + t[0])
elif self.action['action_name'] == 'remove':
status = self.action['action_object1'](repo, t[0])
else:
kargs = {'name': t[0],
'branch': t[1],
'version': t[2]}
# add core recognition
if self.action['cores']:
kargs.update({'groups': 'core'})
# use latest version for update, not necessarily
# version in manifest
if self.action['action_name'] == 'update':
if t[2] != 'HEAD':
repo_commits = self.tools_inst.repo_commits(repo)[
1]
for branch in repo_commits:
if branch[0] == t[1]:
kargs.update(
{'new_version': branch[1][0]})
else:
kargs.update({'new_version': 'HEAD'})
thr = Thread(target=self.action['action_object1'],
args=(),
kwargs=kargs)
popup(originals, self.action['type'], thr,
'Please wait, ' + self.action['present_t'] +
'...')
if self.action['action_name'] != 'configure':
npyscreen.notify_confirm('Done ' + self.action['present_t'] + '.',
title=self.action['past_t'])
self.quit()
else:
if len(tools_to_configure) > 0:
self.parentApp.change_form(tools_to_configure[-1])
else:
npyscreen.notify_confirm('No tools selected, returning to'
' main menu',
title='No action taken')
self.quit()
def on_cancel(self):
""" When user clicks cancel, will return to MAIN """
self.quit()
def test_new():
""" Test the new function """
tools = Tools()
tools.new('image', 'redis')
tools.new('core', '')
tools.new('repo', 'https://github.com/cyberreboot/vent')
def _startup(self):
"""
Automatically detect if a startup file is specified and stand up a vent
host with all necessary tools based on the specifications in that file
"""
status = (True, None)
try:
s_dict = {}
# rewrite the yml file to exclusively lowercase
with open(self.startup_file, 'r') as sup:
vent_startup = sup.read()
with open(self.startup_file, 'w') as sup:
for line in vent_startup:
sup.write(line.lower())
with open(self.startup_file, 'r') as sup:
s_dict = yaml.safe_load(sup.read())
if 'vent.cfg' in s_dict:
v_cfg = Template(self.vent_config)
for section in s_dict['vent.cfg']:
for option in s_dict['vent.cfg'][section]:
val = ('no', 'yes')[
s_dict['vent.cfg'][section][option]]
v_status = v_cfg.add_option(section, option, value=val)
if not v_status[0]:
v_cfg.set_option(section, option, val)
v_cfg.write_config()
del s_dict['vent.cfg']
tool_d = {}
extra_options = ['info', 'service', 'settings', 'docker', 'gpu']
s_dict_c = copy.deepcopy(s_dict)
# TODO check for repo or image type
for repo in s_dict_c:
repository = Repository(System().manifest)
repository.repo = repo
repository._clone()
repo_path, org, r_name = self.path_dirs.get_path(repo)
get_tools = []
for tool in s_dict_c[repo]:
t_branch = 'master'
t_version = 'HEAD'
if 'branch' in s_dict[repo][tool]:
t_branch = s_dict[repo][tool]['branch']
if 'version' in s_dict[repo][tool]:
t_version = s_dict[repo][tool]['version']
get_tools.append((tool, t_branch, t_version))
available_tools = AvailableTools(repo_path, tools=get_tools)
for tool in s_dict_c[repo]:
# if we can't find the tool in that repo, skip over this
# tool and notify in the logs
t_path, t_path_cased = PathDirs.rel_path(
tool, available_tools)
if t_path is None:
self.logger.error("Couldn't find tool " + tool + ' in'
' repo ' + repo)
continue
# ensure no NoneType iteration errors
if s_dict_c[repo][tool] is None:
s_dict[repo][tool] = {}
# check if we need to configure instances along the way
instances = 1
if 'settings' in s_dict[repo][tool]:
if 'instances' in s_dict[repo][tool]['settings']:
instances = int(s_dict[repo][tool]
['settings']['instances'])
# add the tool
t_branch = 'master'
t_version = 'HEAD'
t_image = None
add_tools = None
add_tools = [(t_path_cased, '')]
if 'branch' in s_dict[repo][tool]:
t_branch = s_dict[repo][tool]['branch']
if 'version' in s_dict[repo][tool]:
t_version = s_dict[repo][tool]['version']
if 'image' in s_dict[repo][tool]:
t_image = s_dict[repo][tool]['image']
repository.add(
repo, add_tools, branch=t_branch, version=t_version, image_name=t_image)
manifest = Template(self.manifest)
# update the manifest with extra defined runtime settings
base_section = ':'.join([org, r_name, t_path,
t_branch, t_version])
for option in extra_options:
if option in s_dict[repo][tool]:
opt_dict = manifest.option(base_section, option)
# add new values defined into default options for
# that tool, don't overwrite them
if opt_dict[0]:
opt_dict = json.loads(opt_dict[1])
else:
opt_dict = {}
# stringify values for vent
for v in s_dict[repo][tool][option]:
pval = s_dict[repo][tool][option][v]
s_dict[repo][tool][option][v] = json.dumps(
pval)
opt_dict.update(s_dict[repo][tool][option])
manifest.set_option(base_section, option,
json.dumps(opt_dict))
# copy manifest info into new sections if necessary
if instances > 1:
for i in range(2, instances + 1):
i_section = base_section.rsplit(':', 2)
i_section[0] += str(i)
i_section = ':'.join(i_section)
manifest.add_section(i_section)
for opt_val in manifest.section(base_section)[1]:
if opt_val[0] == 'name':
manifest.set_option(i_section, opt_val[0],
opt_val[1] + str(i))
else:
manifest.set_option(i_section, opt_val[0],
opt_val[1])
manifest.write_config()
tool_d = {}
tools = Tools()
# start tools, if necessary
for repo in s_dict:
for tool in s_dict[repo]:
if 'start' in s_dict[repo][tool]:
if s_dict[repo][tool]['start']:
local_instances = 1
if 'settings' in s_dict[repo][tool] and 'instances' in s_dict[repo][tool]['settings']:
local_instances = int(
s_dict[repo][tool]['settings']['instances'])
t_branch = 'master'
t_version = 'HEAD'
if 'branch' in s_dict[repo][tool]:
t_branch = s_dict[repo][tool]['branch']
if 'version' in s_dict[repo][tool]:
t_version = s_dict[repo][tool]['version']
for i in range(1, local_instances + 1):
i_name = tool + str(i) if i != 1 else tool
i_name = i_name.replace('@', '')
tool_d.update(
tools._prep_start(repo, i_name)[1])
if tool_d:
tools.start(tool_d, None, is_tool_d=True)
except Exception as e: # pragma: no cover
self.logger.error('Startup failed because: {0}'.format(str(e)))
status = (False, str(e))
return status
def test_remove():
""" Test the remove function """
tools = Tools()
tools.remove('https://github.com/cyberreboot/vent', 'rabbitmq')
class ToolForm(npyscreen.ActionForm):
""" Tools form for the Vent CLI """
def __init__(self, *args, **keywords):
""" Initialize tool form objects """
self.logger = Logger(__name__)
self.api_action = System()
self.tools_inst = Tools()
action = {'api_action': self.tools_inst}
self.tools_tc = {}
self.repo_widgets = {}
if keywords['action_dict']:
action.update(keywords['action_dict'])
if keywords['names']:
i = 1
for name in keywords['names']:
try:
action['action_object' + str(i)] = getattr(
self.tools_inst, name)
except AttributeError:
action['action_object' + str(i)] = getattr(
self.api_action, name)
i += 1
self.action = action
# get list of all possible group views to display
self.views = deque()
possible_groups = set()
manifest = Template(self.api_action.manifest)
tools = self.tools_inst.inventory(choices=['tools'])[1]['tools']
for tool in tools:
groups = manifest.option(tool, 'groups')[1].split(',')
for group in groups:
# don't do core because that's the purpose of all in views
if group != '' and group != 'core':
possible_groups.add(group)
self.manifest = manifest
self.views += possible_groups
self.views.append('all groups')
self.no_instance = ['remove']
super(ToolForm, self).__init__(*args, **keywords)
def quit(self, *args, **kwargs):
""" Overridden to switch back to MAIN form """
self.parentApp.switchForm('MAIN')
def toggle_view(self, *args, **kwargs):
""" Toggles the view between different groups """
group_to_display = self.views.popleft()
self.cur_view.value = group_to_display
for repo in self.tools_tc:
for tool in self.tools_tc[repo]:
t_groups = self.manifest.option(tool, 'groups')[1]
if group_to_display not in t_groups and \
group_to_display != 'all groups':
self.tools_tc[repo][tool].value = False
self.tools_tc[repo][tool].hidden = True
else:
self.tools_tc[repo][tool].value = True
self.tools_tc[repo][tool].hidden = False
# redraw elements
self.display()
# add view back to queue
self.views.append(group_to_display)
def create(self, group_view=False):
""" Update with current tools """
self.add_handlers({'^T': self.quit, '^Q': self.quit})
self.add(npyscreen.TitleText,
name='Select which tools to ' + self.action['action'] + ':',
editable=False)
togglable = ['remove']
if self.action['action_name'] in togglable:
self.cur_view = self.add(npyscreen.TitleText,
name='Group view:',
value='all groups',
editable=False,
rely=3)
self.add_handlers({'^V': self.toggle_view})
i = 5
else:
i = 4
if self.action['action_name'] == 'start':
response = self.tools_inst.inventory(
choices=['repos', 'tools', 'built', 'running'])
else:
response = self.tools_inst.inventory(choices=['repos', 'tools'])
if response[0]:
inventory = response[1]
repos = inventory['repos']
# dict has repo as key and list of core/non-core tools as values
has_core = {}
has_non_core = {}
# find all tools that are in this repo
# and list them if they are core
for repo in repos:
core_list = []
ncore_list = []
# splice the repo names for processing
if (repo.startswith('http')):
repo_name = repo.rsplit('/', 2)[1:]
else:
repo_name = repo.split('/')
for tool in inventory['tools']:
tool_repo_name = tool.split(':')
# cross reference repo names
if (repo_name[0] == tool_repo_name[0]
and repo_name[1] == tool_repo_name[1]):
# check to ensure tool not set to locally active = no
# in vent.cfg
externally_active = False
vent_cfg_file = self.api_action.vent_config
vent_cfg = Template(vent_cfg_file)
tool_pairs = vent_cfg.section('external-services')[1]
for ext_tool in tool_pairs:
if ext_tool[0].lower() == inventory['tools'][tool]:
try:
ext_tool_options = json.loads(ext_tool[1])
loc = 'locally_active'
if (loc in ext_tool_options
and ext_tool_options[loc] == 'no'):
externally_active = True
except Exception as e:
self.logger.error("Couldn't check ext"
' because: ' + str(e))
externally_active = False
manifest = Template(self.api_action.manifest)
if not externally_active:
instance_num = re.search(
r'\d+$',
manifest.option(tool, 'name')[1])
if not instance_num:
ncore_list.append(tool)
# multiple instances share same image
elif self.action[
'action_name'] not in self.no_instance:
ncore_list.append(tool)
has_core[repo] = core_list
has_non_core[repo] = ncore_list
for repo in repos:
self.tools_tc[repo] = {}
if self.action['cores']:
# make sure only repos with core tools are displayed
if has_core.get(repo):
self.repo_widgets[repo] = self.add(npyscreen.TitleText,
name='Plugin: ' +
repo,
editable=False,
rely=i,
relx=5)
for tool in has_core[repo]:
tool_name = tool.split(':', 2)[2].split('/')[-1]
if tool_name == '':
tool_name = '/'
self.tools_tc[repo][tool] = self.add(
npyscreen.CheckBox,
name=tool_name,
value=True,
relx=10)
i += 1
i += 3
else:
# make sure only repos with non-core tools are displayed
if has_non_core.get(repo):
self.repo_widgets[repo] = self.add(npyscreen.TitleText,
name='Plugin: ' +
repo,
editable=False,
rely=i,
relx=5)
for tool in has_non_core[repo]:
tool_name = tool.split(':', 2)[2].split('/')[-1]
if tool_name == '':
tool_name = '/'
self.tools_tc[repo][tool] = self.add(
npyscreen.CheckBox,
name=tool_name,
value=True,
relx=10)
i += 1
i += 3
return
def on_ok(self):
"""
Take the tool selections and perform the provided action on them
"""
def diff(first, second):
"""
Get the elements that exist in the first list and not in the second
"""
second = set(second)
return [item for item in first if item not in second]
def popup(original, orig_type, thr, title):
"""
Start the thread and display a popup of info
until the thread is finished
"""
thr.start()
info_str = ''
while thr.is_alive():
if orig_type == 'containers':
info = diff(Containers(), original)
elif orig_type == 'images':
info = diff(Images(), original)
if info:
info_str = ''
for entry in info:
info_str = entry[0] + ': ' + entry[1] + '\n' + info_str
if self.action['action_name'] != 'configure':
npyscreen.notify_wait(info_str, title=title)
time.sleep(1)
thr.join()
try:
result = self.api_action.queue.get(False)
if isinstance(result, tuple) and isinstance(result[1], tuple):
running, failed = result[1]
r_str = ''
for container in running:
r_str += container + ': successful\n'
for container in failed:
r_str += container + ': failed\n'
npyscreen.notify_confirm(r_str)
except Exception as e: # pragma: no cover
pass
return
if self.action['type'] == 'images':
originals = Images()
else:
originals = Containers()
tool_d = {}
if self.action['action_name'] in ['remove', 'stop', 'update']:
reconfirmation_str = ''
if self.action['cores']:
reconfirmation_str = 'Are you sure you want to '
reconfirmation_str += self.action['action_name']
reconfirmation_str += ' core containers?'
else:
reconfirmation_str = 'Are you sure you want to '
reconfirmation_str += self.action['action_name']
reconfirmation_str += ' plugin containers?'
perform = npyscreen.notify_ok_cancel(reconfirmation_str,
title='Confirm command')
if not perform:
return
tools_to_configure = []
for repo in self.tools_tc:
for tool in self.tools_tc[repo]:
if self.tools_tc[repo][tool].value:
t = tool.split(':', 2)[2].split('/')[-1]
if t.startswith('/:'):
t = ' ' + t[1:]
t = t.split(':')
if self.action['action_name'] in ['start', 'stop']:
status = self.action['action_object1'](repo, t[0])
elif self.action['action_name'] == 'configure':
constraints = {
'name': t[0],
'branch': t[1],
'version': t[2],
'repo': repo
}
options = ['type']
action = self.action['api_action']
tool = self.manifest.constrain_opts(
constraints, options)[0]
# only one tool should be returned
name = list(tool.keys())[0]
if tool[name]['type'] == 'registry':
registry_image = True
else:
registry_image = False
kargs = {
'name': 'Configure ' + t[0],
'tool_name': t[0],
'branch': t[1],
'version': t[2],
'repo': repo,
'next_tool': None,
'get_configure': self.api_action.get_configure,
'save_configure': self.api_action.save_configure,
'restart_tools': self.api_action.restart_tools,
'start_tools': action.start,
'from_registry': registry_image
}
if tools_to_configure:
kargs['next_tool'] = tools_to_configure[-1]
self.parentApp.addForm('EDITOR' + t[0], EditorForm,
**kargs)
tools_to_configure.append('EDITOR' + t[0])
elif self.action['action_name'] == 'remove':
status = self.action['action_object1'](repo, t[0])
else:
kargs = {'name': t[0], 'branch': t[1], 'version': t[2]}
# add core recognition
if self.action['cores']:
kargs.update({'groups': 'core'})
# use latest version for update, not necessarily
# version in manifest
if self.action['action_name'] == 'update':
if t[2] != 'HEAD':
repo_commits = self.tools_inst.repo_commits(
repo)[1]
for branch in repo_commits:
if branch[0] == t[1]:
kargs.update(
{'new_version': branch[1][0]})
else:
kargs.update({'new_version': 'HEAD'})
thr = Thread(target=self.action['action_object1'],
args=(),
kwargs=kargs)
popup(
originals, self.action['type'], thr,
'Please wait, ' + self.action['present_t'] + '...')
if self.action['action_name'] != 'configure':
npyscreen.notify_confirm('Done ' + self.action['present_t'] + '.',
title=self.action['past_t'])
self.quit()
else:
if len(tools_to_configure) > 0:
self.parentApp.change_form(tools_to_configure[-1])
else:
npyscreen.notify_confirm(
'No tools selected, returning to'
' main menu',
title='No action taken')
self.quit()
def on_cancel(self):
""" When user clicks cancel, will return to MAIN """
self.quit()
def test_inventory():
""" Test the inventory function """
tools = Tools()
tools.inventory()
def restart_tools(self,
repo=None,
name=None,
groups=None,
branch='master',
version='HEAD',
main_cfg=False,
old_val='',
new_val=''):
"""
Restart necessary tools based on changes that have been made either to
vent.cfg or to vent.template. This includes tools that need to be
restarted because they depend on other tools that were changed.
"""
status = (True, None)
if not main_cfg:
try:
t_identifier = {'name': name,
'branch': branch,
'version': version}
result = Template(System().manifest).constrain_opts(t_identifier,
['running',
'link_name'])
tools = result[0]
tool = list(tools.keys())[0]
if ('running' in tools[tool] and
tools[tool]['running'] == 'yes'):
start_tools = [t_identifier]
dependent_tools = [tools[tool]['link_name']]
start_tools += Dependencies(dependent_tools)
# TODO
start_d = {}
for tool_identifier in start_tools:
self.clean(**tool_identifier)
start_d.update(self.prep_start(**tool_identifier)[1])
if start_d:
Tools().start(start_d, '', is_tool_d=True)
except Exception as e: # pragma: no cover
self.logger.error('Trouble restarting tool ' + name +
' because: ' + str(e))
status = (False, str(e))
else:
try:
# string manipulation to get tools into arrays
ext_start = old_val.find('[external-services]')
if ext_start >= 0:
ot_str = old_val[old_val.find('[external-services]') + 20:]
else:
ot_str = ''
old_tools = []
for old_tool in ot_str.split('\n'):
if old_tool != '':
old_tools.append(old_tool.split('=')[0].strip())
ext_start = new_val.find('[external-services]')
if ext_start >= 0:
nt_str = new_val[new_val.find('[external-services]') + 20:]
else:
nt_str = ''
new_tools = []
for new_tool in nt_str.split('\n'):
if new_tool != '':
new_tools.append(new_tool.split('=')[0].strip())
# find tools changed
tool_changes = []
for old_tool in old_tools:
if old_tool not in new_tools:
tool_changes.append(old_tool)
for new_tool in new_tools:
if new_tool not in old_tools:
tool_changes.append(new_tool)
else:
# tool name will be the same
oconf = old_val[old_val.find(new_tool):].split('\n')[0]
nconf = new_val[new_val.find(new_tool):].split('\n')[0]
if oconf != nconf:
tool_changes.append(new_tool)
# put link names in a dictionary for finding dependencies
dependent_tools = []
for i, entry in enumerate(tool_changes):
dependent_tools.append(entry)
# change names to lowercase for use in clean, prep_start
tool_changes[i] = {'name': entry.lower().replace('-', '_')}
dependencies = Dependencies(dependent_tools)
# restart tools
restart = tool_changes + dependencies
tool_d = {}
for tool in restart:
self.clean(**tool)
tool_d.update(self.prep_start(**tool)[1])
if tool_d:
# TODO fix the arguments
Tools().start(tool_d)
except Exception as e: # pragma: no cover
self.logger.error('Problem restarting tools: ' + str(e))
status = (False, str(e))
return status
def _startup(self):
"""
Automatically detect if a startup file is specified and stand up a vent
host with all necessary tools based on the specifications in that file
"""
status = (True, None)
try:
s_dict = {}
# rewrite the yml file to exclusively lowercase
with open(self.startup_file, 'r') as sup:
vent_startup = sup.read()
with open(self.startup_file, 'w') as sup:
for line in vent_startup:
sup.write(line.lower())
with open(self.startup_file, 'r') as sup:
s_dict = yaml.safe_load(sup.read())
if 'vent.cfg' in s_dict:
v_cfg = Template(self.vent_config)
for section in s_dict['vent.cfg']:
for option in s_dict['vent.cfg'][section]:
val = ('no', 'yes')[
s_dict['vent.cfg'][section][option]]
v_status = v_cfg.add_option(section, option, value=val)
if not v_status[0]:
v_cfg.set_option(section, option, val)
v_cfg.write_config()
del s_dict['vent.cfg']
tool_d = {}
extra_options = ['info', 'service', 'settings', 'docker', 'gpu']
s_dict_c = copy.deepcopy(s_dict)
# TODO check for repo or image type
for repo in s_dict_c:
repository = Repository(System().manifest)
repository.repo = repo
repository._clone()
repo_path, org, r_name = self.path_dirs.get_path(repo)
get_tools = []
for tool in s_dict_c[repo]:
t_branch = 'master'
t_version = 'HEAD'
if 'branch' in s_dict[repo][tool]:
t_branch = s_dict[repo][tool]['branch']
if 'version' in s_dict[repo][tool]:
t_version = s_dict[repo][tool]['version']
get_tools.append((tool, t_branch, t_version))
available_tools = AvailableTools(repo_path, tools=get_tools)
for tool in s_dict_c[repo]:
# if we can't find the tool in that repo, skip over this
# tool and notify in the logs
t_path, t_path_cased = PathDirs.rel_path(
tool, available_tools)
if t_path is None:
self.logger.error("Couldn't find tool " + tool + ' in'
' repo ' + repo)
continue
# ensure no NoneType iteration errors
if s_dict_c[repo][tool] is None:
s_dict[repo][tool] = {}
# check if we need to configure instances along the way
instances = 1
if 'settings' in s_dict[repo][tool]:
if 'instances' in s_dict[repo][tool]['settings']:
instances = int(s_dict[repo][tool]
['settings']['instances'])
# add the tool
t_branch = 'master'
t_version = 'HEAD'
t_image = None
add_tools = None
add_tools = [(t_path_cased, '')]
if 'branch' in s_dict[repo][tool]:
t_branch = s_dict[repo][tool]['branch']
if 'version' in s_dict[repo][tool]:
t_version = s_dict[repo][tool]['version']
if 'image' in s_dict[repo][tool]:
t_image = s_dict[repo][tool]['image']
repository.add(
repo, tools=add_tools, branch=t_branch, version=t_version, image_name=t_image)
manifest = Template(self.manifest)
# update the manifest with extra defined runtime settings
base_section = ':'.join([org, r_name, t_path,
t_branch, t_version])
for option in extra_options:
if option in s_dict[repo][tool]:
opt_dict = manifest.option(base_section, option)
# add new values defined into default options for
# that tool, don't overwrite them
if opt_dict[0]:
opt_dict = json.loads(opt_dict[1])
else:
opt_dict = {}
# stringify values for vent
for v in s_dict[repo][tool][option]:
pval = s_dict[repo][tool][option][v]
s_dict[repo][tool][option][v] = json.dumps(
pval)
opt_dict.update(s_dict[repo][tool][option])
manifest.set_option(base_section, option,
json.dumps(opt_dict))
# copy manifest info into new sections if necessary
if instances > 1:
for i in range(2, instances + 1):
i_section = base_section.rsplit(':', 2)
i_section[0] += str(i)
i_section = ':'.join(i_section)
manifest.add_section(i_section)
for opt_val in manifest.section(base_section)[1]:
if opt_val[0] == 'name':
manifest.set_option(i_section, opt_val[0],
opt_val[1] + str(i))
else:
manifest.set_option(i_section, opt_val[0],
opt_val[1])
manifest.write_config()
tool_d = {}
tools = Tools()
# start tools, if necessary
for repo in s_dict:
for tool in s_dict[repo]:
if 'start' in s_dict[repo][tool]:
if s_dict[repo][tool]['start']:
local_instances = 1
if 'settings' in s_dict[repo][tool] and 'instances' in s_dict[repo][tool]['settings']:
local_instances = int(
s_dict[repo][tool]['settings']['instances'])
t_branch = 'master'
t_version = 'HEAD'
if 'branch' in s_dict[repo][tool]:
t_branch = s_dict[repo][tool]['branch']
if 'version' in s_dict[repo][tool]:
t_version = s_dict[repo][tool]['version']
for i in range(1, local_instances + 1):
i_name = tool + str(i) if i != 1 else tool
i_name = i_name.replace('@', '')
tool_d.update(
tools._prep_start(repo, i_name)[1])
if tool_d:
tools.start(tool_d, None, is_tool_d=True)
except Exception as e: # pragma: no cover
self.logger.error('Startup failed because: {0}'.format(str(e)))
status = (False, str(e))
return status
def on_ok(self):
""" Add the repository """
def popup(thr, add_type, title):
"""
Start the thread and display a popup of the plugin being cloned
until the thread is finished
"""
thr.start()
tool_str = 'Cloning repository...'
if add_type == 'image':
tool_str = 'Pulling image...'
npyscreen.notify_wait(tool_str, title=title)
while thr.is_alive():
time.sleep(1)
return
if self.image.value and self.link_name.value:
api_action = Tools()
api_image = Image(System().manifest)
api_system = System()
thr = threading.Thread(target=api_image.add,
args=(),
kwargs={
'image': self.image.value,
'link_name': self.link_name.value,
'tag': self.tag.value,
'registry': self.registry.value,
'groups': self.groups.value
})
popup(thr, 'image', 'Please wait, adding image...')
npyscreen.notify_confirm('Done adding image.', title='Added image')
editor_args = {
'tool_name': self.image.value,
'version': self.tag.value,
'get_configure': api_system.get_configure,
'save_configure': api_system.save_configure,
'restart_tools': api_system.restart_tools,
'start_tools': api_action.start,
'from_registry': True,
'just_downloaded': True,
'link_name': self.link_name.value,
'groups': self.groups.value
}
self.parentApp.addForm('CONFIGUREIMAGE',
EditorForm,
name='Specify vent.template settings for '
'image pulled (optional)',
**editor_args)
self.parentApp.change_form('CONFIGUREIMAGE')
elif self.image.value:
npyscreen.notify_confirm(
'A name needs to be supplied for '
'the image being added!',
title='Specify a name for the image',
form_color='CAUTION')
elif self.repo.value:
self.parentApp.repo_value['repo'] = self.repo.value.lower()
api_repo = Repository(System().manifest)
api_repo.repo = self.repo.value.lower()
thr = threading.Thread(target=api_repo._clone,
args=(),
kwargs={
'user': self.user.value,
'pw': self.pw.value
})
popup(thr, 'repository', 'Please wait, adding repository...')
self.parentApp.addForm('ADDOPTIONS',
AddOptionsForm,
name='Set options for new plugin'
'\t\t\t\t\t\t^Q to quit',
color='CONTROL')
self.parentApp.change_form('ADDOPTIONS')
else:
npyscreen.notify_confirm(
'Either a repository or an image '
'name must be specified!',
title='Specify plugin to add',
form_color='CAUTION')
return
def test_stop():
""" Test the stop function """
tools = Tools()
tools.stop('https://github.com/cyberreboot/vent', 'rabbitmq')
def test_configure():
""" Test the configure function """
tools = Tools()
tools.configure('foo')