def ManifestTools(**kargs):
""" Get tools that exist in the manifest """
path_dirs = PathDirs(**kargs)
manifest = join(path_dirs.meta_dir, 'plugin_manifest.cfg')
template = Template(template=manifest)
tools = template.sections()
return tools[1]
def _build(self):
status = (True, None)
status = self._get_tools()
matches = status[1]
status = self.path_dirs.apply_path(self.repo)
original_path = status[1]
if status[0] and len(matches) > 0:
repo, org, name = self.path_dirs.get_path(self.repo)
cmd = 'git rev-parse --short ' + self.version
commit_id = ''
try:
commit_id = check_output(shlex.split(cmd), stderr=STDOUT,
close_fds=True).strip().decode('utf-8')
except Exception as e: # pragma: no cover
self.logger.error(
'Unable to get commit ID because: {0}'.format(str(e)))
template = Template(template=self.manifest)
for match in matches:
status, template, match_path, image_name, section = self._build_manifest(
match, template, repo, org, name, commit_id)
if not status[0]:
break
status, template = self._build_image(template,
match_path,
image_name,
section)
if not status[0]:
break
if status[0]:
# write out configuration to the manifest file
template.write_config()
chdir(original_path)
return status
def toggle_view(self, *args, **kwargs):
group = self.views.popleft()
new_display = []
new_display.append('Tools for group ' + group + ' found:')
manifest = Template(self.api_action.manifest)
cur_repo = ''
for i in range(1, len(self.all_tools) - 1):
val = self.all_tools[i]
# get repo val
if val.startswith(' Plugin:'):
new_display.append(val)
cur_repo = val.split(':', 1)[1].strip()
# determine if tool should be displayed in this group
elif val.startswith(' ') and not val.startswith(' '):
name = val.strip()
constraints = {'repo': cur_repo, 'name': name}
t_section = manifest.constrain_opts(constraints, [])[0]
t_section = list(t_section.keys())[0]
if group in manifest.option(t_section, 'groups')[1].split(','):
new_display += self.all_tools[i:i+4]
elif val == '':
new_display.append(val)
# if all groups display all groups
if group == 'all groups':
self.display_val.values = self.all_tools
else:
self.display_val.values = new_display
# redraw
self.display()
# add group back into cycle
self.views.append(group)
def DropLocation():
""" Get the directory that file drop is watching """
template = Template(template=PathDirs().cfg_file)
drop_loc = template.option('main', 'files')[1]
drop_loc = expanduser(drop_loc)
drop_loc = abspath(drop_loc)
return (True, drop_loc)
def _start_priority_containers(self, groups, group_orders, tool_d):
""" Select containers based on priorities to start """
vent_cfg = Template(self.path_dirs.cfg_file)
cfg_groups = vent_cfg.option('groups', 'start_order')
if cfg_groups[0]:
cfg_groups = cfg_groups[1].split(',')
else:
cfg_groups = []
all_groups = sorted(set(groups))
s_conts = []
f_conts = []
# start tools in order of group defined in vent.cfg
for group in cfg_groups:
# remove from all_groups because already checked out
if group in all_groups:
all_groups.remove(group)
if group in group_orders:
for cont_t in sorted(group_orders[group]):
if cont_t[1] not in s_conts:
s_conts, f_conts = self._start_container(
cont_t[1], tool_d, s_conts, f_conts)
# start tools that haven't been specified in the vent.cfg, if any
for group in all_groups:
if group in group_orders:
for cont_t in sorted(group_orders[group]):
if cont_t[1] not in s_conts:
s_conts, f_conts = self._start_container(
cont_t[1], tool_d, s_conts, f_conts)
return (s_conts, f_conts)
def toggle_view(self, *args, **kwargs):
group = self.views.popleft()
new_display = []
new_display.append('Tools for group ' + group + ' found:')
manifest = Template(self.api_action.manifest)
cur_repo = ''
for i in range(1, len(self.all_tools) - 1):
val = self.all_tools[i]
# get repo val
if val.startswith(' Plugin:'):
new_display.append(val)
cur_repo = val.split(':', 1)[1].strip()
# determine if tool should be displayed in this group
elif val.startswith(' ') and not val.startswith(' '):
name = val.strip()
constraints = {'repo': cur_repo, 'name': name}
t_section = manifest.constrain_opts(constraints, [])[0]
t_section = list(t_section.keys())[0]
if group in manifest.option(t_section, 'groups')[1].split(','):
new_display += self.all_tools[i:i + 4]
elif val == '':
new_display.append(val)
# if all groups display all groups
if group == 'all groups':
self.display_val.values = self.all_tools
else:
self.display_val.values = new_display
# redraw
self.display()
# add group back into cycle
self.views.append(group)
def _start_priority_containers(self, groups, group_orders, tool_d):
""" Select containers based on priorities to start """
vent_cfg = Template(self.path_dirs.cfg_file)
cfg_groups = vent_cfg.option('groups', 'start_order')
if cfg_groups[0]:
cfg_groups = cfg_groups[1].split(',')
else:
cfg_groups = []
all_groups = sorted(set(groups))
s_conts = []
f_conts = []
# start tools in order of group defined in vent.cfg
for group in cfg_groups:
# remove from all_groups because already checked out
if group in all_groups:
all_groups.remove(group)
if group in group_orders:
for cont_t in sorted(group_orders[group]):
if cont_t[1] not in s_conts:
s_conts, f_conts = self._start_container(cont_t[1],
tool_d,
s_conts,
f_conts)
# start tools that haven't been specified in the vent.cfg, if any
for group in all_groups:
if group in group_orders:
for cont_t in sorted(group_orders[group]):
if cont_t[1] not in s_conts:
s_conts, f_conts = self._start_container(cont_t[1],
tool_d,
s_conts,
f_conts)
return (s_conts, f_conts)
def _build(self):
status = (True, None)
status = self._get_tools()
matches = status[1]
status = self.path_dirs.apply_path(self.repo)
original_path = status[1]
if status[0] and len(matches) > 0:
repo, org, name = self.path_dirs.get_path(self.repo)
cmd = 'git rev-parse --short ' + self.version
commit_id = ''
try:
commit_id = check_output(
shlex.split(cmd), stderr=STDOUT,
close_fds=True).strip().decode('utf-8')
except Exception as e: # pragma: no cover
self.logger.error(
'Unable to get commit ID because: {0}'.format(str(e)))
template = Template(template=self.manifest)
for match in matches:
status, template, match_path, image_name, section = self._build_manifest(
match, template, repo, org, name, commit_id)
if not status[0]:
break
status, template = self._build_image(template, match_path,
image_name, section)
if not status[0]:
break
if status[0]:
# write out configuration to the manifest file
template.write_config()
chdir(original_path)
return status
def Dependencies(tools):
"""
Takes in a list of tools that are being updated and returns any tools that
depend on linking to them
"""
dependencies = []
if tools:
path_dirs = PathDirs()
man = Template(join(path_dirs.meta_dir, 'plugin_manifest.cfg'))
for section in man.sections()[1]:
# don't worry about dealing with tool if it's not running
running = man.option(section, 'running')
if not running[0] or running[1] != 'yes':
continue
t_name = man.option(section, 'name')[1]
t_branch = man.option(section, 'branch')[1]
t_version = man.option(section, 'version')[1]
t_identifier = {'name': t_name,
'branch': t_branch,
'version': t_version}
options = man.options(section)[1]
if 'docker' in options:
d_settings = json.loads(man.option(section,
'docker')[1])
if 'links' in d_settings:
for link in json.loads(d_settings['links']):
if link in tools:
dependencies.append(t_identifier)
return dependencies
def remove(self, repo, name):
args = locals()
status = (True, None)
# get resulting dict of sections with options that match constraints
template = Template(template=self.manifest)
results, _ = template.constrain_opts(args, [])
for result in results:
response, image_name = template.option(result, 'image_name')
name = template.option(result, 'name')[1]
try:
settings_dict = json.loads(template.option(result,
'settings')[1])
instances = int(settings_dict['instances'])
except Exception:
instances = 1
try:
# check for container and remove
c_name = image_name.replace(':', '-').replace('/', '-')
for i in range(1, instances + 1):
container_name = c_name + str(i) if i != 1 else c_name
container = self.d_client.containers.get(container_name)
response = container.remove(v=True, force=True)
self.logger.info(
'Removing container: {0}'.format(container_name))
except Exception as e: # pragma: no cover
self.logger.warning('Unable to remove the container: ' +
container_name + ' because: ' + str(e))
# check for image and remove
try:
response = None
image_id = template.option(result, 'image_id')[1]
response = self.d_client.images.remove(image_id, force=True)
self.logger.info('Removing image: ' + image_name)
except Exception as e: # pragma: no cover
self.logger.warning('Unable to remove the image: ' +
image_name + ' because: ' + str(e))
# remove tool from the manifest
for i in range(1, instances + 1):
res = result.rsplit(':', 2)
res[0] += str(i) if i != 1 else ''
res = ':'.join(res)
if template.section(res)[0]:
status = template.del_section(res)
self.logger.info('Removing tool: ' + res)
# TODO if all tools from a repo have been removed, remove the repo
template.write_config()
return status
def host_config(self):
""" Ensure the host configuration file exists """
if platform.system() == 'Darwin':
default_file_dir = join(expanduser('~'), 'vent_files')
else:
default_file_dir = '/opt/vent_files'
status = self.ensure_dir(default_file_dir)
if not isfile(self.cfg_file):
config = Template(template=self.cfg_file)
sections = {
'main': {
'files': default_file_dir
},
'network-mapping': {},
'nvidia-docker-plugin': {
'port': '3476'
}
}
for s in sections:
if sections[s]:
for option in sections[s]:
config.add_option(s, option, sections[s][option])
else:
config.add_section(s)
config.write_config()
return status
def test_del_option():
""" Test the del_option function """
instance = Template()
instance.del_option('foo', 'bar')
instance.add_option('foo', 'bar2')
instance.del_option('foo', 'bar2')
instance.del_option('foo', 'bar2')
def get_configure(self,
repo=None,
name=None,
groups=None,
main_cfg=False):
"""
Get the vent.template settings for a given tool by looking at the
plugin_manifest
"""
constraints = locals()
del constraints['main_cfg']
status = (True, None)
template_dict = {}
return_str = ''
if main_cfg:
vent_cfg = Template(self.vent_config)
for section in vent_cfg.sections()[1]:
template_dict[section] = {}
for vals in vent_cfg.section(section)[1]:
template_dict[section][vals[0]] = vals[1]
else:
# all possible vent.template options stored in plugin_manifest
options = ['info', 'service', 'settings', 'docker', 'gpu']
tools = Template(System().manifest).constrain_opts(
constraints, options)[0]
if tools:
# should only be one tool
tool = list(tools.keys())[0]
# load all vent.template options into dict
for section in tools[tool]:
template_dict[section] = json.loads(tools[tool][section])
else:
status = (False, "Couldn't get vent.template information")
if status[0]:
# display all those options as they would in the file
for section in template_dict:
return_str += '[' + section + ']\n'
# ensure instances shows up in configuration
for option in template_dict[section]:
if option.startswith('#'):
return_str += option + '\n'
else:
return_str += option + ' = '
return_str += template_dict[section][option] + '\n'
return_str += '\n'
# only one newline at end of file
status = (True, return_str[:-1])
return status
def stop(self, repo, name):
args = locals()
status = (True, None)
try:
# !! TODO need to account for plugin containers that have random
# names, use labels perhaps
options = [
'name', 'namespace', 'built', 'groups', 'path', 'image_name',
'branch', 'version'
]
s, _ = Template(template=self.manifest).constrain_opts(
args, options)
for section in s:
container_name = s[section]['image_name'].replace(':', '-')
container_name = container_name.replace('/', '-')
try:
container = self.d_client.containers.get(container_name)
container.stop()
self.logger.info('Stopped {0}'.format(str(container_name)))
except Exception as e: # pragma: no cover
self.logger.error('Failed to stop ' + str(container_name) +
' because: ' + str(e))
except Exception as e: # pragma: no cover
self.logger.error('Stop failed with error: ' + str(e))
status = (False, e)
return status
def new(self,
tool_type,
uri,
tools=None,
link_name=None,
image_name=None,
overrides=None,
tag=None,
registry=None,
groups=None):
try:
# remove tools that are already installed from being added
if isinstance(tools, list):
i = len(tools) - 1
while i >= 0:
tool = tools[i]
if tool[0].find('@') >= 0:
tool_name = tool[0].split('@')[-1]
else:
tool_name = tool[0].rsplit('/', 1)[-1]
constraints = {
'name': tool_name,
'repo': uri.split('.git')[0]
}
prev_installed, _ = Template(
template=self.manifest).constrain_opts(
constraints, [])
# don't reinstall
if prev_installed:
tools.remove(tool)
i -= 1
if len(tools) == 0:
tools = None
except Exception as e: # pragma: no cover
self.logger.error('Add failed with error: {0}'.format(str(e)))
return (False, str(e))
if tool_type == 'image':
status = Image(self.manifest).add(uri,
link_name,
tag=tag,
registry=registry,
groups=groups)
else:
if tool_type == 'core':
uri = 'https://github.com/cyberreboot/vent'
core = True
else:
core = False
status = Repository(self.manifest).add(uri,
tools,
overrides=overrides,
version=self.version,
image_name=image_name,
branch=self.branch,
user=self.user,
pw=self.pw,
core=core)
return status
def __init__(self, action=None, logger=None, *args, **keywords):
""" Initialize inventory form objects """
self.action = action
self.logger = logger
self.api_action = self.action['api_action']
# get list of all possible group views to display
self.views = deque()
possible_groups = set()
manifest = Template(self.api_action.manifest)
tools = self.api_action.inventory(choices=['tools'])[1]['tools']
for tool in tools:
groups = manifest.option(tool, 'groups')[1].split(',')
for group in groups:
# don't do core because that's the purpose of all in views
if group != '' and group != 'core':
possible_groups.add(group)
self.views += possible_groups
self.views.append('all groups')
super(InventoryForm, self).__init__(*args, **keywords)
def __init__(self, action=None, logger=None, *args, **keywords):
""" Initialize inventory form objects """
self.action = action
self.logger = logger
self.api_action = self.action['api_action']
# get list of all possible group views to display
self.views = deque()
possible_groups = set()
manifest = Template(self.api_action.manifest)
tools = self.api_action.inventory(choices=['tools'])[1]['tools']
for tool in tools:
groups = manifest.option(tool, 'groups')[1].split(',')
for group in groups:
# don't do core because that's the purpose of all in views
if group != '' and group != 'core':
possible_groups.add(group)
self.views += possible_groups
self.views.append('all groups')
super(InventoryForm, self).__init__(*args, **keywords)
def override_config(self, path):
"""
Will take a yml located in home directory titled '.plugin_config.yml'.
It'll then override, using the yml, the plugin's config file
"""
status = (True, None)
config_override = False
try:
# parse the yml file
c_dict = {}
if exists(self.plugin_config_file):
with open(self.plugin_config_file, 'r') as config_file:
c_dict = yaml.safe_load(config_file.read())
# check for environment variable overrides
check_c_dict = c_dict.copy()
for tool in check_c_dict:
for section in check_c_dict[tool]:
for key in check_c_dict[tool][section]:
if key in environ:
c_dict[tool][section][key] = getenv(key)
# assume the name of the plugin is its directory
plugin_name = path.split('/')[-1]
if plugin_name == '':
plugin_name = path.split('/')[-2]
plugin_config_path = path + '/config/' + plugin_name + '.config'
if exists(plugin_config_path):
plugin_template = Template(plugin_config_path)
plugin_options = c_dict[plugin_name]
for section in plugin_options:
for option in plugin_options[section]:
plugin_template.set_option(section, option,
str(plugin_options[section][option]))
plugin_template.write_config()
config_override = True
except Exception as e: # pragma: no cover
status = (False, str(e))
return status, config_override
def _start_container(self, container, tool_d, s_containers, f_containers):
""" Start container that was passed in and return status """
# use section to add info to manifest
section = tool_d[container]['section']
del tool_d[container]['section']
manifest = Template(self.manifest)
try:
# try to start an existing container first
c = self.d_client.containers.get(container)
c.start()
s_containers.append(container)
manifest.set_option(section, 'running', 'yes')
self.logger.info('started ' + str(container) +
' with ID: ' + str(c.short_id))
except Exception as err:
s_containers, f_containers = self._run_container(
container, tool_d, section, s_containers, f_containers)
# save changes made to manifest
manifest.write_config()
return s_containers, f_containers
def __init__(self, *args, **keywords):
""" Initialize tool form objects """
self.logger = Logger(__name__)
self.api_action = System()
self.tools_inst = Tools()
action = {'api_action': self.tools_inst}
self.tools_tc = {}
self.repo_widgets = {}
if keywords['action_dict']:
action.update(keywords['action_dict'])
if keywords['names']:
i = 1
for name in keywords['names']:
try:
action['action_object' + str(i)] = getattr(
self.tools_inst, name)
except AttributeError:
action['action_object' + str(i)] = getattr(
self.api_action, name)
i += 1
self.action = action
# get list of all possible group views to display
self.views = deque()
possible_groups = set()
manifest = Template(self.api_action.manifest)
tools = self.tools_inst.inventory(choices=['tools'])[1]['tools']
for tool in tools:
groups = manifest.option(tool, 'groups')[1].split(',')
for group in groups:
# don't do core because that's the purpose of all in views
if group != '' and group != 'core':
possible_groups.add(group)
self.manifest = manifest
self.views += possible_groups
self.views.append('all groups')
self.no_instance = ['remove']
super(ToolForm, self).__init__(*args, **keywords)
def __init__(self, *args, **keywords):
""" Initialize tool form objects """
self.logger = Logger(__name__)
self.api_action = System()
self.tools_inst = Tools()
action = {'api_action': self.tools_inst}
self.tools_tc = {}
self.repo_widgets = {}
if keywords['action_dict']:
action.update(keywords['action_dict'])
if keywords['names']:
i = 1
for name in keywords['names']:
try:
action['action_object' +
str(i)] = getattr(self.tools_inst, name)
except AttributeError:
action['action_object' +
str(i)] = getattr(self.api_action, name)
i += 1
self.action = action
# get list of all possible group views to display
self.views = deque()
possible_groups = set()
manifest = Template(self.api_action.manifest)
tools = self.tools_inst.inventory(choices=['tools'])[1]['tools']
for tool in tools:
groups = manifest.option(tool, 'groups')[1].split(',')
for group in groups:
# don't do core because that's the purpose of all in views
if group != '' and group != 'core':
possible_groups.add(group)
self.manifest = manifest
self.views += possible_groups
self.views.append('all groups')
self.no_instance = ['remove']
super(ToolForm, self).__init__(*args, **keywords)
def get_configure(self,
repo=None,
name=None,
groups=None,
main_cfg=False):
"""
Get the vent.template settings for a given tool by looking at the
plugin_manifest
"""
constraints = locals()
del constraints['main_cfg']
status = (True, None)
template_dict = {}
return_str = ''
if main_cfg:
vent_cfg = Template(self.vent_config)
for section in vent_cfg.sections()[1]:
template_dict[section] = {}
for vals in vent_cfg.section(section)[1]:
template_dict[section][vals[0]] = vals[1]
else:
# all possible vent.template options stored in plugin_manifest
options = ['info', 'service', 'settings', 'docker', 'gpu']
tools = Template(System().manifest).constrain_opts(
constraints, options)[0]
if tools:
# should only be one tool
tool = list(tools.keys())[0]
# load all vent.template options into dict
for section in tools[tool]:
template_dict[section] = json.loads(tools[tool][section])
else:
status = (False, "Couldn't get vent.template information")
if status[0]:
# display all those options as they would in the file
for section in template_dict:
return_str += '[' + section + ']\n'
# ensure instances shows up in configuration
for option in template_dict[section]:
if option.startswith('#'):
return_str += option + '\n'
else:
return_str += option + ' = '
return_str += template_dict[section][option] + '\n'
return_str += '\n'
# only one newline at end of file
status = (True, return_str[:-1])
return status
def host_config(self):
""" Ensure the host configuration file exists """
if platform.system() == 'Darwin':
default_file_dir = join(expanduser('~'),
'vent_files')
else:
default_file_dir = '/opt/vent_files'
status = self.ensure_dir(default_file_dir)
if not isfile(self.cfg_file):
config = Template(template=self.cfg_file)
sections = {'main': {'files': default_file_dir},
'network-mapping': {},
'nvidia-docker-plugin': {'port': '3476'}}
for s in sections:
if sections[s]:
for option in sections[s]:
config.add_option(s, option, sections[s][option])
else:
config.add_section(s)
config.write_config()
return status
def override_config(self, path):
"""
Will take a yml located in home directory titled '.plugin_config.yml'.
It'll then override, using the yml, the plugin's config file
"""
status = (True, None)
config_override = False
try:
# parse the yml file
c_dict = {}
if exists(self.plugin_config_file):
with open(self.plugin_config_file, 'r') as config_file:
c_dict = yaml.safe_load(config_file.read())
# check for environment variable overrides
check_c_dict = c_dict.copy()
for tool in check_c_dict:
for section in check_c_dict[tool]:
for key in check_c_dict[tool][section]:
if key in environ:
c_dict[tool][section][key] = getenv(key)
# assume the name of the plugin is its directory
plugin_name = path.split('/')[-1]
if plugin_name == '':
plugin_name = path.split('/')[-2]
plugin_config_path = path + '/config/' + plugin_name + '.config'
if exists(plugin_config_path):
plugin_template = Template(plugin_config_path)
plugin_options = c_dict[plugin_name]
for section in plugin_options:
for option in plugin_options[section]:
plugin_template.set_option(
section, option,
str(plugin_options[section][option]))
plugin_template.write_config()
config_override = True
except Exception as e: # pragma: no cover
status = (False, str(e))
return status, config_override
def _start_container(self, container, tool_d, s_containers, f_containers):
""" Start container that was passed in and return status """
# use section to add info to manifest
section = tool_d[container]['section']
del tool_d[container]['section']
manifest = Template(self.manifest)
try:
# try to start an existing container first
c = self.d_client.containers.get(container)
c.start()
s_containers.append(container)
manifest.set_option(section, 'running', 'yes')
self.logger.info('started ' + str(container) + ' with ID: ' +
str(c.short_id))
except Exception as err:
s_containers, f_containers = self._run_container(
container, tool_d, section, s_containers, f_containers)
# save changes made to manifest
manifest.write_config()
return s_containers, f_containers
def __init__(self,
repo='',
tool_name='',
next_tool=None,
just_downloaded=False,
vent_cfg=False,
from_registry=False,
new_instance=False,
*args,
**keywords):
""" Initialize EditorForm objects """
# default for any editor
self.settings = locals()
self.settings.update(keywords)
del self.settings['self']
del self.settings['args']
del self.settings['keywords']
del self.settings['parentApp']
self.tool_identifier = {'name': tool_name}
self.settings.update(self.tool_identifier)
del self.settings['name']
self.settings['tool_name'] = tool_name
self.settings['next_tool'] = next_tool
self.settings['repo'] = repo
# setup checks
self.just_downloaded = ('just_downloaded' in self.settings
and self.settings['just_downloaded'])
self.vent_cfg = ('vent_cfg' in self.settings
and self.settings['vent_cfg'])
self.registry_tool = ('from_registry' in self.settings
and self.settings['from_registry'])
self.instance_cfg = ('new_instance' in self.settings
and self.settings['new_instance'])
# get manifest info for tool that will be used throughout
if not self.just_downloaded and not self.vent_cfg:
result = Template(System().manifest).constrain_opts(
self.tool_identifier, [])
tool, self.manifest = result
self.section = list(tool.keys())[0]
# get configuration information depending on type
if self.just_downloaded:
self.config_val = '[info]\n'
self.config_val += 'name = ' + keywords['link_name'] + '\n'
self.config_val += 'groups = ' + keywords['groups'] + '\n'
elif self.vent_cfg:
self.config_val = keywords['get_configure'](main_cfg=True)[1]
self.settings['tool_name'] = 'vent configuration'
elif self.instance_cfg:
path = self.manifest.option(self.section, 'path')[1]
multi_tool = self.manifest.option(self.section, 'multi_tool')
if multi_tool[0] and multi_tool[1] == 'yes':
name = self.manifest.option(self.section, 'name')[1]
if name == 'unspecified':
name = 'vent'
template_path = os.path.join(path, name + '.template')
else:
template_path = os.path.join(path, 'vent.template')
# ensure instances is in the editor and that it is the right number
template = Template(template_path)
template.add_section('settings')
template.set_option('settings', 'instances',
str(self.settings['new_instances']))
template.write_config()
with open(template_path, 'r') as vent_template:
self.config_val = vent_template.read()
else:
self.config_val = keywords['get_configure'](
**self.tool_identifier)[1]
super(EditorForm, self).__init__(*args, **keywords)
def add(self, image, link_name, tag=None, registry=None, groups=None):
status = (True, None)
try:
pull_name = image
org = ''
name = image
if '/' in image:
org, name = image.split('/')
else:
org = 'official'
if not tag:
tag = 'latest'
if not registry:
registry = 'docker.io'
if not link_name:
link_name = name
if not groups:
groups = ''
full_image = registry + '/' + image + ':' + tag
image = self.d_client.images.pull(full_image)
section = ':'.join([registry, org, name, '', tag])
namespace = org + '/' + name
# set template section and options for tool at version and branch
template = Template(template=self.manifest)
template.add_section(section)
template.set_option(section, 'name', name)
template.set_option(section, 'pull_name', pull_name)
template.set_option(section, 'namespace', namespace)
template.set_option(section, 'path', '')
template.set_option(section, 'repo', registry + '/' + org)
template.set_option(section, 'branch', '')
template.set_option(section, 'version', tag)
template.set_option(section, 'last_updated',
str(datetime.utcnow()) + ' UTC')
template.set_option(section, 'image_name',
image.attrs['RepoTags'][0])
template.set_option(section, 'type', 'registry')
template.set_option(section, 'link_name', link_name)
template.set_option(section, 'commit_id', '')
template.set_option(section, 'built', 'yes')
template.set_option(section, 'image_id',
image.attrs['Id'].split(':')[1][:12])
template.set_option(section, 'groups', groups)
# write out configuration to the manifest file
template.write_config()
status = (True, 'Successfully added ' + full_image)
except Exception as e: # pragma: no cover
self.logger.error('Failed to add image because: {0}'.format(
str(e)))
status = (False, str(e))
return status
def _build_manifest(self, match, template, repo, org, name, commit_id):
status = (True, None)
# keep track of whether or not to write an additional manifest
# entry for multiple instances, and how many additional entries
# to write
addtl_entries = 1
# remove @ in match for template setting purposes
if match[0].find('@') >= 0:
true_name = match[0].split('@')[1]
else:
true_name = match[0]
# TODO check for special settings here first for the specific match
self.version = match[1]
section = org + ':' + name + ':' + true_name + ':'
section += self.branch + ':' + self.version
# need to get rid of temp identifiers for tools in same repo
match_path = repo + match[0].split('@')[0]
if self.image_name:
image_name = self.image_name
elif not self.core:
image_name = org + '/' + name
if match[0] != '':
# if tool is in a subdir, add that to the name of the
# image
image_name += '-' + '-'.join(match[0].split('/')[1:])
image_name += ':' + self.branch
else:
image_name = ('cyberreboot/vent-' +
match[0].split('/')[-1] + ':' + self.branch)
image_name = image_name.replace('_', '-')
# check if the section already exists
is_there, options = template.section(section)
previous_commit = None
previous_commits = None
head = False
if is_there:
for option in options:
# TODO check if tool name but a different version
# exists - then disable/remove if set
if option[0] == 'version' and option[1] == 'HEAD':
head = True
if option[0] == 'built' and option[1] == 'yes':
# !! TODO remove pre-existing image
pass
if option[0] == 'commit_id':
previous_commit = option[1]
if option[0] == 'previous_versions':
previous_commits = option[1]
# check if tool comes from multi directory
multi_tool = 'no'
if match[0].find('@') >= 0:
multi_tool = 'yes'
# !! TODO
# check if section should be removed from config i.e. all tools
# but new commit removed one that was in a previous commit
image_name = image_name.lower()
image_name = image_name.replace('@', '-')
# special case for vent images
if image_name.startswith('cyberreboot/vent'):
image_name = image_name.replace('vent-vent-core-', 'vent-')
image_name = image_name.replace('vent-vent-extras-', 'vent-')
# set template section & options for tool at version and branch
template.add_section(section)
template.set_option(section, 'name', true_name.split('/')[-1])
template.set_option(section, 'namespace', org + '/' + name)
template.set_option(section, 'path', match_path)
template.set_option(section, 'repo', self.repo)
template.set_option(section, 'multi_tool', multi_tool)
template.set_option(section, 'branch', self.branch)
template.set_option(section, 'version', self.version)
template.set_option(section, 'last_updated',
str(datetime.utcnow()) + ' UTC')
template.set_option(section, 'image_name', image_name)
template.set_option(section, 'type', 'repository')
# save settings in vent.template to plugin_manifest
# watch for multiple tools in same directory
# just wanted to store match path with @ for path for use in
# other actions
tool_template = 'vent.template'
if match[0].find('@') >= 0:
tool_template = match[0].split('@')[1] + '.template'
vent_template_path = join(match_path, tool_template)
if exists(vent_template_path):
with open(vent_template_path, 'r') as f:
vent_template_val = f.read()
else:
vent_template_val = ''
settings_dict = ParsedSections(vent_template_val)
for setting in settings_dict:
template.set_option(section, setting,
json.dumps(settings_dict[setting]))
# TODO do we need this if we save as a dictionary?
vent_template = Template(vent_template_path)
vent_status, response = vent_template.option('info', 'name')
instances = vent_template.option('settings', 'instances')
if instances[0]:
addtl_entries = int(instances[1])
if vent_status:
template.set_option(section, 'link_name', response)
else:
template.set_option(section,
'link_name',
true_name.split('/')[-1])
if self.version == 'HEAD':
template.set_option(section, 'commit_id', commit_id)
if head:
# no need to store previous commits if not HEAD, since
# the version will always be the same commit ID
if previous_commit and previous_commit != commit_id:
if (previous_commits and
previous_commit not in previous_commits):
previous_commits = (previous_commit +
',' +
previous_commits)
elif not previous_commits:
previous_commits = previous_commit
if previous_commits and previous_commits != commit_id:
template.set_option(section,
'previous_versions',
previous_commits)
groups = vent_template.option('info', 'groups')
if groups[0]:
template.set_option(section, 'groups', groups[1])
# set groups to empty string if no groups defined for tool
else:
template.set_option(section, 'groups', '')
# write additional entries for multiple instances
if addtl_entries > 1:
# add 2 for naming conventions
for i in range(2, addtl_entries + 1):
addtl_section = section.rsplit(':', 2)
addtl_section[0] += str(i)
addtl_section = ':'.join(addtl_section)
template.add_section(addtl_section)
orig_vals = template.section(section)[1]
for val in orig_vals:
template.set_option(addtl_section, val[0], val[1])
template.set_option(addtl_section, 'name',
true_name.split('/')[-1]+str(i))
return status, template, match_path, image_name, section
def create(self, group_view=False):
""" Update with current tools """
self.add_handlers({'^T': self.quit, '^Q': self.quit})
self.add(npyscreen.TitleText,
name='Select which tools to ' + self.action['action'] + ':',
editable=False)
togglable = ['remove']
if self.action['action_name'] in togglable:
self.cur_view = self.add(npyscreen.TitleText,
name='Group view:',
value='all groups', editable=False,
rely=3)
self.add_handlers({'^V': self.toggle_view})
i = 5
else:
i = 4
if self.action['action_name'] == 'start':
response = self.tools_inst.inventory(choices=['repos',
'tools',
'built',
'running'])
else:
response = self.tools_inst.inventory(choices=['repos', 'tools'])
if response[0]:
inventory = response[1]
repos = inventory['repos']
# dict has repo as key and list of core/non-core tools as values
has_core = {}
has_non_core = {}
# find all tools that are in this repo
# and list them if they are core
for repo in repos:
core_list = []
ncore_list = []
# splice the repo names for processing
if (repo.startswith('http')):
repo_name = repo.rsplit('/', 2)[1:]
else:
repo_name = repo.split('/')
for tool in inventory['tools']:
tool_repo_name = tool.split(':')
# cross reference repo names
if (repo_name[0] == tool_repo_name[0] and
repo_name[1] == tool_repo_name[1]):
# check to ensure tool not set to locally active = no
# in vent.cfg
externally_active = False
vent_cfg_file = self.api_action.vent_config
vent_cfg = Template(vent_cfg_file)
tool_pairs = vent_cfg.section('external-services')[1]
for ext_tool in tool_pairs:
if ext_tool[0].lower() == inventory['tools'][tool]:
try:
ext_tool_options = json.loads(ext_tool[1])
loc = 'locally_active'
if (loc in ext_tool_options and
ext_tool_options[loc] == 'no'):
externally_active = True
except Exception as e:
self.logger.error("Couldn't check ext"
' because: ' + str(e))
externally_active = False
manifest = Template(self.api_action.manifest)
if not externally_active:
instance_num = re.search(r'\d+$',
manifest.option(
tool, 'name')[1])
if not instance_num:
ncore_list.append(tool)
# multiple instances share same image
elif self.action['action_name'] not in self.no_instance:
ncore_list.append(tool)
has_core[repo] = core_list
has_non_core[repo] = ncore_list
for repo in repos:
self.tools_tc[repo] = {}
if self.action['cores']:
# make sure only repos with core tools are displayed
if has_core.get(repo):
self.repo_widgets[repo] = self.add(npyscreen.TitleText,
name='Plugin: '+repo,
editable=False,
rely=i, relx=5)
for tool in has_core[repo]:
tool_name = tool.split(':', 2)[2].split('/')[-1]
if tool_name == '':
tool_name = '/'
self.tools_tc[repo][tool] = self.add(
npyscreen.CheckBox, name=tool_name,
value=True, relx=10)
i += 1
i += 3
else:
# make sure only repos with non-core tools are displayed
if has_non_core.get(repo):
self.repo_widgets[repo] = self.add(npyscreen.TitleText,
name='Plugin: '+repo,
editable=False,
rely=i, relx=5)
for tool in has_non_core[repo]:
tool_name = tool.split(':', 2)[2].split('/')[-1]
if tool_name == '':
tool_name = '/'
self.tools_tc[repo][tool] = self.add(
npyscreen.CheckBox, name=tool_name,
value=True, relx=10)
i += 1
i += 3
return
def _run_container(self, container, tool_d, section, s_containers,
f_containers):
manifest = Template(self.manifest)
try:
gpu = 'gpu.enabled'
failed = False
if (gpu in tool_d[container]['labels']
and tool_d[container]['labels'][gpu] == 'yes'):
vent_config = Template(template=self.path_dirs.cfg_file)
port = ''
host = ''
result = vent_config.option('nvidia-docker-plugin', 'port')
if result[0]:
port = result[1]
else:
port = '3476'
result = vent_config.option('nvidia-docker-plugin', 'host')
if result[0]:
host = result[1]
else:
# now just requires ip, ifconfig
try:
route = check_output(
('ip', 'route')).decode('utf-8').split('\n')
default = ''
# grab the default network device.
for device in route:
if 'default' in device:
default = device.split()[4]
break
# grab the IP address for the default device
ip_addr = check_output(
('ifconfig', default)).decode('utf-8')
ip_addr = ip_addr.split('\n')[1].split()[1]
host = ip_addr
except Exception as e: # pragma no cover
self.logger.error('failed to grab ip. Ensure that \
ip and ifconfig are installed')
nd_url = 'http://' + host + ':' + port + '/v1.0/docker/cli'
params = {'vol': 'nvidia_driver'}
r = requests.get(nd_url, params=params)
if r.status_code == 200:
options = r.text.split()
for option in options:
if option.startswith('--volume-driver='):
tool_d[container]['volume_driver'] = option.split(
'=', 1)[1]
elif option.startswith('--volume='):
vol = option.split('=', 1)[1].split(':')
if 'volumes' in tool_d[container]:
if isinstance(tool_d[container]['volumes'],
list):
if len(vol) == 2:
c_vol = vol[0] + \
':' + vol[1] + ':rw'
else:
c_vol = vol[0] + ':' + \
vol[1] + ':' + vol[2]
tool_d[container]['volumes'].append(c_vol)
else: # Dictionary
tool_d[container]['volumes'][vol[0]] = {
'bind': vol[1],
'mode': vol[2]
}
else:
tool_d[container]['volumes'] = {
vol[0]: {
'bind': vol[1],
'mode': vol[2]
}
}
elif option.startswith('--device='):
dev = option.split('=', 1)[1]
if 'devices' in tool_d[container]:
tool_d[container]['devices'].append(dev + ':' +
dev +
':rwm')
else:
tool_d[container]['devices'] = [
dev + ':' + dev + ':rwm'
]
else:
self.logger.error('Unable to parse ' +
'nvidia-docker option: ' +
str(option))
else:
failed = True
f_containers.append(container)
manifest.set_option(section, 'running', 'failed')
self.logger.error('failed to start ' + str(container) +
' because nvidia-docker-plugin ' +
'failed with: ' + str(r.status_code))
if not failed:
try:
self.d_client.containers.remove(container, force=True)
self.logger.info('removed old existing container: ' +
str(container))
except Exception as e:
pass
cont_id = self.d_client.containers.run(detach=True,
**tool_d[container])
s_containers.append(container)
manifest.set_option(section, 'running', 'yes')
self.logger.info('started ' + str(container) + ' with ID: ' +
str(cont_id))
except Exception as e: # pragma: no cover
f_containers.append(container)
manifest.set_option(section, 'running', 'failed')
self.logger.error('failed to start ' + str(container) +
' because: ' + str(e))
return s_containers, f_containers
def _auto_install(self):
"""
Automatically detects images and installs them in the manifest if they
are not there already
"""
template = Template(template=self.manifest)
sections = template.sections()
images = self.d_client.images.list(filters={'label': 'vent'})
add_sections = []
status = (True, None)
for image in images:
ignore = False
if ('Labels' in image.attrs['Config'] and
'vent.section' in image.attrs['Config']['Labels'] and
not image.attrs['Config']['Labels']['vent.section'] in sections[1]):
section = image.attrs['Config']['Labels']['vent.section']
section_str = image.attrs['Config']['Labels']['vent.section'].split(
':')
template.add_section(section)
if 'vent.name' in image.attrs['Config']['Labels']:
template.set_option(section,
'name',
image.attrs['Config']['Labels']['vent.name'])
if 'vent.repo' in image.attrs['Config']['Labels']:
template.set_option(section,
'repo',
image.attrs['Config']['Labels']['vent.repo'])
git_path = join(self.path_dirs.plugins_dir,
'/'.join(section_str[:2]))
# TODO clone it down
template.set_option(section, 'path', join(
git_path, section_str[-3][1:]))
# get template settings
# TODO account for template files not named vent.template
v_template = Template(template=join(
git_path, section_str[-3][1:], 'vent.template'))
tool_sections = v_template.sections()
if tool_sections[0]:
for s in tool_sections[1]:
section_dict = {}
options = v_template.options(s)
if options[0]:
for option in options[1]:
option_name = option
if option == 'name':
# get link name
template.set_option(section,
'link_name',
v_template.option(s, option)[1])
option_name = 'link_name'
opt_val = v_template.option(s, option)[1]
section_dict[option_name] = opt_val
if section_dict:
template.set_option(section, s,
json.dumps(section_dict))
if ('vent.type' in image.attrs['Config']['Labels'] and
image.attrs['Config']['Labels']['vent.type'] == 'repository'):
template.set_option(
section, 'namespace', '/'.join(section_str[:2]))
template.set_option(section, 'branch', section_str[-2])
template.set_option(section, 'version', section_str[-1])
template.set_option(section, 'last_updated', str(
datetime.utcnow()) + ' UTC')
if image.attrs['RepoTags']:
template.set_option(
section, 'image_name', image.attrs['RepoTags'][0])
else:
# image with none tag is outdated, don't add it
ignore = True
template.set_option(section, 'type', 'repository')
if 'vent.groups' in image.attrs['Config']['Labels']:
template.set_option(section,
'groups',
image.attrs['Config']['Labels']['vent.groups'])
template.set_option(section, 'built', 'yes')
template.set_option(section, 'image_id',
image.attrs['Id'].split(':')[1][:12])
template.set_option(section, 'running', 'no')
# check if image is running as a container
containers = self.d_client.containers.list(
filters={'label': 'vent'})
for container in containers:
if container.attrs['Image'] == image.attrs['Id']:
template.set_option(section, 'running', 'yes')
if not ignore:
add_sections.append(section)
template.write_config()
# TODO this check will always be true, need to actually validate the above logic
if status[0]:
status = (True, add_sections)
return status
def _run_container(self, container, tool_d, section, s_containers, f_containers):
manifest = Template(self.manifest)
try:
gpu = 'gpu.enabled'
failed = False
if (gpu in tool_d[container]['labels'] and
tool_d[container]['labels'][gpu] == 'yes'):
vent_config = Template(template=self.path_dirs.cfg_file)
port = ''
host = ''
result = vent_config.option('nvidia-docker-plugin', 'port')
if result[0]:
port = result[1]
else:
port = '3476'
result = vent_config.option('nvidia-docker-plugin', 'host')
if result[0]:
host = result[1]
else:
# now just requires ip, ifconfig
try:
route = check_output(('ip', 'route')).decode(
'utf-8').split('\n')
default = ''
# grab the default network device.
for device in route:
if 'default' in device:
default = device.split()[4]
break
# grab the IP address for the default device
ip_addr = check_output(
('ifconfig', default)).decode('utf-8')
ip_addr = ip_addr.split('\n')[1].split()[1]
host = ip_addr
except Exception as e: # pragma no cover
self.logger.error('failed to grab ip. Ensure that \
ip and ifconfig are installed')
nd_url = 'http://' + host + ':' + port + '/v1.0/docker/cli'
params = {'vol': 'nvidia_driver'}
r = requests.get(nd_url, params=params)
if r.status_code == 200:
options = r.text.split()
for option in options:
if option.startswith('--volume-driver='):
tool_d[container]['volume_driver'] = option.split('=', 1)[
1]
elif option.startswith('--volume='):
vol = option.split('=', 1)[1].split(':')
if 'volumes' in tool_d[container]:
if isinstance(tool_d[container]['volumes'], list):
if len(vol) == 2:
c_vol = vol[0] + \
':' + vol[1] + ':rw'
else:
c_vol = vol[0] + ':' + \
vol[1] + ':' + vol[2]
tool_d[container]['volumes'].append(
c_vol)
else: # Dictionary
tool_d[container]['volumes'][vol[0]] = {'bind': vol[1],
'mode': vol[2]}
else:
tool_d[container]['volumes'] = {vol[0]:
{'bind': vol[1],
'mode': vol[2]}}
elif option.startswith('--device='):
dev = option.split('=', 1)[1]
if 'devices' in tool_d[container]:
tool_d[container]['devices'].append(dev +
':' +
dev +
':rwm')
else:
tool_d[container]['devices'] = [
dev + ':' + dev + ':rwm']
else:
self.logger.error('Unable to parse ' +
'nvidia-docker option: ' +
str(option))
else:
failed = True
f_containers.append(container)
manifest.set_option(section, 'running', 'failed')
self.logger.error('failed to start ' + str(container) +
' because nvidia-docker-plugin ' +
'failed with: ' + str(r.status_code))
if not failed:
try:
self.d_client.containers.remove(container, force=True)
self.logger.info(
'removed old existing container: ' + str(container))
except Exception as e:
pass
cont_id = self.d_client.containers.run(detach=True,
**tool_d[container])
s_containers.append(container)
manifest.set_option(section, 'running', 'yes')
self.logger.info('started ' + str(container) +
' with ID: ' + str(cont_id))
except Exception as e: # pragma: no cover
f_containers.append(container)
manifest.set_option(section, 'running', 'failed')
self.logger.error('failed to start ' + str(container) +
' because: ' + str(e))
return s_containers, f_containers
def test_add_option():
""" Test the add_option function """
instance = Template()
instance.add_option('foo', 'bar')
instance.add_option('foo', 'bar')
instance.add_option('bad', 'x')
def _auto_install(self):
"""
Automatically detects images and installs them in the manifest if they
are not there already
"""
template = Template(template=self.manifest)
sections = template.sections()
images = self.d_client.images.list(filters={'label': 'vent'})
add_sections = []
status = (True, None)
for image in images:
ignore = False
if ('Labels' in image.attrs['Config'] and
'vent.section' in image.attrs['Config']['Labels'] and
not image.attrs['Config']['Labels']['vent.section'] in sections[1]):
section = image.attrs['Config']['Labels']['vent.section']
section_str = image.attrs['Config']['Labels']['vent.section'].split(
':')
template.add_section(section)
if 'vent.name' in image.attrs['Config']['Labels']:
template.set_option(section,
'name',
image.attrs['Config']['Labels']['vent.name'])
if 'vent.repo' in image.attrs['Config']['Labels']:
template.set_option(section,
'repo',
image.attrs['Config']['Labels']['vent.repo'])
git_path = join(self.path_dirs.plugins_dir,
'/'.join(section_str[:2]))
# TODO clone it down
template.set_option(section, 'path', join(
git_path, section_str[-3][1:]))
# get template settings
# TODO account for template files not named vent.template
v_template = Template(template=join(
git_path, section_str[-3][1:], 'vent.template'))
tool_sections = v_template.sections()
if tool_sections[0]:
for s in tool_sections[1]:
section_dict = {}
options = v_template.options(s)
if options[0]:
for option in options[1]:
option_name = option
if option == 'name':
# get link name
template.set_option(section,
'link_name',
v_template.option(s, option)[1])
option_name = 'link_name'
opt_val = v_template.option(s, option)[1]
section_dict[option_name] = opt_val
if section_dict:
template.set_option(section, s,
json.dumps(section_dict))
if ('vent.type' in image.attrs['Config']['Labels'] and
image.attrs['Config']['Labels']['vent.type'] == 'repository'):
template.set_option(
section, 'namespace', '/'.join(section_str[:2]))
template.set_option(section, 'branch', section_str[-2])
template.set_option(section, 'version', section_str[-1])
template.set_option(section, 'last_updated', str(
datetime.utcnow()) + ' UTC')
if image.attrs['RepoTags']:
template.set_option(
section, 'image_name', image.attrs['RepoTags'][0])
else:
# image with none tag is outdated, don't add it
ignore = True
template.set_option(section, 'type', 'repository')
if 'vent.groups' in image.attrs['Config']['Labels']:
template.set_option(section,
'groups',
image.attrs['Config']['Labels']['vent.groups'])
template.set_option(section, 'built', 'yes')
template.set_option(section, 'image_id',
image.attrs['Id'].split(':')[1][:12])
template.set_option(section, 'running', 'no')
# check if image is running as a container
containers = self.d_client.containers.list(
filters={'label': 'vent'})
for container in containers:
if container.attrs['Image'] == image.attrs['Id']:
template.set_option(section, 'running', 'yes')
if not ignore:
add_sections.append(section)
template.write_config()
# TODO this check will always be true, need to actually validate the above logic
if status[0]:
status = (True, add_sections)
return status
def _start_sections(self, s, files):
tool_d = {}
status = (True, None)
for section in s:
# initialize needed vars
c_name = s[section]['image_name'].replace(':', '-')
c_name = c_name.replace('/', '-')
instance_num = re.search(r'\d+$', s[section]['name'])
if instance_num:
c_name += instance_num.group()
image_name = s[section]['image_name']
# checkout the right version and branch of the repo
tool_d[c_name] = {'image': image_name, 'name': c_name}
# get rid of all commented sections in various runtime
# configurations
manifest = Template(self.manifest)
overall_dict = {}
for setting in ['info', 'docker', 'gpu', 'settings', 'service']:
option = manifest.option(section, setting)
if option[0]:
overall_dict[setting] = {}
settings_dict = json.loads(option[1])
for opt in settings_dict:
if not opt.startswith('#'):
overall_dict[setting][opt] = settings_dict[opt]
if 'docker' in overall_dict:
options_dict = overall_dict['docker']
for option in options_dict:
options = options_dict[option]
# check for commands to evaluate
if '`' in options:
cmds = options.split('`')
if len(cmds) > 2:
i = 1
while i < len(cmds):
try:
cmds[i] = check_output(
shlex.split(cmds[i]),
stderr=STDOUT,
close_fds=True).strip().decode('utf-8')
except Exception as e: # pragma: no cover
self.logger.error(
'unable to evaluate command specified in vent.template: '
+ str(e))
i += 2
options = ''.join(cmds)
# check for commands to evaluate
# store options set for docker
try:
tool_d[c_name][option] = ast.literal_eval(options)
except Exception as e: # pragma: no cover
self.logger.debug('Unable to literal_eval: {0}'.format(
str(options)))
tool_d[c_name][option] = options
if 'labels' not in tool_d[c_name]:
tool_d[c_name]['labels'] = {}
# get the service uri info
if 'service' in overall_dict:
try:
options_dict = overall_dict['service']
for option in options_dict:
tool_d[c_name]['labels'][option] = options_dict[option]
except Exception as e: # pragma: no cover
self.logger.error('unable to store service options for '
'docker: ' + str(e))
# check for gpu settings
if 'gpu' in overall_dict:
try:
options_dict = json.loads(status[1])
for option in options_dict:
tool_d[c_name]['labels']['gpu.' +
option] = options_dict[option]
except Exception as e: # pragma: no cover
self.logger.error('unable to store gpu options for '
'docker: ' + str(e))
# get temporary name for links, etc.
plugin_c = Template(template=self.manifest)
status, plugin_sections = plugin_c.sections()
for plugin_section in plugin_sections:
status = plugin_c.option(plugin_section, 'link_name')
image_status = plugin_c.option(plugin_section, 'image_name')
if status[0] and image_status[0]:
cont_name = image_status[1].replace(':', '-')
cont_name = cont_name.replace('/', '-')
if cont_name not in tool_d:
tool_d[cont_name] = {
'image': image_status[1],
'name': cont_name,
'start': False
}
tool_d[cont_name]['tmp_name'] = status[1]
# add extra labels
tool_d[c_name]['labels']['vent'] = Version()
tool_d[c_name]['labels']['vent.namespace'] = s[section][
'namespace']
tool_d[c_name]['labels']['vent.branch'] = s[section]['branch']
tool_d[c_name]['labels']['vent.version'] = s[section]['version']
tool_d[c_name]['labels']['vent.name'] = s[section]['name']
tool_d[c_name]['labels']['vent.section'] = section
tool_d[c_name]['labels']['vent.repo'] = s[section]['repo']
tool_d[c_name]['labels']['vent.type'] = s[section]['type']
# check for log_config settings in external-services
externally_configured = False
vent_config = Template(self.path_dirs.cfg_file)
for ext_tool in vent_config.section('external-services')[1]:
if ext_tool[0].lower() == 'syslog':
try:
log_dict = json.loads(ext_tool[1])
# configure if not locally active
if ('locally_active' not in log_dict
or log_dict['locally_active'] == 'no'):
del log_dict['locally_active']
log_config = {}
log_config['type'] = 'syslog'
log_config['config'] = {}
ip_address = ''
port = ''
for option in log_dict:
if option == 'ip_address':
ip_address = log_dict[option]
elif option == 'port':
port = log_dict['port']
syslog_address = 'tcp://' + ip_address + ':' + port
syslog_config = {
'syslog-address': syslog_address,
'syslog-facility': 'daemon',
'tag': '{{.Name}}'
}
log_config['config'].update(syslog_config)
externally_configured = True
except Exception as e: # pragma: no cover
self.logger.error('external settings for log_config'
" couldn't be stored because: " +
str(e))
externally_configured = False
if not externally_configured:
log_config = {
'type': 'syslog',
'config': {
'syslog-address': 'tcp://0.0.0.0:514',
'syslog-facility': 'daemon',
'tag': '{{.Name}}'
}
}
if 'groups' in s[section]:
# add labels for groups
tool_d[c_name]['labels']['vent.groups'] = s[section]['groups']
# add restart=always to core containers
if 'core' in s[section]['groups']:
tool_d[c_name]['restart_policy'] = {'Name': 'always'}
# map network names to environment variables
if 'network' in s[section]['groups']:
vent_config = Template(template=self.path_dirs.cfg_file)
nic_mappings = vent_config.section('network-mapping')
nics = ''
if nic_mappings[0]:
for nic in nic_mappings[1]:
nics += nic[0] + ':' + nic[1] + ','
nics = nics[:-1]
if nics:
if 'environment' in tool_d[c_name]:
tool_d[c_name]['environment'].append('VENT_NICS=' +
nics)
else:
tool_d[c_name]['environment'] = [
'VENT_NICS=' + nics
]
# send logs to syslog
if ('syslog' not in s[section]['groups']
and 'core' in s[section]['groups']):
log_config['config']['tag'] = '{{.Name}}'
tool_d[c_name]['log_config'] = log_config
if 'syslog' not in s[section]['groups']:
tool_d[c_name]['log_config'] = log_config
# mount necessary directories
if 'files' in s[section]['groups']:
ulimits = []
ulimits.append(
docker.types.Ulimit(name='nofile',
soft=1048576,
hard=1048576))
tool_d[c_name]['ulimits'] = ulimits
# check if running in a docker container
if 'VENT_CONTAINERIZED' in environ and environ[
'VENT_CONTAINERIZED'] == 'true':
if 'volumes_from' in tool_d[c_name]:
tool_d[c_name]['volumes_from'].append(
environ['HOSTNAME'])
else:
tool_d[c_name]['volumes_from'] = [
environ['HOSTNAME']
]
else:
if 'volumes' in tool_d[c_name]:
tool_d[c_name]['volumes'][
self.path_dirs.base_dir[:-1]] = {
'bind': '/vent',
'mode': 'ro'
}
else:
tool_d[c_name]['volumes'] = {
self.path_dirs.base_dir[:-1]: {
'bind': '/vent',
'mode': 'ro'
}
}
if files[0]:
if 'volumes' in tool_d[c_name]:
tool_d[c_name]['volumes'][files[1]] = {
'bind': '/files',
'mode': 'rw'
}
else:
tool_d[c_name]['volumes'] = {
files[1]: {
'bind': '/files',
'mode': 'rw'
}
}
else:
tool_d[c_name]['log_config'] = log_config
# add label for priority
if 'settings' in overall_dict:
try:
options_dict = overall_dict['settings']
for option in options_dict:
if option == 'priority':
tool_d[c_name]['labels'][
'vent.priority'] = options_dict[option]
except Exception as e: # pragma: no cover
self.logger.error('unable to store settings options '
'for docker ' + str(e))
# only start tools that have been built
if s[section]['built'] != 'yes':
del tool_d[c_name]
# store section information for adding info to manifest later
else:
tool_d[c_name]['section'] = section
return status, tool_d
def _startup(self):
"""
Automatically detect if a startup file is specified and stand up a vent
host with all necessary tools based on the specifications in that file
"""
status = (True, None)
try:
s_dict = {}
# rewrite the yml file to exclusively lowercase
with open(self.startup_file, 'r') as sup:
vent_startup = sup.read()
with open(self.startup_file, 'w') as sup:
for line in vent_startup:
sup.write(line.lower())
with open(self.startup_file, 'r') as sup:
s_dict = yaml.safe_load(sup.read())
if 'vent.cfg' in s_dict:
v_cfg = Template(self.vent_config)
for section in s_dict['vent.cfg']:
for option in s_dict['vent.cfg'][section]:
val = ('no', 'yes')[
s_dict['vent.cfg'][section][option]]
v_status = v_cfg.add_option(section, option, value=val)
if not v_status[0]:
v_cfg.set_option(section, option, val)
v_cfg.write_config()
del s_dict['vent.cfg']
tool_d = {}
extra_options = ['info', 'service', 'settings', 'docker', 'gpu']
s_dict_c = copy.deepcopy(s_dict)
# TODO check for repo or image type
for repo in s_dict_c:
repository = Repository(System().manifest)
repository.repo = repo
repository._clone()
repo_path, org, r_name = self.path_dirs.get_path(repo)
get_tools = []
for tool in s_dict_c[repo]:
t_branch = 'master'
t_version = 'HEAD'
if 'branch' in s_dict[repo][tool]:
t_branch = s_dict[repo][tool]['branch']
if 'version' in s_dict[repo][tool]:
t_version = s_dict[repo][tool]['version']
get_tools.append((tool, t_branch, t_version))
available_tools = AvailableTools(repo_path, tools=get_tools)
for tool in s_dict_c[repo]:
# if we can't find the tool in that repo, skip over this
# tool and notify in the logs
t_path, t_path_cased = PathDirs.rel_path(
tool, available_tools)
if t_path is None:
self.logger.error("Couldn't find tool " + tool + ' in'
' repo ' + repo)
continue
# ensure no NoneType iteration errors
if s_dict_c[repo][tool] is None:
s_dict[repo][tool] = {}
# check if we need to configure instances along the way
instances = 1
if 'settings' in s_dict[repo][tool]:
if 'instances' in s_dict[repo][tool]['settings']:
instances = int(s_dict[repo][tool]
['settings']['instances'])
# add the tool
t_branch = 'master'
t_version = 'HEAD'
t_image = None
add_tools = None
add_tools = [(t_path_cased, '')]
if 'branch' in s_dict[repo][tool]:
t_branch = s_dict[repo][tool]['branch']
if 'version' in s_dict[repo][tool]:
t_version = s_dict[repo][tool]['version']
if 'image' in s_dict[repo][tool]:
t_image = s_dict[repo][tool]['image']
repository.add(
repo, tools=add_tools, branch=t_branch, version=t_version, image_name=t_image)
manifest = Template(self.manifest)
# update the manifest with extra defined runtime settings
base_section = ':'.join([org, r_name, t_path,
t_branch, t_version])
for option in extra_options:
if option in s_dict[repo][tool]:
opt_dict = manifest.option(base_section, option)
# add new values defined into default options for
# that tool, don't overwrite them
if opt_dict[0]:
opt_dict = json.loads(opt_dict[1])
else:
opt_dict = {}
# stringify values for vent
for v in s_dict[repo][tool][option]:
pval = s_dict[repo][tool][option][v]
s_dict[repo][tool][option][v] = json.dumps(
pval)
opt_dict.update(s_dict[repo][tool][option])
manifest.set_option(base_section, option,
json.dumps(opt_dict))
# copy manifest info into new sections if necessary
if instances > 1:
for i in range(2, instances + 1):
i_section = base_section.rsplit(':', 2)
i_section[0] += str(i)
i_section = ':'.join(i_section)
manifest.add_section(i_section)
for opt_val in manifest.section(base_section)[1]:
if opt_val[0] == 'name':
manifest.set_option(i_section, opt_val[0],
opt_val[1] + str(i))
else:
manifest.set_option(i_section, opt_val[0],
opt_val[1])
manifest.write_config()
tool_d = {}
tools = Tools()
# start tools, if necessary
for repo in s_dict:
for tool in s_dict[repo]:
if 'start' in s_dict[repo][tool]:
if s_dict[repo][tool]['start']:
local_instances = 1
if 'settings' in s_dict[repo][tool] and 'instances' in s_dict[repo][tool]['settings']:
local_instances = int(
s_dict[repo][tool]['settings']['instances'])
t_branch = 'master'
t_version = 'HEAD'
if 'branch' in s_dict[repo][tool]:
t_branch = s_dict[repo][tool]['branch']
if 'version' in s_dict[repo][tool]:
t_version = s_dict[repo][tool]['version']
for i in range(1, local_instances + 1):
i_name = tool + str(i) if i != 1 else tool
i_name = i_name.replace('@', '')
tool_d.update(
tools._prep_start(repo, i_name)[1])
if tool_d:
tools.start(tool_d, None, is_tool_d=True)
except Exception as e: # pragma: no cover
self.logger.error('Startup failed because: {0}'.format(str(e)))
status = (False, str(e))
return status
def save_configure(self,
repo=None,
name=None,
groups=None,
config_val='',
from_registry=False,
main_cfg=False,
instances=1):
"""
Save changes made to vent.template through npyscreen to the template
and to plugin_manifest
"""
def template_to_manifest(vent_template, manifest, tool, instances):
"""
Helper function to transfer information from vent.template
to plugin_manifest
"""
sections = vent_template.sections()
if sections[0]:
for section in sections[1]:
section_dict = {}
if section == 'settings':
section_dict.update({'instances': str(instances)})
options = vent_template.options(section)
if options[0]:
for option in options[1]:
option_name = option
if option == 'name':
option_name = 'link_name'
opt_val = vent_template.option(section,
option)[1]
section_dict[option_name] = opt_val
if section_dict:
manifest.set_option(tool, section,
json.dumps(section_dict))
elif manifest.option(tool, section)[0]:
manifest.del_option(tool, section)
constraints = locals()
del constraints['config_val']
del constraints['from_registry']
del constraints['main_cfg']
del constraints['instances']
del constraints['template_to_manifest']
status = (True, None)
fd = None
# ensure instances is an int and remove instances from config_val to
# ensure correct info
instances = int(instances)
config_val = re.sub(r'instances\ *=\ *\d+\n', '', config_val)
api_system = System()
manifest = api_system.manifest
if not main_cfg:
if not from_registry:
# creating new instances
if instances > 1:
fd, template_path = tempfile.mkstemp(suffix='.template')
# scrub name for clean section name
if re.search(r'\d+$', name):
name = re.sub(r'\d+$', '', name)
t_identifier = {'name': name,
'branch': branch,
'version': version}
result = Template(manifest).constrain_opts(
t_identifier, [])
tools = result[0]
tool = list(tools.keys())[0]
else:
options = ['path', 'multi_tool', 'name']
tools, _ = Template(manifest).constrain_opts(
constraints, options)
# only one tool in tools because perform this function for
# every tool
if tools:
tool = list(tools.keys())[0]
if ('multi_tool' in tools[tool] and
tools[tool]['multi_tool'] == 'yes'):
name = tools[tool]['name']
if name == 'unspecified':
name = 'vent'
template_path = join(tools[tool]['path'],
name+'.template')
else:
template_path = join(tools[tool]['path'],
'vent.template')
else:
status = (False, "Couldn't save configuration")
else:
fd, template_path = tempfile.mkstemp(suffix='.template')
options = ['namespace']
constraints.update({'type': 'registry'})
tools, _ = Template(manifest).constrain_opts(constraints,
options)
if tools:
tool = list(tools.keys())[0]
else:
status = (False, "Couldn't save configuration")
if status[0]:
try:
# save in vent.template
with open(template_path, 'w') as f:
f.write(config_val)
# save in plugin_manifest
vent_template = Template(template_path)
manifest = Template(manifest)
if instances > 1:
# add instances as needed
for i in range(1, instances + 1):
i_section = tool.rsplit(':', 2)
i_section[0] += str(i) if i != 1 else ''
i_section = ':'.join(i_section)
if not manifest.section(i_section)[0]:
manifest.add_section(i_section)
for val_pair in manifest.section(tool)[1]:
name = val_pair[0]
val = val_pair[1]
if name == 'name':
val += str(i)
elif name == 'last_updated':
val = Timestamp()
elif name == 'running':
val = 'no'
manifest.set_option(i_section, name, val)
template_to_manifest(vent_template, manifest,
i_section, instances)
else:
settings = manifest.option(i_section,
'settings')
if settings[0]:
settings_dict = json.loads(settings[1])
settings_dict['instances'] = str(instances)
manifest.set_option(i_section, 'settings',
json.dumps(
settings_dict))
else:
inst = str(instances)
settings_dict = {'instances': inst}
manifest.set_option(i_section, 'settings',
json.dumps(
settings_dict))
else:
try:
settings_str = manifest.option(tool, 'settings')[1]
settings_dict = json.loads(settings_str)
old_instances = int(settings_dict['instances'])
except Exception:
old_instances = 1
template_to_manifest(vent_template, manifest,
tool, old_instances)
manifest.write_config()
status = (True, manifest)
except Exception as e: # pragma: no cover
self.logger.error('save_configure error: ' + str(e))
status = (False, str(e))
# close os file handle and remove temp file
if from_registry or instances > 1:
try:
close(fd)
remove(template_path)
except Exception as e: # pragma: no cover
self.logger.error('save_configure error: ' + str(e))
else:
with open(self.vent_config, 'w') as f:
f.write(config_val)
return status
def test_option():
""" Test the option function """
instance = Template()
instance.option('foo', 'bar')
def test_options():
""" Test the options function """
instance = Template()
instance.options('foo')
def restart_tools(self,
repo=None,
name=None,
groups=None,
branch='master',
version='HEAD',
main_cfg=False,
old_val='',
new_val=''):
"""
Restart necessary tools based on changes that have been made either to
vent.cfg or to vent.template. This includes tools that need to be
restarted because they depend on other tools that were changed.
"""
status = (True, None)
if not main_cfg:
try:
t_identifier = {'name': name,
'branch': branch,
'version': version}
result = Template(System().manifest).constrain_opts(t_identifier,
['running',
'link_name'])
tools = result[0]
tool = list(tools.keys())[0]
if ('running' in tools[tool] and
tools[tool]['running'] == 'yes'):
start_tools = [t_identifier]
dependent_tools = [tools[tool]['link_name']]
start_tools += Dependencies(dependent_tools)
# TODO
start_d = {}
for tool_identifier in start_tools:
self.clean(**tool_identifier)
start_d.update(self.prep_start(**tool_identifier)[1])
if start_d:
Tools().start(start_d, '', is_tool_d=True)
except Exception as e: # pragma: no cover
self.logger.error('Trouble restarting tool ' + name +
' because: ' + str(e))
status = (False, str(e))
else:
try:
# string manipulation to get tools into arrays
ext_start = old_val.find('[external-services]')
if ext_start >= 0:
ot_str = old_val[old_val.find('[external-services]') + 20:]
else:
ot_str = ''
old_tools = []
for old_tool in ot_str.split('\n'):
if old_tool != '':
old_tools.append(old_tool.split('=')[0].strip())
ext_start = new_val.find('[external-services]')
if ext_start >= 0:
nt_str = new_val[new_val.find('[external-services]') + 20:]
else:
nt_str = ''
new_tools = []
for new_tool in nt_str.split('\n'):
if new_tool != '':
new_tools.append(new_tool.split('=')[0].strip())
# find tools changed
tool_changes = []
for old_tool in old_tools:
if old_tool not in new_tools:
tool_changes.append(old_tool)
for new_tool in new_tools:
if new_tool not in old_tools:
tool_changes.append(new_tool)
else:
# tool name will be the same
oconf = old_val[old_val.find(new_tool):].split('\n')[0]
nconf = new_val[new_val.find(new_tool):].split('\n')[0]
if oconf != nconf:
tool_changes.append(new_tool)
# put link names in a dictionary for finding dependencies
dependent_tools = []
for i, entry in enumerate(tool_changes):
dependent_tools.append(entry)
# change names to lowercase for use in clean, prep_start
tool_changes[i] = {'name': entry.lower().replace('-', '_')}
dependencies = Dependencies(dependent_tools)
# restart tools
restart = tool_changes + dependencies
tool_d = {}
for tool in restart:
self.clean(**tool)
tool_d.update(self.prep_start(**tool)[1])
if tool_d:
# TODO fix the arguments
Tools().start(tool_d)
except Exception as e: # pragma: no cover
self.logger.error('Problem restarting tools: ' + str(e))
status = (False, str(e))
return status
def GpuUsage(**kargs):
""" Get the current GPU usage of available GPUs """
usage = (False, None)
gpu_status = {'vent_usage': {'dedicated': [], 'mem_mb': {}}}
path_dirs = PathDirs(**kargs)
path_dirs.host_config()
template = Template(template=path_dirs.cfg_file)
# get running jobs using gpus
try:
d_client = docker.from_env()
c = d_client.containers.list(all=False,
filters={'label': 'vent-plugin'})
for container in c:
if ('vent.gpu' in container.attrs['Config']['Labels'] and
container.attrs['Config']['Labels']['vent.gpu'] == 'yes'):
device = container.attrs['Config']['Labels']['vent.gpu.device']
if ('vent.gpu.dedicated' in container.attrs['Config']['Labels'] and
container.attrs['Config']['Labels']['vent.gpu.dedicated'] == 'yes'):
gpu_status['vent_usage']['dedicated'].append(device)
elif 'vent.gpu.mem_mb' in container.attrs['Config']['Labels']:
if device not in gpu_status['vent_usage']['mem_mb']:
gpu_status['vent_usage']['mem_mb'][device] = 0
gpu_status['vent_usage']['mem_mb'][device] += int(
container.attrs['Config']['Labels']['vent.gpu.mem_mb'])
except Exception as e: # pragma: no cover
logger.error('Could not get running jobs ' + str(e))
port = '3476'
# default docker gateway
host = '172.17.0.1'
result = template.option('nvidia-docker-plugin', 'port')
if result[0]:
port = result[1]
result = template.option('nvidia-docker-plugin', 'host')
if result[0]:
host = result[1]
else:
try:
# now just requires ip, ifconfig
route = check_output(('ip', 'route')).decode('utf-8').split('\n')
default = ''
# grab the default network device.
for device in route:
if 'default' in device:
default = device.split()[4]
break
# grab the IP address for the default device
ip_addr = check_output(('ifconfig', default)).decode('utf-8')
ip_addr = ip_addr.split('\n')[1].split()[1]
host = ip_addr
except Exception as e: # pragma: no cover
logger.error('Something with the ip addresses'
'went wrong ' + str(e))
# have to get the info separately to determine how much memory is availabe
nd_url = 'http://' + host + ':' + port + '/v1.0/gpu/info/json'
try:
r = requests.get(nd_url)
if r.status_code == 200:
status = r.json()
for i, device in enumerate(status['Devices']):
gm = int(round(math.log(int(device['Memory']['Global']), 2)))
gpu_status[i] = {'global_memory': 2**gm,
'cores': device['Cores']}
else:
usage = (False, 'Unable to get GPU usage request error code: ' +
str(r.status_code))
except Exception as e: # pragma: no cover
usage = (False, 'Error: ' + str(e))
# get actual status of each gpu
nd_url = 'http://' + host + ':' + port + '/v1.0/gpu/status/json'
try:
r = requests.get(nd_url)
if r.status_code == 200:
status = r.json()
for i, device in enumerate(status['Devices']):
if i not in gpu_status:
gpu_status[i] = {}
gpu_status[i]['utilization'] = device['Utilization']
gpu_status[i]['memory'] = device['Memory']
gpu_status[i]['processes'] = device['Processes']
usage = (True, gpu_status)
else:
usage = (False, 'Unable to get GPU usage request error code: ' +
str(r.status_code))
except Exception as e: # pragma: no cover
usage = (False, 'Error: ' + str(e))
return usage
def _start_sections(self, s, files):
tool_d = {}
status = (True, None)
for section in s:
# initialize needed vars
c_name = s[section]['image_name'].replace(':', '-')
c_name = c_name.replace('/', '-')
instance_num = re.search(r'\d+$', s[section]['name'])
if instance_num:
c_name += instance_num.group()
image_name = s[section]['image_name']
# checkout the right version and branch of the repo
tool_d[c_name] = {'image': image_name,
'name': c_name}
# get rid of all commented sections in various runtime
# configurations
manifest = Template(self.manifest)
overall_dict = {}
for setting in ['info', 'docker', 'gpu', 'settings', 'service']:
option = manifest.option(section, setting)
if option[0]:
overall_dict[setting] = {}
settings_dict = json.loads(option[1])
for opt in settings_dict:
if not opt.startswith('#'):
overall_dict[setting][opt] = settings_dict[opt]
if 'docker' in overall_dict:
options_dict = overall_dict['docker']
for option in options_dict:
options = options_dict[option]
# check for commands to evaluate
if '`' in options:
cmds = options.split('`')
if len(cmds) > 2:
i = 1
while i < len(cmds):
try:
cmds[i] = check_output(shlex.split(cmds[i]),
stderr=STDOUT,
close_fds=True).strip().decode('utf-8')
except Exception as e: # pragma: no cover
self.logger.error(
'unable to evaluate command specified in vent.template: ' + str(e))
i += 2
options = ''.join(cmds)
# check for commands to evaluate
# store options set for docker
try:
tool_d[c_name][option] = ast.literal_eval(options)
except Exception as e: # pragma: no cover
self.logger.debug(
'Unable to literal_eval: {0}'.format(str(options)))
tool_d[c_name][option] = options
if 'labels' not in tool_d[c_name]:
tool_d[c_name]['labels'] = {}
# get the service uri info
if 'service' in overall_dict:
try:
options_dict = overall_dict['service']
for option in options_dict:
tool_d[c_name]['labels'][option] = options_dict[option]
except Exception as e: # pragma: no cover
self.logger.error('unable to store service options for '
'docker: ' + str(e))
# check for gpu settings
if 'gpu' in overall_dict:
try:
options_dict = json.loads(status[1])
for option in options_dict:
tool_d[c_name]['labels']['gpu.' +
option] = options_dict[option]
except Exception as e: # pragma: no cover
self.logger.error('unable to store gpu options for '
'docker: ' + str(e))
# get temporary name for links, etc.
plugin_c = Template(template=self.manifest)
status, plugin_sections = plugin_c.sections()
for plugin_section in plugin_sections:
status = plugin_c.option(plugin_section, 'link_name')
image_status = plugin_c.option(plugin_section, 'image_name')
if status[0] and image_status[0]:
cont_name = image_status[1].replace(':', '-')
cont_name = cont_name.replace('/', '-')
if cont_name not in tool_d:
tool_d[cont_name] = {'image': image_status[1],
'name': cont_name,
'start': False}
tool_d[cont_name]['tmp_name'] = status[1]
# add extra labels
tool_d[c_name]['labels']['vent'] = Version()
tool_d[c_name]['labels']['vent.namespace'] = s[section]['namespace']
tool_d[c_name]['labels']['vent.branch'] = s[section]['branch']
tool_d[c_name]['labels']['vent.version'] = s[section]['version']
tool_d[c_name]['labels']['vent.name'] = s[section]['name']
tool_d[c_name]['labels']['vent.section'] = section
tool_d[c_name]['labels']['vent.repo'] = s[section]['repo']
tool_d[c_name]['labels']['vent.type'] = s[section]['type']
# check for log_config settings in external-services
externally_configured = False
vent_config = Template(self.path_dirs.cfg_file)
for ext_tool in vent_config.section('external-services')[1]:
if ext_tool[0].lower() == 'syslog':
try:
log_dict = json.loads(ext_tool[1])
# configure if not locally active
if ('locally_active' not in log_dict or
log_dict['locally_active'] == 'no'):
del log_dict['locally_active']
log_config = {}
log_config['type'] = 'syslog'
log_config['config'] = {}
ip_address = ''
port = ''
for option in log_dict:
if option == 'ip_address':
ip_address = log_dict[option]
elif option == 'port':
port = log_dict['port']
syslog_address = 'tcp://' + ip_address + ':' + port
syslog_config = {'syslog-address': syslog_address,
'syslog-facility': 'daemon',
'tag': '{{.Name}}'}
log_config['config'].update(syslog_config)
externally_configured = True
except Exception as e: # pragma: no cover
self.logger.error('external settings for log_config'
" couldn't be stored because: " +
str(e))
externally_configured = False
if not externally_configured:
log_config = {'type': 'syslog',
'config': {'syslog-address': 'tcp://0.0.0.0:514',
'syslog-facility': 'daemon',
'tag': '{{.Name}}'}}
if 'groups' in s[section]:
# add labels for groups
tool_d[c_name]['labels']['vent.groups'] = s[section]['groups']
# add restart=always to core containers
if 'core' in s[section]['groups']:
tool_d[c_name]['restart_policy'] = {'Name': 'always'}
# map network names to environment variables
if 'network' in s[section]['groups']:
vent_config = Template(template=self.path_dirs.cfg_file)
nic_mappings = vent_config.section('network-mapping')
nics = ''
if nic_mappings[0]:
for nic in nic_mappings[1]:
nics += nic[0] + ':' + nic[1] + ','
nics = nics[:-1]
if nics:
if 'environment' in tool_d[c_name]:
tool_d[c_name]['environment'].append(
'VENT_NICS='+nics)
else:
tool_d[c_name]['environment'] = ['VENT_NICS='+nics]
# send logs to syslog
if ('syslog' not in s[section]['groups'] and
'core' in s[section]['groups']):
log_config['config']['tag'] = '{{.Name}}'
tool_d[c_name]['log_config'] = log_config
if 'syslog' not in s[section]['groups']:
tool_d[c_name]['log_config'] = log_config
# mount necessary directories
if 'files' in s[section]['groups']:
ulimits = []
ulimits.append(docker.types.Ulimit(
name='nofile', soft=1048576, hard=1048576))
tool_d[c_name]['ulimits'] = ulimits
# check if running in a docker container
if 'VENT_CONTAINERIZED' in environ and environ['VENT_CONTAINERIZED'] == 'true':
if 'volumes_from' in tool_d[c_name]:
tool_d[c_name]['volumes_from'].append(
environ['HOSTNAME'])
else:
tool_d[c_name]['volumes_from'] = [
environ['HOSTNAME']]
else:
if 'volumes' in tool_d[c_name]:
tool_d[c_name]['volumes'][self.path_dirs.base_dir[:-1]
] = {'bind': '/vent', 'mode': 'ro'}
else:
tool_d[c_name]['volumes'] = {
self.path_dirs.base_dir[:-1]: {'bind': '/vent', 'mode': 'ro'}}
if files[0]:
if 'volumes' in tool_d[c_name]:
tool_d[c_name]['volumes'][files[1]] = {
'bind': '/files', 'mode': 'rw'}
else:
tool_d[c_name]['volumes'] = {
files[1]: {'bind': '/files', 'mode': 'rw'}}
else:
tool_d[c_name]['log_config'] = log_config
# add label for priority
if 'settings' in overall_dict:
try:
options_dict = overall_dict['settings']
for option in options_dict:
if option == 'priority':
tool_d[c_name]['labels']['vent.priority'] = options_dict[option]
except Exception as e: # pragma: no cover
self.logger.error('unable to store settings options '
'for docker ' + str(e))
# only start tools that have been built
if s[section]['built'] != 'yes':
del tool_d[c_name]
# store section information for adding info to manifest later
else:
tool_d[c_name]['section'] = section
return status, tool_d
def Services(core, vent=True, external=False, **kargs):
"""
Get services that have exposed ports, expects param core to be True or
False based on which type of services to return, by default limit to vent
containers and processes not running externally, if not limited by vent
containers, then core is ignored.
"""
services = []
path_dirs = PathDirs(**kargs)
template = Template(template=path_dirs.cfg_file)
services_uri = template.option('main', 'services_uri')
try:
# look for internal services
if not external:
d_client = docker.from_env()
if vent:
c_filter = {'label': 'vent'}
containers = d_client.containers.list(filters=c_filter)
else:
containers = d_client.containers.list()
for c in containers:
uris = {}
name = None
if vent and 'vent.name' in c.attrs['Config']['Labels']:
if ((core and
'vent.groups' in c.attrs['Config']['Labels'] and
'core' in c.attrs['Config']['Labels']['vent.groups']) or
(not core and
'vent.groups' in c.attrs['Config']['Labels'] and
'core' not in c.attrs['Config']['Labels']['vent.groups'])):
name = c.attrs['Config']['Labels']['vent.name']
if name == '':
name = c.attrs['Config']['Labels']['vent.namespace'].split(
'/')[1]
for label in c.attrs['Config']['Labels']:
if label.startswith('uri'):
try:
val = int(label[-1])
if val not in uris:
uris[val] = {}
uris[val][label[:-1]
] = c.attrs['Config']['Labels'][label]
except Exception as e: # pragma: no cover
logger.error('Malformed services section'
' in the template file '
+ str(e))
else:
name = c.name
if name and 'vent.repo' in c.attrs['Config']['Labels']:
name = c.attrs['Config']['Labels']['vent.repo'].split(
'/')[-1] + ': ' + name
ports = c.attrs['NetworkSettings']['Ports']
p = []
port_num = 1
for port in ports:
if ports[port]:
try:
service_str = ''
if 'uri_prefix' in uris[port_num]:
service_str += uris[port_num]['uri_prefix']
host = ports[port][0]['HostIp']
if services_uri[0] and host == '0.0.0.0':
host = services_uri[1]
service_str += host + ':'
service_str += ports[port][0]['HostPort']
if 'uri_postfix' in uris[port_num]:
service_str += uris[port_num]['uri_postfix']
uri_creds = ''
if 'uri_user' in uris[port_num]:
uri_creds += ' user:'******'uri_user']
if 'uri_pw' in uris[port_num]:
uri_creds += ' pw:'
uri_creds += uris[port_num]['uri_pw']
if uri_creds:
service_str += ' - (' + uri_creds + ' )'
p.append(service_str)
except Exception as e: # pragma: no cover
logger.info('No services defined for ' + str(name) + ' with exposed port ' +
str(port_num) + ' because: ' + str(e))
port_num += 1
if p and name:
services.append((name, p))
logger.info(services)
# look for external services
else:
ext_tools = template.section('external-services')[1]
for ext_tool in ext_tools:
try:
name = ext_tool[0].lower()
p = []
settings_dict = json.loads(ext_tool[1])
if ('locally_active' in settings_dict and
settings_dict['locally_active'] == 'no'):
# default protocol to display will be http
protocol = 'http'
ip_address = ''
port = ''
for setting in settings_dict:
if setting == 'ip_address':
ip_address = settings_dict[setting]
if setting == 'port':
port = settings_dict[setting]
if setting == 'protocol':
protocol = settings_dict[setting]
p.append(protocol + '://' + ip_address + ':' + port)
if p and name:
services.append((name, p))
except Exception: # pragma: no cover
p = None
except Exception as e: # pragma: no cover
logger.error('Could not get services ' + str(e))
return services
def test_set_option():
""" Test the set_option function """
instance = Template()
instance.set_option('bad', 'x', 'x')
def _build_image(self, template, match_path, image_name, section, build_local=False):
status = (True, None)
output = ''
def set_instances(template, section, built, image_id=None):
""" Set build information for multiple instances """
i = 2
while True:
addtl_section = section.rsplit(':', 2)
addtl_section[0] += str(i)
addtl_section = ':'.join(addtl_section)
if template.section(addtl_section)[0]:
template.set_option(addtl_section, 'built', built)
if image_id:
template.set_option(addtl_section, 'image_id',
image_id)
template.set_option(addtl_section,
'last_updated', Timestamp())
else:
break
i += 1
# determine whether a tool should be considered a multi instance
multi_instance = False
try:
settings = template.option(section, 'settings')
if settings[0]:
settings_dict = json.loads(settings[1])
if 'instances' in settings_dict and int(settings_dict['instances']) > 1:
multi_instance = True
except Exception as e: # pragma: no cover
self.logger.error(
'Failed to check for multi instance because: {0}'.format(str(e)))
status = (False, str(e))
cwd = getcwd()
chdir(match_path)
try:
name = template.option(section, 'name')
groups = template.option(section, 'groups')
t_type = template.option(section, 'type')
path = template.option(section, 'path')
status, config_override = self.path_dirs.override_config(path[1])
if groups[1] == '' or not groups[0]:
groups = (True, 'none')
if not name[0]:
name = (True, image_name)
pull = False
image_exists = False
cfg_template = Template(template=self.path_dirs.cfg_file)
use_existing_image = False
result = cfg_template.option('build-options',
'use_existing_images')
if result[0]:
use_existing_image = result[1]
if use_existing_image == 'yes' and not config_override:
try:
self.d_client.images.get(image_name)
i_attrs = self.d_client.images.get(image_name).attrs
image_id = i_attrs['Id'].split(':')[1][:12]
template.set_option(section, 'built', 'yes')
template.set_option(section, 'image_id', image_id)
template.set_option(section, 'last_updated',
str(datetime.utcnow()) + ' UTC')
# set other instances too
if multi_instance:
set_instances(template, section, 'yes', image_id)
status = (True, 'Found {0}'.format(image_name))
self.logger.info(str(status))
image_exists = True
except docker.errors.ImageNotFound:
image_exists = False
except Exception as e: # pragma: no cover
self.logger.warning(
'Failed to query Docker for images because: {0}'.format(str(e)))
if not image_exists:
# pull if '/' in image_name, fallback to build
if '/' in image_name and not build_local and not config_override:
try:
image = self.d_client.images.pull(image_name)
i_attrs = self.d_client.images.get(
image_name).attrs
image_id = i_attrs['Id'].split(':')[1][:12]
if image_id:
template.set_option(section, 'built', 'yes')
template.set_option(section,
'image_id',
image_id)
template.set_option(section, 'last_updated',
str(datetime.utcnow()) +
' UTC')
# set other instances too
if multi_instance:
set_instances(template, section, 'yes',
image_id)
status = (True, 'Pulled {0}'.format(image_name))
self.logger.info(str(status))
else:
template.set_option(section, 'built', 'failed')
template.set_option(section, 'last_updated',
str(datetime.utcnow()) +
' UTC')
# set other instances too
if multi_instance:
set_instances(template, section, 'failed')
status = (False, 'Failed to pull image {0}'.format(
str(output.split('\n')[-1])))
self.logger.warning(str(status))
pull = True
except Exception as e: # pragma: no cover
self.logger.warning(
'Failed to pull image, going to build instead: {0}'.format(str(e)))
status = (
False, 'Failed to pull image because: {0}'.format(str(e)))
if not pull and not image_exists:
# get username to label built image with
username = getpass.getuser()
# see if additional file arg needed for building multiple
# images from same directory
file_tag = 'Dockerfile'
multi_tool = template.option(section, 'multi_tool')
if multi_tool[0] and multi_tool[1] == 'yes':
specific_file = template.option(section, 'name')[1]
if specific_file != 'unspecified':
file_tag = 'Dockerfile.' + specific_file
# update image name with new version for update
image_name = image_name.rsplit(':', 1)[0]+':'+self.branch
labels = {}
labels['vent'] = ''
labels['vent.section'] = section
labels['vent.repo'] = self.repo
labels['vent.type'] = t_type[1]
labels['vent.name'] = name[1]
labels['vent.groups'] = groups[1]
labels['built-by'] = username
image = self.d_client.images.build(path='.',
dockerfile=file_tag,
tag=image_name,
labels=labels, rm=True)
image_id = image[0].id.split(':')[1][:12]
template.set_option(section, 'built', 'yes')
template.set_option(section, 'image_id', image_id)
template.set_option(section, 'last_updated',
str(datetime.utcnow()) +
' UTC')
# set other instances too
if multi_instance:
set_instances(template, section, 'yes', image_id)
status = (True, 'Built {0}'.format(image_name))
except Exception as e: # pragma: no cover
self.logger.error('Unable to build image {0} because: {1} | {2}'.format(
str(image_name), str(e), str(output)))
template.set_option(section, 'built', 'failed')
template.set_option(section, 'last_updated',
str(datetime.utcnow()) + ' UTC')
if multi_instance:
set_instances(template, section, 'failed')
status = (
False, 'Failed to build image because: {0}'.format(str(e)))
chdir(cwd)
template.set_option(section, 'running', 'no')
return status, template
def _prep_start(self, repo, name):
args = locals()
status = (True, None)
try:
options = ['name',
'namespace',
'built',
'groups',
'path',
'image_name',
'branch',
'repo',
'type',
'version']
vent_config = Template(template=self.path_dirs.cfg_file)
manifest = Template(self.manifest)
files = vent_config.option('main', 'files')
files = (files[0], expanduser(files[1]))
s, _ = manifest.constrain_opts(args, options)
status, tool_d = self._start_sections(s, files)
# look out for links to delete because they're defined externally
links_to_delete = set()
# get instances for each tool
tool_instances = {}
sections = manifest.sections()[1]
for section in sections:
settings = manifest.option(section, 'settings')
if settings[0]:
settings = json.loads(settings[1])
if 'instances' in settings:
l_name = manifest.option(section, 'link_name')
if l_name[0]:
tool_instances[l_name[1]] = int(
settings['instances'])
# check and update links, volumes_from, network_mode
for container in list(tool_d.keys()):
if 'labels' not in tool_d[container] or 'vent.groups' not in tool_d[container]['labels'] or 'core' not in tool_d[container]['labels']['vent.groups']:
tool_d[container]['remove'] = True
if 'links' in tool_d[container]:
for link in list(tool_d[container]['links'].keys()):
# add links to external services already running if
# necessary, by default configure local services too
configure_local = True
ext = 'external-services'
if link in vent_config.options(ext)[1]:
try:
lconf = json.loads(vent_config.option(ext,
link)[1])
if ('locally_active' not in lconf or
lconf['locally_active'] == 'no'):
ip_adr = lconf['ip_address']
port = lconf['port']
tool_d[container]['extra_hosts'] = {}
# containers use lowercase names for
# connections
tool_d[container]['extra_hosts'][link.lower()
] = ip_adr
# create an environment variable for container
# to access port later
env_variable = link.upper() + \
'_CUSTOM_PORT=' + port
if 'environment' not in tool_d[container]:
tool_d[container]['environment'] = []
tool_d[container]['environment'].append(
env_variable)
# remove the entry from links because no
# longer connecting to local container
links_to_delete.add(link)
configure_local = False
except Exception as e: # pragma: no cover
self.logger.error(
'Could not load external settings because: {0}'.format(str(e)))
configure_local = True
status = False
if configure_local:
for c in list(tool_d.keys()):
if ('tmp_name' in tool_d[c] and
tool_d[c]['tmp_name'] == link):
tool_d[container]['links'][tool_d[c]['name']
] = tool_d[container]['links'].pop(link)
if link in tool_instances and tool_instances[link] > 1:
for i in range(2, tool_instances[link] + 1):
tool_d[container]['links'][tool_d[c]['name'] + str(
i)] = tool_d[container]['links'][tool_d[c]['name']] + str(i)
if 'volumes_from' in tool_d[container]:
tmp_volumes_from = tool_d[container]['volumes_from']
tool_d[container]['volumes_from'] = []
for volumes_from in list(tmp_volumes_from):
for c in list(tool_d.keys()):
if ('tmp_name' in tool_d[c] and
tool_d[c]['tmp_name'] == volumes_from):
tool_d[container]['volumes_from'].append(
tool_d[c]['name'])
tmp_volumes_from.remove(volumes_from)
tool_d[container]['volumes_from'] += tmp_volumes_from
if 'network_mode' in tool_d[container]:
if tool_d[container]['network_mode'].startswith('container:'):
network_c_name = tool_d[container]['network_mode'].split('container:')[
1]
for c in list(tool_d.keys()):
if ('tmp_name' in tool_d[c] and
tool_d[c]['tmp_name'] == network_c_name):
tool_d[container]['network_mode'] = 'container:' + \
tool_d[c]['name']
# remove tmp_names
for c in list(tool_d.keys()):
if 'tmp_name' in tool_d[c]:
del tool_d[c]['tmp_name']
# remove links section if all were externally configured
for c in list(tool_d.keys()):
if 'links' in tool_d[c]:
for link in links_to_delete:
if link in tool_d[c]['links']:
del tool_d[c]['links'][link]
# delete links if no more defined
if not tool_d[c]['links']:
del tool_d[c]['links']
# remove containers that shouldn't be started
for c in list(tool_d.keys()):
deleted = False
if 'start' in tool_d[c] and not tool_d[c]['start']:
del tool_d[c]
deleted = True
if not deleted:
# look for tools services that are being done externally
# tools are capitalized in vent.cfg, so make them lowercase
# for comparison
ext = 'external-services'
external_tools = vent_config.section(ext)[1]
name = tool_d[c]['labels']['vent.name']
for tool in external_tools:
if name == tool[0].lower():
try:
tool_config = json.loads(tool[1])
if ('locally_active' in tool_config and
tool_config['locally_active'] == 'no'):
del tool_d[c]
except Exception as e: # pragma: no cover
self.logger.warning('Locally running container ' +
name + ' may be redundant')
if status:
status = (True, tool_d)
else:
status = (False, tool_d)
except Exception as e: # pragma: no cover
self.logger.error('_prep_start failed with error: '+str(e))
status = (False, e)
return status
def create(self, group_view=False):
""" Update with current tools """
self.add_handlers({'^T': self.quit, '^Q': self.quit})
self.add(npyscreen.TitleText,
name='Select which tools to ' + self.action['action'] + ':',
editable=False)
togglable = ['remove']
if self.action['action_name'] in togglable:
self.cur_view = self.add(npyscreen.TitleText,
name='Group view:',
value='all groups',
editable=False,
rely=3)
self.add_handlers({'^V': self.toggle_view})
i = 5
else:
i = 4
if self.action['action_name'] == 'start':
response = self.tools_inst.inventory(
choices=['repos', 'tools', 'built', 'running'])
else:
response = self.tools_inst.inventory(choices=['repos', 'tools'])
if response[0]:
inventory = response[1]
repos = inventory['repos']
# dict has repo as key and list of core/non-core tools as values
has_core = {}
has_non_core = {}
# find all tools that are in this repo
# and list them if they are core
for repo in repos:
core_list = []
ncore_list = []
# splice the repo names for processing
if (repo.startswith('http')):
repo_name = repo.rsplit('/', 2)[1:]
else:
repo_name = repo.split('/')
for tool in inventory['tools']:
tool_repo_name = tool.split(':')
# cross reference repo names
if (repo_name[0] == tool_repo_name[0]
and repo_name[1] == tool_repo_name[1]):
# check to ensure tool not set to locally active = no
# in vent.cfg
externally_active = False
vent_cfg_file = self.api_action.vent_config
vent_cfg = Template(vent_cfg_file)
tool_pairs = vent_cfg.section('external-services')[1]
for ext_tool in tool_pairs:
if ext_tool[0].lower() == inventory['tools'][tool]:
try:
ext_tool_options = json.loads(ext_tool[1])
loc = 'locally_active'
if (loc in ext_tool_options
and ext_tool_options[loc] == 'no'):
externally_active = True
except Exception as e:
self.logger.error("Couldn't check ext"
' because: ' + str(e))
externally_active = False
manifest = Template(self.api_action.manifest)
if not externally_active:
instance_num = re.search(
r'\d+$',
manifest.option(tool, 'name')[1])
if not instance_num:
ncore_list.append(tool)
# multiple instances share same image
elif self.action[
'action_name'] not in self.no_instance:
ncore_list.append(tool)
has_core[repo] = core_list
has_non_core[repo] = ncore_list
for repo in repos:
self.tools_tc[repo] = {}
if self.action['cores']:
# make sure only repos with core tools are displayed
if has_core.get(repo):
self.repo_widgets[repo] = self.add(npyscreen.TitleText,
name='Plugin: ' +
repo,
editable=False,
rely=i,
relx=5)
for tool in has_core[repo]:
tool_name = tool.split(':', 2)[2].split('/')[-1]
if tool_name == '':
tool_name = '/'
self.tools_tc[repo][tool] = self.add(
npyscreen.CheckBox,
name=tool_name,
value=True,
relx=10)
i += 1
i += 3
else:
# make sure only repos with non-core tools are displayed
if has_non_core.get(repo):
self.repo_widgets[repo] = self.add(npyscreen.TitleText,
name='Plugin: ' +
repo,
editable=False,
rely=i,
relx=5)
for tool in has_non_core[repo]:
tool_name = tool.split(':', 2)[2].split('/')[-1]
if tool_name == '':
tool_name = '/'
self.tools_tc[repo][tool] = self.add(
npyscreen.CheckBox,
name=tool_name,
value=True,
relx=10)
i += 1
i += 3
return
def save_configure(self,
repo=None,
name=None,
groups=None,
config_val='',
from_registry=False,
main_cfg=False,
instances=1):
"""
Save changes made to vent.template through npyscreen to the template
and to plugin_manifest
"""
def template_to_manifest(vent_template, manifest, tool, instances):
"""
Helper function to transfer information from vent.template
to plugin_manifest
"""
sections = vent_template.sections()
if sections[0]:
for section in sections[1]:
section_dict = {}
if section == 'settings':
section_dict.update({'instances': str(instances)})
options = vent_template.options(section)
if options[0]:
for option in options[1]:
option_name = option
if option == 'name':
option_name = 'link_name'
opt_val = vent_template.option(section,
option)[1]
section_dict[option_name] = opt_val
if section_dict:
manifest.set_option(tool, section,
json.dumps(section_dict))
elif manifest.option(tool, section)[0]:
manifest.del_option(tool, section)
constraints = locals()
del constraints['config_val']
del constraints['from_registry']
del constraints['main_cfg']
del constraints['instances']
del constraints['template_to_manifest']
status = (True, None)
fd = None
# ensure instances is an int and remove instances from config_val to
# ensure correct info
instances = int(instances)
config_val = re.sub(r'instances\ *=\ *\d+\n', '', config_val)
api_system = System()
manifest = api_system.manifest
if not main_cfg:
if not from_registry:
# creating new instances
if instances > 1:
fd, template_path = tempfile.mkstemp(suffix='.template')
# scrub name for clean section name
if re.search(r'\d+$', name):
name = re.sub(r'\d+$', '', name)
t_identifier = {'name': name,
'branch': branch,
'version': version}
result = Template(manifest).constrain_opts(
t_identifier, [])
tools = result[0]
tool = list(tools.keys())[0]
else:
options = ['path', 'multi_tool', 'name']
tools, _ = Template(manifest).constrain_opts(
constraints, options)
# only one tool in tools because perform this function for
# every tool
if tools:
tool = list(tools.keys())[0]
if ('multi_tool' in tools[tool] and
tools[tool]['multi_tool'] == 'yes'):
name = tools[tool]['name']
if name == 'unspecified':
name = 'vent'
template_path = join(tools[tool]['path'],
name+'.template')
else:
template_path = join(tools[tool]['path'],
'vent.template')
else:
status = (False, "Couldn't save configuration")
else:
fd, template_path = tempfile.mkstemp(suffix='.template')
options = ['namespace']
constraints.update({'type': 'registry'})
tools, _ = Template(manifest).constrain_opts(constraints,
options)
if tools:
tool = list(tools.keys())[0]
else:
status = (False, "Couldn't save configuration")
if status[0]:
try:
# save in vent.template
with open(template_path, 'w') as f:
f.write(config_val)
# save in plugin_manifest
vent_template = Template(template_path)
manifest = Template(manifest)
if instances > 1:
# add instances as needed
for i in range(1, instances + 1):
i_section = tool.rsplit(':', 2)
i_section[0] += str(i) if i != 1 else ''
i_section = ':'.join(i_section)
if not manifest.section(i_section)[0]:
manifest.add_section(i_section)
for val_pair in manifest.section(tool)[1]:
name = val_pair[0]
val = val_pair[1]
if name == 'name':
val += str(i)
elif name == 'last_updated':
val = Timestamp()
elif name == 'running':
val = 'no'
manifest.set_option(i_section, name, val)
template_to_manifest(vent_template, manifest,
i_section, instances)
else:
settings = manifest.option(i_section,
'settings')
if settings[0]:
settings_dict = json.loads(settings[1])
settings_dict['instances'] = str(instances)
manifest.set_option(i_section, 'settings',
json.dumps(
settings_dict))
else:
inst = str(instances)
settings_dict = {'instances': inst}
manifest.set_option(i_section, 'settings',
json.dumps(
settings_dict))
else:
try:
settings_str = manifest.option(tool, 'settings')[1]
settings_dict = json.loads(settings_str)
old_instances = int(settings_dict['instances'])
except Exception:
old_instances = 1
template_to_manifest(vent_template, manifest,
tool, old_instances)
manifest.write_config()
status = (True, manifest)
except Exception as e: # pragma: no cover
self.logger.error('save_configure error: ' + str(e))
status = (False, str(e))
# close os file handle and remove temp file
if from_registry or instances > 1:
try:
close(fd)
remove(template_path)
except Exception as e: # pragma: no cover
self.logger.error('save_configure error: ' + str(e))
else:
with open(self.vent_config, 'w') as f:
f.write(config_val)
return status
def __init__(self, repo='', tool_name='',
next_tool=None, just_downloaded=False, vent_cfg=False,
from_registry=False, new_instance=False, *args, **keywords):
""" Initialize EditorForm objects """
# default for any editor
self.settings = locals()
self.settings.update(keywords)
del self.settings['self']
del self.settings['args']
del self.settings['keywords']
del self.settings['parentApp']
self.tool_identifier = {'name': tool_name}
self.settings.update(self.tool_identifier)
del self.settings['name']
self.settings['tool_name'] = tool_name
self.settings['next_tool'] = next_tool
self.settings['repo'] = repo
# setup checks
self.just_downloaded = ('just_downloaded' in self.settings and
self.settings['just_downloaded'])
self.vent_cfg = ('vent_cfg' in self.settings and
self.settings['vent_cfg'])
self.registry_tool = ('from_registry' in self.settings and
self.settings['from_registry'])
self.instance_cfg = ('new_instance' in self.settings and
self.settings['new_instance'])
# get manifest info for tool that will be used throughout
if not self.just_downloaded and not self.vent_cfg:
result = Template(System().manifest).constrain_opts(
self.tool_identifier, [])
tool, self.manifest = result
self.section = list(tool.keys())[0]
# get configuration information depending on type
if self.just_downloaded:
self.config_val = '[info]\n'
self.config_val += 'name = ' + keywords['link_name'] + '\n'
self.config_val += 'groups = ' + keywords['groups'] + '\n'
elif self.vent_cfg:
self.config_val = keywords['get_configure'](main_cfg=True)[1]
self.settings['tool_name'] = 'vent configuration'
elif self.instance_cfg:
path = self.manifest.option(self.section, 'path')[1]
multi_tool = self.manifest.option(self.section, 'multi_tool')
if multi_tool[0] and multi_tool[1] == 'yes':
name = self.manifest.option(self.section, 'name')[1]
if name == 'unspecified':
name = 'vent'
template_path = os.path.join(path,
name + '.template')
else:
template_path = os.path.join(path, 'vent.template')
# ensure instances is in the editor and that it is the right number
template = Template(template_path)
template.add_section('settings')
template.set_option('settings', 'instances',
str(self.settings['new_instances']))
template.write_config()
with open(template_path, 'r') as vent_template:
self.config_val = vent_template.read()
else:
self.config_val = keywords['get_configure'](
**self.tool_identifier)[1]
super(EditorForm, self).__init__(*args, **keywords)
def _startup(self):
"""
Automatically detect if a startup file is specified and stand up a vent
host with all necessary tools based on the specifications in that file
"""
status = (True, None)
try:
s_dict = {}
# rewrite the yml file to exclusively lowercase
with open(self.startup_file, 'r') as sup:
vent_startup = sup.read()
with open(self.startup_file, 'w') as sup:
for line in vent_startup:
sup.write(line.lower())
with open(self.startup_file, 'r') as sup:
s_dict = yaml.safe_load(sup.read())
if 'vent.cfg' in s_dict:
v_cfg = Template(self.vent_config)
for section in s_dict['vent.cfg']:
for option in s_dict['vent.cfg'][section]:
val = ('no', 'yes')[
s_dict['vent.cfg'][section][option]]
v_status = v_cfg.add_option(section, option, value=val)
if not v_status[0]:
v_cfg.set_option(section, option, val)
v_cfg.write_config()
del s_dict['vent.cfg']
tool_d = {}
extra_options = ['info', 'service', 'settings', 'docker', 'gpu']
s_dict_c = copy.deepcopy(s_dict)
# TODO check for repo or image type
for repo in s_dict_c:
repository = Repository(System().manifest)
repository.repo = repo
repository._clone()
repo_path, org, r_name = self.path_dirs.get_path(repo)
get_tools = []
for tool in s_dict_c[repo]:
t_branch = 'master'
t_version = 'HEAD'
if 'branch' in s_dict[repo][tool]:
t_branch = s_dict[repo][tool]['branch']
if 'version' in s_dict[repo][tool]:
t_version = s_dict[repo][tool]['version']
get_tools.append((tool, t_branch, t_version))
available_tools = AvailableTools(repo_path, tools=get_tools)
for tool in s_dict_c[repo]:
# if we can't find the tool in that repo, skip over this
# tool and notify in the logs
t_path, t_path_cased = PathDirs.rel_path(
tool, available_tools)
if t_path is None:
self.logger.error("Couldn't find tool " + tool + ' in'
' repo ' + repo)
continue
# ensure no NoneType iteration errors
if s_dict_c[repo][tool] is None:
s_dict[repo][tool] = {}
# check if we need to configure instances along the way
instances = 1
if 'settings' in s_dict[repo][tool]:
if 'instances' in s_dict[repo][tool]['settings']:
instances = int(s_dict[repo][tool]
['settings']['instances'])
# add the tool
t_branch = 'master'
t_version = 'HEAD'
t_image = None
add_tools = None
add_tools = [(t_path_cased, '')]
if 'branch' in s_dict[repo][tool]:
t_branch = s_dict[repo][tool]['branch']
if 'version' in s_dict[repo][tool]:
t_version = s_dict[repo][tool]['version']
if 'image' in s_dict[repo][tool]:
t_image = s_dict[repo][tool]['image']
repository.add(
repo, add_tools, branch=t_branch, version=t_version, image_name=t_image)
manifest = Template(self.manifest)
# update the manifest with extra defined runtime settings
base_section = ':'.join([org, r_name, t_path,
t_branch, t_version])
for option in extra_options:
if option in s_dict[repo][tool]:
opt_dict = manifest.option(base_section, option)
# add new values defined into default options for
# that tool, don't overwrite them
if opt_dict[0]:
opt_dict = json.loads(opt_dict[1])
else:
opt_dict = {}
# stringify values for vent
for v in s_dict[repo][tool][option]:
pval = s_dict[repo][tool][option][v]
s_dict[repo][tool][option][v] = json.dumps(
pval)
opt_dict.update(s_dict[repo][tool][option])
manifest.set_option(base_section, option,
json.dumps(opt_dict))
# copy manifest info into new sections if necessary
if instances > 1:
for i in range(2, instances + 1):
i_section = base_section.rsplit(':', 2)
i_section[0] += str(i)
i_section = ':'.join(i_section)
manifest.add_section(i_section)
for opt_val in manifest.section(base_section)[1]:
if opt_val[0] == 'name':
manifest.set_option(i_section, opt_val[0],
opt_val[1] + str(i))
else:
manifest.set_option(i_section, opt_val[0],
opt_val[1])
manifest.write_config()
tool_d = {}
tools = Tools()
# start tools, if necessary
for repo in s_dict:
for tool in s_dict[repo]:
if 'start' in s_dict[repo][tool]:
if s_dict[repo][tool]['start']:
local_instances = 1
if 'settings' in s_dict[repo][tool] and 'instances' in s_dict[repo][tool]['settings']:
local_instances = int(
s_dict[repo][tool]['settings']['instances'])
t_branch = 'master'
t_version = 'HEAD'
if 'branch' in s_dict[repo][tool]:
t_branch = s_dict[repo][tool]['branch']
if 'version' in s_dict[repo][tool]:
t_version = s_dict[repo][tool]['version']
for i in range(1, local_instances + 1):
i_name = tool + str(i) if i != 1 else tool
i_name = i_name.replace('@', '')
tool_d.update(
tools._prep_start(repo, i_name)[1])
if tool_d:
tools.start(tool_d, None, is_tool_d=True)
except Exception as e: # pragma: no cover
self.logger.error('Startup failed because: {0}'.format(str(e)))
status = (False, str(e))
return status
