def host_config(self):
""" Ensure the host configuration file exists """
if platform.system() == 'Darwin':
default_file_dir = os.path.join(os.path.expanduser('~'),
'vent_files')
else:
default_file_dir = '/opt/vent_files'
status = self.ensure_dir(default_file_dir)
config = Template(template=self.cfg_file)
sections = {
'main': {
'files': default_file_dir
},
'network-mapping': {},
'nvidia-docker-plugin': {
'port': '3476'
}
}
for s in sections:
if sections[s]:
for option in sections[s]:
config.add_option(s, option, sections[s][option])
else:
config.add_section(s)
config.write_config()
return status
def auto_install(self):
"""
Automatically detects images and installs them in the manifest if they
are not there already
"""
template = Template(template=self.manifest)
sections = template.sections()
images = self.d_client.images.list(filters={'label': 'vent'})
add_sections = []
status = (True, None)
for image in images:
if ('Labels' in image.attrs
and 'vent.section' in image.attrs['Config']['Labels']
and not image.attrs['Config']['Labels']['vent.section']
in sections[1]):
section = image.attrs['Config']['Labels']['vent.section']
section_str = image.attrs['Config']['Labels'][
'vent.section'].split(":")
template.add_section(section)
if 'vent.name' in image.attrs['Config']['Labels']:
template.set_option(
section, 'name',
image.attrs['Config']['Labels']['vent.name'])
if 'vent.repo' in image.attrs['Config']['Labels']:
template.set_option(
section, 'repo',
image.attrs['Config']['Labels']['vent.repo'])
git_path = join(self.path_dirs.plugins_dir,
"/".join(section_str[:2]))
if not isdir(git_path):
# clone it down
status = self.p_helper.clone(
image.attrs['Config']['Labels']['vent.repo'])
template.set_option(section, 'path',
join(git_path, section_str[-3][1:]))
# get template settings
# TODO account for template files not named vent.template
v_template = Template(template=join(
git_path, section_str[-3][1:], 'vent.template'))
tool_sections = v_template.sections()
if tool_sections[0]:
for s in tool_sections[1]:
section_dict = {}
options = v_template.options(s)
if options[0]:
for option in options[1]:
option_name = option
if option == 'name':
# get link name
template.set_option(
section, "link_name",
v_template.option(s, option)[1])
option_name = 'link_name'
opt_val = v_template.option(s, option)[1]
section_dict[option_name] = opt_val
if section_dict:
template.set_option(section, s,
json.dumps(section_dict))
if ('vent.type' in image.attrs['Config']['Labels']
and image.attrs['Config']['Labels']['vent.type']
== 'repository'):
template.set_option(section, 'namespace',
"/".join(section_str[:2]))
template.set_option(section, 'enabled', 'yes')
template.set_option(section, 'branch', section_str[-2])
template.set_option(section, 'version', section_str[-1])
template.set_option(section, 'last_updated',
str(datetime.utcnow()) + " UTC")
template.set_option(section, 'image_name',
image.attrs['RepoTags'][0])
template.set_option(section, 'type', 'repository')
if 'vent.groups' in image.attrs['Config']['Labels']:
template.set_option(
section, 'groups',
image.attrs['Config']['Labels']['vent.groups'])
template.set_option(section, 'built', 'yes')
template.set_option(section, 'image_id',
image.attrs['Id'].split(":")[1][:12])
template.set_option(section, 'running', 'no')
# check if image is running as a container
containers = self.d_client.containers.list(
filters={'label': 'vent'})
for container in containers:
if container.attrs['Image'] == image.attrs['Id']:
template.set_option(section, 'running', 'yes')
add_sections.append(section)
template.write_config()
if status[0]:
status = (True, add_sections)
return status
def _build_manifest(self, matches):
"""
Builds and writes the manifest for the tools being added
"""
# !! TODO check for pre-existing that conflict with request and
# disable and/or remove image
for match in matches:
# keep track of whether or not to write an additional manifest
# entry for multiple instances, and how many additional entries
# to write
addtl_entries = 0
# remove the .git for adding repo info to manifest
if self.repo.endswith('.git'):
self.repo = self.repo[:-4]
# remove @ in match for template setting purposes
if match[0].find('@') >= 0:
true_name = match[0].split('@')[1]
else:
true_name = match[0]
template = Template(template=self.manifest)
# TODO check for special settings here first for the specific match
self.version = match[1]
response = self.p_helper.checkout(branch=self.branch,
version=self.version)
if response[0]:
section = self.org + ":" + self.name + ":" + true_name + ":"
section += self.branch + ":" + self.version
# need to get rid of temp identifiers for tools in same repo
match_path = self.path + match[0].split('@')[0]
if not self.core:
image_name = self.org + "-" + self.name + "-"
if match[0] != '':
# if tool is in a subdir, add that to the name of the
# image
image_name += '-'.join(match[0].split('/')[1:]) + "-"
image_name += self.branch + ":" + self.version
else:
image_name = ('cyberreboot/vent-' +
match[0].split('/')[-1] + ':' + self.branch)
image_name = image_name.replace('_', '-')
# check if the section already exists
exists, options = template.section(section)
previous_commit = None
previous_commits = None
head = False
if exists:
for option in options:
# TODO check if tool name but a different version
# exists - then disable/remove if set
if option[0] == 'version' and option[1] == 'HEAD':
head = True
if option[0] == 'built' and option[1] == 'yes':
# !! TODO remove pre-existing image
pass
if option[0] == 'commit_id':
previous_commit = option[1]
if option[0] == 'previous_versions':
previous_commits = option[1]
# check if tool comes from multi directory
multi_tool = "no"
if match[0].find('@') >= 0:
multi_tool = "yes"
# !! TODO
# check if section should be removed from config i.e. all tools
# but new commit removed one that was in a previous commit
image_name = image_name.lower()
if image_name.endswith(":head"):
image_name = image_name.split(":head")[0] + ":HEAD"
# set template section & options for tool at version and branch
template.add_section(section)
template.set_option(section, "name", true_name.split('/')[-1])
template.set_option(section, "namespace",
self.org + '/' + self.name)
template.set_option(section, "path", match_path)
template.set_option(section, "repo", self.repo)
template.set_option(section, "enabled", "yes")
template.set_option(section, "multi_tool", multi_tool)
template.set_option(section, "branch", self.branch)
template.set_option(section, "version", self.version)
template.set_option(section, "last_updated",
str(datetime.utcnow()) + " UTC")
template.set_option(section, "image_name",
image_name.replace('@', '-'))
template.set_option(section, "type", "repository")
# save settings in vent.template to plugin_manifest
# watch for multiple tools in same directory
# just wanted to store match path with @ for path for use in
# other actions
tool_template = 'vent.template'
if match[0].find('@') >= 0:
tool_template = match[0].split('@')[1] + '.template'
vent_template_path = join(match_path, tool_template)
if os.path.exists(vent_template_path):
with open(vent_template_path) as f:
vent_template_val = f.read()
else:
vent_template_val = ''
settings_dict = ParsedSections(vent_template_val)
for setting in settings_dict:
template.set_option(section, setting,
json.dumps(settings_dict[setting]))
# TODO do we need this if we save as a dictionary?
vent_template = Template(vent_template_path)
vent_status, response = vent_template.option("info", "name")
if vent_status:
template.set_option(section, "link_name", response)
else:
template.set_option(section, "link_name",
true_name.split('/')[-1])
commit_id = None
if self.version == 'HEAD':
# remove @ in multi-tools
chdir(match_path)
cmd = "git rev-parse --short HEAD"
commit_id = check_output(shlex.split(cmd),
stderr=STDOUT,
close_fds=True).strip()
template.set_option(section, "commit_id", commit_id)
if head:
# no need to store previous commits if not HEAD, since
# the version will always be the same commit ID
if previous_commit and previous_commit != commit_id:
if (previous_commits
and previous_commit not in previous_commits):
previous_commits = (previous_commit + ',' +
previous_commits)
elif not previous_commits:
previous_commits = previous_commit
if previous_commits and previous_commits != commit_id:
template.set_option(section, "previous_versions",
previous_commits)
if self.version_alias:
template.set_option(section, "version_alias",
self.version_alias)
if self.groups:
template.set_option(section, "groups", self.groups)
else:
groups = vent_template.option("info", "groups")
if groups[0]:
template.set_option(section, "groups", groups[1])
# set groups to empty string if no groups defined for tool
else:
template.set_option(section, "groups", '')
template = self._build_image(template, match_path, image_name,
section)
# write additional entries for multiple instances
if addtl_entries > 0:
# add 2 for naming conventions
for i in range(2, addtl_entries + 2):
addtl_section = section.rsplit(':', 2)
addtl_section[0] += str(i)
addtl_section = ':'.join(addtl_section)
template.add_section(addtl_section)
orig_vals = template.section(section)[1]
for val in orig_vals:
template.set_option(addtl_section, val[0], val[1])
template.set_option(addtl_section, "name",
true_name.split('/')[-1] + str(i))
# write out configuration to the manifest file
template.write_config()
# reset to repo directory
chdir(self.path)
return
def add_image(self,
image,
link_name,
tag=None,
registry=None,
groups=None):
"""
Add an image with a tag from a Docker registry. Defaults to the Docker
Hub if not specified. Use a Template object to write an image's
information to `plugin_manifest.cfg'
Args:
image(type): docker image
link_name(type): fill me
Kwargs:
tag(type):
registry(type):
groups(type): Group that the docker image belongs to.
Returns:
tuple(bool,str): if the function completed successfully,
(True, name of image).
If the function failed, (False, message about failure)
"""
status = (True, None)
try:
pull_name = image
org = ''
name = image
if '/' in image:
org, name = image.split('/')
else:
org = "official"
if not tag:
tag = "latest"
if not registry:
registry = "docker.io"
full_image = registry + "/" + image + ":" + tag
image = self.d_client.images.pull(full_image)
section = ':'.join([registry, org, name, '', tag])
namespace = org + '/' + name
# set template section and options for tool at version and branch
template = Template(template=self.manifest)
template.add_section(section)
template.set_option(section, "name", name)
template.set_option(section, "pull_name", pull_name)
template.set_option(section, "namespace", namespace)
template.set_option(section, "path", "")
template.set_option(section, "repo", registry + '/' + org)
template.set_option(section, "enabled", "yes")
template.set_option(section, "branch", "")
template.set_option(section, "version", tag)
template.set_option(section, "last_updated",
str(datetime.utcnow()) + " UTC")
template.set_option(section, "image_name",
image.attrs['RepoTags'][0])
template.set_option(section, "type", "registry")
template.set_option(section, "link_name", link_name)
template.set_option(section, "commit_id", "")
template.set_option(section, "built", "yes")
template.set_option(section, "image_id",
image.attrs['Id'].split(':')[1][:12])
template.set_option(section, "groups", groups)
# write out configuration to the manifest file
template.write_config()
status = (True, "Successfully added " + full_image)
except Exception as e: # pragma: no cover
self.logger.error("Couldn't add image because " + str(e))
status = (False, str(e))
return status
def __init__(self,
repo='',
tool_name='',
branch='',
version='',
next_tool=None,
just_downloaded=False,
vent_cfg=False,
from_registry=False,
new_instance=False,
*args,
**keywords):
""" Initialize EditorForm objects """
# default for any editor
self.settings = locals()
self.settings.update(keywords)
del self.settings['self']
del self.settings['args']
del self.settings['keywords']
del self.settings['parentApp']
self.p_helper = PluginHelper(plugins_dir='.internals/')
self.tool_identifier = {
'name': tool_name,
'branch': branch,
'version': version
}
self.settings.update(self.tool_identifier)
del self.settings['name']
self.settings['tool_name'] = tool_name
self.settings['next_tool'] = next_tool
self.settings['repo'] = repo
# setup checks
self.just_downloaded = ('just_downloaded' in self.settings
and self.settings['just_downloaded'])
self.vent_cfg = ('vent_cfg' in self.settings
and self.settings['vent_cfg'])
self.registry_tool = ('from_registry' in self.settings
and self.settings['from_registry'])
self.instance_cfg = ('new_instance' in self.settings
and self.settings['new_instance'])
# get manifest info for tool that will be used throughout
if not self.just_downloaded and not self.vent_cfg:
result = self.p_helper.constraint_options(self.tool_identifier, [])
tool, self.manifest = result
self.section = tool.keys()[0]
# get configuration information depending on type
if self.just_downloaded:
self.config_val = '[info]\n'
self.config_val += 'name = ' + keywords['link_name'] + '\n'
self.config_val += 'groups = ' + keywords['groups'] + '\n'
elif self.vent_cfg:
self.config_val = keywords['get_configure'](main_cfg=True)[1]
self.settings['tool_name'] = 'vent configuration'
elif self.instance_cfg:
path = self.manifest.option(self.section, 'path')[1]
# defaults in .internals
path = path.replace('.vent/plugins', '.vent/.internals')
multi_tool = self.manifest.option(self.section, 'multi_tool')
if multi_tool[0] and multi_tool[1] == 'yes':
name = self.manifest.option(self.section, 'name')[1]
if name == 'unspecified':
name = 'vent'
template_path = os.path.join(path, name + '.template')
else:
template_path = os.path.join(path, 'vent.template')
# ensure instances is in the editor and that it is the right number
template = Template(template_path)
template.add_section('settings')
template.set_option('settings', 'instances',
str(self.settings['new_instances']))
template.write_config()
with open(template_path) as vent_template:
self.config_val = vent_template.read()
else:
self.config_val = keywords['get_configure'](
**self.tool_identifier)[1]
super(EditorForm, self).__init__(*args, **keywords)
def _build_manifest(self, matches):
""" Builds and writes the manifest for the tools being added """
# !! TODO check for pre-existing that conflict with request and disable and/or remove image
for match in matches:
template = Template(template=self.manifest)
# !! TODO check for special settings here first for the specific match
self.version = match[1]
response = self.checkout()
if response[0]:
section = self.org + ":" + self.name + ":" + match[0] + ":" + self.branch + ":" + self.version
match_path = self.path + match[0]
image_name = self.org + "-" + self.name + "-"
if match[0] != '':
# if tool is in a subdir, add that to the name of the image
image_name += '-'.join(match[0].split('/')[1:]) + "-"
image_name += self.branch + ":" + self.version
# check if the section already exists
exists, options = template.section(section)
previous_commit = None
previous_commits = None
head = False
if exists:
for option in options:
# TODO check if tool name but a different version exists - then disable/remove if set
if option[0] == 'version' and option[1] == 'HEAD':
head = True
if option[0] == 'built' and option[1] == 'yes':
# !! TODO remove pre-existing image
pass
if option[0] == 'commit_id':
previous_commit = option[1]
if option[0] == 'previous_versions':
previous_commits = option[1]
# !! TODO
# check if section should be removed from config - i.e. all tools,
# but new commit removed one that was in a previous commit
# set template section and options for tool at version and branch
template.add_section(section)
template.set_option(section, "name", match[0].split('/')[-1])
template.set_option(section, "namespace", self.org+'/'+self.name)
template.set_option(section, "path", match_path)
template.set_option(section, "repo", self.repo)
template.set_option(section, "enabled", "yes")
template.set_option(section, "branch", self.branch)
template.set_option(section, "version", self.version)
template.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC")
template.set_option(section, "image_name", image_name)
vent_template = Template(template=os.path.join(match_path, 'vent.template'))
vent_status, response = vent_template.option("info", "name")
if vent_status:
template.set_option(section, "link_name", response)
else:
template.set_option(section, "link_name", match[0].split('/')[-1])
commit_id = None
if self.version == 'HEAD':
os.chdir(match_path)
commit_id = subprocess.check_output(shlex.split("git rev-parse --short HEAD"), stderr=subprocess.STDOUT, close_fds=True).strip()
template.set_option(section, "commit_id", commit_id)
if head:
# no need to store previous commits if not HEAD, since
# the version will always be the same commit ID
if previous_commit and previous_commit != commit_id:
if previous_commits and previous_commit not in previous_commits:
previous_commits = previous_commit+','+previous_commits
elif not previous_commits:
previous_commits = previous_commit
if previous_commits and previous_commits != commit_id:
template.set_option(section, "previous_versions", previous_commits)
if self.version_alias:
template.set_option(section, "version_alias", self.version_alias)
if self.groups:
template.set_option(section, "groups", self.groups)
else:
vent_template = os.path.join(match_path, 'vent.template')
if os.path.exists(vent_template):
v_template = Template(template=vent_template)
groups = v_template.option("info", "groups")
if groups[0]:
template.set_option(section, "groups", groups[1])
template = self._build_image(template, match_path, image_name, section)
# write out configuration to the manifest file
template.write_config()
# reset to repo directory
os.chdir(self.path)
return
def _build_manifest(self, matches):
""" Builds and writes the manifest for the tools being added """
# !! TODO check for pre-existing that conflict with request and disable and/or remove image
for match in matches:
template = Template(template=self.manifest)
# !! TODO check for special settings here first for the specific match
self.version = match[1]
response = self.checkout()
if response[0]:
section = self.org + ":" + self.name + ":" + match[
0] + ":" + self.branch + ":" + self.version
match_path = self.path + match[0]
image_name = self.org + "-" + self.name + "-"
if match[0] != '':
# if tool is in a subdir, add that to the name of the image
image_name += '-'.join(match[0].split('/')[1:]) + "-"
image_name += self.branch + ":" + self.version
# check if the section already exists
exists, options = template.section(section)
previous_commit = None
previous_commits = None
head = False
if exists:
for option in options:
# TODO check if tool name but a different version exists - then disable/remove if set
if option[0] == 'version' and option[1] == 'HEAD':
head = True
if option[0] == 'built' and option[1] == 'yes':
# !! TODO remove pre-existing image
pass
if option[0] == 'commit_id':
previous_commit = option[1]
if option[0] == 'previous_versions':
previous_commits = option[1]
# !! TODO
# check if section should be removed from config - i.e. all tools,
# but new commit removed one that was in a previous commit
# set template section and options for tool at version and branch
template.add_section(section)
template.set_option(section, "name", match[0].split('/')[-1])
template.set_option(section, "namespace",
self.org + '/' + self.name)
template.set_option(section, "path", match_path)
template.set_option(section, "repo", self.repo)
template.set_option(section, "enabled", "yes")
template.set_option(section, "branch", self.branch)
template.set_option(section, "version", self.version)
template.set_option(section, "last_updated",
str(datetime.datetime.utcnow()) + " UTC")
template.set_option(section, "image_name", image_name)
vent_template = Template(
template=os.path.join(match_path, 'vent.template'))
vent_status, response = vent_template.option("info", "name")
if vent_status:
template.set_option(section, "link_name", response)
else:
template.set_option(section, "link_name",
match[0].split('/')[-1])
commit_id = None
if self.version == 'HEAD':
os.chdir(match_path)
commit_id = subprocess.check_output(
shlex.split("git rev-parse --short HEAD"),
stderr=subprocess.STDOUT,
close_fds=True).strip()
template.set_option(section, "commit_id", commit_id)
if head:
# no need to store previous commits if not HEAD, since
# the version will always be the same commit ID
if previous_commit and previous_commit != commit_id:
if previous_commits and previous_commit not in previous_commits:
previous_commits = previous_commit + ',' + previous_commits
elif not previous_commits:
previous_commits = previous_commit
if previous_commits and previous_commits != commit_id:
template.set_option(section, "previous_versions",
previous_commits)
if self.version_alias:
template.set_option(section, "version_alias",
self.version_alias)
if self.groups:
template.set_option(section, "groups", self.groups)
else:
vent_template = os.path.join(match_path, 'vent.template')
if os.path.exists(vent_template):
v_template = Template(template=vent_template)
groups = v_template.option("info", "groups")
if groups[0]:
template.set_option(section, "groups", groups[1])
template = self._build_image(template, match_path, image_name,
section)
# write out configuration to the manifest file
template.write_config()
# reset to repo directory
os.chdir(self.path)
return