def host_config(self):
""" Ensure the host configuration file exists """
if platform.system() == 'Darwin':
default_file_dir = os.path.join(os.path.expanduser('~'),
'vent_files')
else:
default_file_dir = '/opt/vent_files'
status = self.ensure_dir(default_file_dir)
config = Template(template=self.cfg_file)
sections = {
'main': {
'files': default_file_dir
},
'network-mapping': {},
'nvidia-docker-plugin': {
'port': '3476'
}
}
for s in sections:
if sections[s]:
for option in sections[s]:
config.add_option(s, option, sections[s][option])
else:
config.add_section(s)
config.write_config()
return status
def fill_config(self, path):
"""
Will take a yml located in home directory titled '.plugin_config.yml'.
It'll then fill in, using the yml, the plugin's config file
"""
self.logger.info("Starting: fill_config")
status = (True, None)
try:
# parse the yml file
if os.path.exists(self.plugin_config_file):
c_dict = {}
with open(self.plugin_config_file) as config_file:
c_dict = yaml.safe_load(config_file.read())
# assume the name of the plugin is its directory
plugin_name = path.split('/')[-1]
plugin_config_path = path + '/config/' + plugin_name + '.config'
if os.path.exists(plugin_config_path):
plugin_template = Template(plugin_config_path)
plugin_options = c_dict[plugin_name]
for section in plugin_options:
for option in plugin_options[section]:
plugin_template.set_option(
section, option, plugin_options[section][option])
plugin_template.write_config()
except Exception as e: # pragma: no cover
status = (False, e)
self.logger.info("Failed to fill_config: " + str(e))
self.logger.info("Status of fill_config: " + str(status[0]))
self.logger.info("Finished: fill_config")
def host_config(self):
""" Ensure the host configuration file exists """
default_file_dir = "/tmp/vent_files"
config = Template(template=os.path.join(self.base_dir, "vent.cfg"))
resp = config.section("main")
if resp[0]:
resp = config.option("main", "files")
if not resp[0]:
config.add_option("main", "files", default_file_dir)
self.ensure_dir(default_file_dir)
else:
config.add_option("main", "files", default_file_dir)
self.ensure_dir(default_file_dir)
config.write_config()
return
def fill_config(self, path):
"""
Will take a yml located in home directory titled '.plugin_config.yml'.
It'll then fill in, using the yml, the plugin's config file
"""
self.logger.info('Starting: fill_config')
status = (True, None)
must_build = False
try:
# parse the yml file
c_dict = {}
if os.path.exists(self.plugin_config_file):
with open(self.plugin_config_file, 'r') as config_file:
c_dict = yaml.safe_load(config_file.read())
# check for environment variable overrides
check_c_dict = c_dict.copy()
for tool in check_c_dict:
for section in check_c_dict[tool]:
for key in check_c_dict[tool][section]:
if key in os.environ:
c_dict[tool][section][key] = os.getenv(key)
# assume the name of the plugin is its directory
plugin_name = path.split('/')[-1]
if plugin_name == '':
plugin_name = path.split('/')[-2]
plugin_config_path = path + '/config/' + plugin_name + '.config'
if os.path.exists(plugin_config_path):
plugin_template = Template(plugin_config_path)
plugin_options = c_dict[plugin_name]
for section in plugin_options:
for option in plugin_options[section]:
plugin_template.set_option(
section, option,
str(plugin_options[section][option]))
plugin_template.write_config()
must_build = True
except Exception as e: # pragma: no cover
status = (False, e)
self.logger.error('Failed to fill_config: ' + str(e))
self.logger.info('Status of fill_config: ' + str(status[0]))
self.logger.info('Finished: fill_config')
return must_build
def cores(self, action, branch="master", version='HEAD'):
"""
Supply action (install, build, start, stop, clean) for core tools
"""
self.logger.info("Starting: cores")
status = (False, None)
try:
self.logger.info("action provided: " + str(action))
core = self.tools_status(True, branch=branch, version=version)[1]
if action in ["install", "build"]:
tools = []
core_repo = 'https://github.com/cyberreboot/vent'
resp = self.p_helper.apply_path(core_repo)
if resp[0]:
cwd = resp[1]
else:
self.logger.info("apply_path failed. Exiting cores"
" with status " + str(resp))
return resp
path = os.path.join(self.plugin.path_dirs.plugins_dir,
'cyberreboot/vent')
response = self.p_helper.checkout(branch=branch,
version=version)
self.logger.info("status of plugin checkout " +
str(response))
matches = self.p_helper.available_tools(path,
version=version,
groups='core')
for match in matches:
name = match[0].rsplit('/')[-1]
constraints = {'name': name,
'repo': core_repo}
prev_installed, _ = self.p_helper. \
constraint_options(constraints, [])
if not prev_installed:
tools.append((match[0], ''))
# only add stuff not already installed or repo specification
if ((tools) or
(isinstance(matches, list) and len(matches) == 0)):
status = self.plugin.add(core_repo,
tools=tools,
branch=branch,
build=False, core=True)
self.logger.info("status of plugin add: " + str(status))
else:
self.logger.info("no new tools to install")
status = (True, "previously installed")
plugin_c = Template(template=self.plugin.manifest)
sections = plugin_c.sections()
for tool in core['normal']:
for section in sections[1]:
name = plugin_c.option(section, "name")
orig_branch = plugin_c.option(section, "branch")
namespace = plugin_c.option(section, "namespace")
version = plugin_c.option(section, "version")
if (name[1] == tool and
orig_branch[1] == branch and
namespace[1] == "cyberreboot/vent" and
version[1] == "HEAD"):
plugin_c.set_option(section,
"image_name",
"cyberreboot/vent-" +
tool.replace('_', '-') + ":" +
branch)
plugin_c.write_config()
chdir(cwd)
if action == "build":
plugin_c = Template(template=self.plugin.manifest)
sections = plugin_c.sections()
try:
for tool in core['normal']:
for section in sections[1]:
tool = tool.replace('_', '-')
image_name = plugin_c.option(section,
"image_name")
check_image = "cyberreboot/vent-"
check_image += tool + ":" + branch
if image_name[1] == check_image:
timestamp = str(datetime.utcnow()) + " UTC"
try:
# currently can't use docker-py because it
# returns a 404 on pull so no way to valid
# if it worked or didn't
image_id = None
cmd = "docker pull " + check_image
output = check_output(shlex.split(cmd),
stderr=STDOUT)
# image_name in format of (bool, image_name)
name = image_name[1]
d_client = docker.from_env()
image_attrs = d_client.images.get(name)
image_attrs = image_attrs.attrs
image_id = image_attrs['Id'].split(':')[1][:12]
if image_id:
plugin_c.set_option(section,
"built",
"yes")
plugin_c.set_option(section,
"image_id",
image_id)
plugin_c.set_option(section,
"last_updated",
timestamp)
status = (True, "Pulled " + tool)
self.logger.info(str(status))
else:
plugin_c.set_option(section,
"built",
"failed")
plugin_c.set_option(section,
"last_updated",
timestamp)
status = (False,
"Failed to pull image " +
str(output.split('\n')[-1]))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
plugin_c.set_option(section,
"built",
"failed")
plugin_c.set_option(section,
"last_updated",
timestamp)
status = (False,
"Failed to pull image " + str(e))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
status = (False, "Failed to pull images " + str(e))
self.logger.error(str(status))
plugin_c.write_config()
elif action == "start":
status = self.api_action.prep_start(groups="core",
branch=branch)
if status[0]:
tool_d = status[1]
status = self.api_action.start(tool_d)
elif action == "stop":
status = self.api_action.stop(groups="core", branch=branch)
elif action == "clean":
status = self.api_action.clean(groups="core", branch=branch)
except Exception as e: # pragma: no cover
self.logger.info("core failed with error: " + str(e))
status = (False, e)
self.logger.info("Status of core: " + str(status[0]))
self.logger.info("Finished: core")
return status
def auto_install(self):
"""
Automatically detects images and installs them in the manifest if they
are not there already
"""
template = Template(template=self.manifest)
sections = template.sections()
images = self.d_client.images.list(filters={'label': 'vent'})
add_sections = []
status = (True, None)
for image in images:
if ('Labels' in image.attrs
and 'vent.section' in image.attrs['Config']['Labels']
and not image.attrs['Config']['Labels']['vent.section']
in sections[1]):
section = image.attrs['Config']['Labels']['vent.section']
section_str = image.attrs['Config']['Labels'][
'vent.section'].split(":")
template.add_section(section)
if 'vent.name' in image.attrs['Config']['Labels']:
template.set_option(
section, 'name',
image.attrs['Config']['Labels']['vent.name'])
if 'vent.repo' in image.attrs['Config']['Labels']:
template.set_option(
section, 'repo',
image.attrs['Config']['Labels']['vent.repo'])
git_path = join(self.path_dirs.plugins_dir,
"/".join(section_str[:2]))
if not isdir(git_path):
# clone it down
status = self.p_helper.clone(
image.attrs['Config']['Labels']['vent.repo'])
template.set_option(section, 'path',
join(git_path, section_str[-3][1:]))
# get template settings
# TODO account for template files not named vent.template
v_template = Template(template=join(
git_path, section_str[-3][1:], 'vent.template'))
tool_sections = v_template.sections()
if tool_sections[0]:
for s in tool_sections[1]:
section_dict = {}
options = v_template.options(s)
if options[0]:
for option in options[1]:
option_name = option
if option == 'name':
# get link name
template.set_option(
section, "link_name",
v_template.option(s, option)[1])
option_name = 'link_name'
opt_val = v_template.option(s, option)[1]
section_dict[option_name] = opt_val
if section_dict:
template.set_option(section, s,
json.dumps(section_dict))
if ('vent.type' in image.attrs['Config']['Labels']
and image.attrs['Config']['Labels']['vent.type']
== 'repository'):
template.set_option(section, 'namespace',
"/".join(section_str[:2]))
template.set_option(section, 'enabled', 'yes')
template.set_option(section, 'branch', section_str[-2])
template.set_option(section, 'version', section_str[-1])
template.set_option(section, 'last_updated',
str(datetime.utcnow()) + " UTC")
template.set_option(section, 'image_name',
image.attrs['RepoTags'][0])
template.set_option(section, 'type', 'repository')
if 'vent.groups' in image.attrs['Config']['Labels']:
template.set_option(
section, 'groups',
image.attrs['Config']['Labels']['vent.groups'])
template.set_option(section, 'built', 'yes')
template.set_option(section, 'image_id',
image.attrs['Id'].split(":")[1][:12])
template.set_option(section, 'running', 'no')
# check if image is running as a container
containers = self.d_client.containers.list(
filters={'label': 'vent'})
for container in containers:
if container.attrs['Image'] == image.attrs['Id']:
template.set_option(section, 'running', 'yes')
add_sections.append(section)
template.write_config()
if status[0]:
status = (True, add_sections)
return status
def _build_manifest(self, matches):
"""
Builds and writes the manifest for the tools being added
"""
# !! TODO check for pre-existing that conflict with request and
# disable and/or remove image
for match in matches:
# keep track of whether or not to write an additional manifest
# entry for multiple instances, and how many additional entries
# to write
addtl_entries = 0
# remove the .git for adding repo info to manifest
if self.repo.endswith('.git'):
self.repo = self.repo[:-4]
# remove @ in match for template setting purposes
if match[0].find('@') >= 0:
true_name = match[0].split('@')[1]
else:
true_name = match[0]
template = Template(template=self.manifest)
# TODO check for special settings here first for the specific match
self.version = match[1]
response = self.p_helper.checkout(branch=self.branch,
version=self.version)
if response[0]:
section = self.org + ":" + self.name + ":" + true_name + ":"
section += self.branch + ":" + self.version
# need to get rid of temp identifiers for tools in same repo
match_path = self.path + match[0].split('@')[0]
if not self.core:
image_name = self.org + "-" + self.name + "-"
if match[0] != '':
# if tool is in a subdir, add that to the name of the
# image
image_name += '-'.join(match[0].split('/')[1:]) + "-"
image_name += self.branch + ":" + self.version
else:
image_name = ('cyberreboot/vent-' +
match[0].split('/')[-1] + ':' + self.branch)
image_name = image_name.replace('_', '-')
# check if the section already exists
exists, options = template.section(section)
previous_commit = None
previous_commits = None
head = False
if exists:
for option in options:
# TODO check if tool name but a different version
# exists - then disable/remove if set
if option[0] == 'version' and option[1] == 'HEAD':
head = True
if option[0] == 'built' and option[1] == 'yes':
# !! TODO remove pre-existing image
pass
if option[0] == 'commit_id':
previous_commit = option[1]
if option[0] == 'previous_versions':
previous_commits = option[1]
# check if tool comes from multi directory
multi_tool = "no"
if match[0].find('@') >= 0:
multi_tool = "yes"
# !! TODO
# check if section should be removed from config i.e. all tools
# but new commit removed one that was in a previous commit
image_name = image_name.lower()
if image_name.endswith(":head"):
image_name = image_name.split(":head")[0] + ":HEAD"
# set template section & options for tool at version and branch
template.add_section(section)
template.set_option(section, "name", true_name.split('/')[-1])
template.set_option(section, "namespace",
self.org + '/' + self.name)
template.set_option(section, "path", match_path)
template.set_option(section, "repo", self.repo)
template.set_option(section, "enabled", "yes")
template.set_option(section, "multi_tool", multi_tool)
template.set_option(section, "branch", self.branch)
template.set_option(section, "version", self.version)
template.set_option(section, "last_updated",
str(datetime.utcnow()) + " UTC")
template.set_option(section, "image_name",
image_name.replace('@', '-'))
template.set_option(section, "type", "repository")
# save settings in vent.template to plugin_manifest
# watch for multiple tools in same directory
# just wanted to store match path with @ for path for use in
# other actions
tool_template = 'vent.template'
if match[0].find('@') >= 0:
tool_template = match[0].split('@')[1] + '.template'
vent_template_path = join(match_path, tool_template)
if os.path.exists(vent_template_path):
with open(vent_template_path) as f:
vent_template_val = f.read()
else:
vent_template_val = ''
settings_dict = ParsedSections(vent_template_val)
for setting in settings_dict:
template.set_option(section, setting,
json.dumps(settings_dict[setting]))
# TODO do we need this if we save as a dictionary?
vent_template = Template(vent_template_path)
vent_status, response = vent_template.option("info", "name")
if vent_status:
template.set_option(section, "link_name", response)
else:
template.set_option(section, "link_name",
true_name.split('/')[-1])
commit_id = None
if self.version == 'HEAD':
# remove @ in multi-tools
chdir(match_path)
cmd = "git rev-parse --short HEAD"
commit_id = check_output(shlex.split(cmd),
stderr=STDOUT,
close_fds=True).strip()
template.set_option(section, "commit_id", commit_id)
if head:
# no need to store previous commits if not HEAD, since
# the version will always be the same commit ID
if previous_commit and previous_commit != commit_id:
if (previous_commits
and previous_commit not in previous_commits):
previous_commits = (previous_commit + ',' +
previous_commits)
elif not previous_commits:
previous_commits = previous_commit
if previous_commits and previous_commits != commit_id:
template.set_option(section, "previous_versions",
previous_commits)
if self.version_alias:
template.set_option(section, "version_alias",
self.version_alias)
if self.groups:
template.set_option(section, "groups", self.groups)
else:
groups = vent_template.option("info", "groups")
if groups[0]:
template.set_option(section, "groups", groups[1])
# set groups to empty string if no groups defined for tool
else:
template.set_option(section, "groups", '')
template = self._build_image(template, match_path, image_name,
section)
# write additional entries for multiple instances
if addtl_entries > 0:
# add 2 for naming conventions
for i in range(2, addtl_entries + 2):
addtl_section = section.rsplit(':', 2)
addtl_section[0] += str(i)
addtl_section = ':'.join(addtl_section)
template.add_section(addtl_section)
orig_vals = template.section(section)[1]
for val in orig_vals:
template.set_option(addtl_section, val[0], val[1])
template.set_option(addtl_section, "name",
true_name.split('/')[-1] + str(i))
# write out configuration to the manifest file
template.write_config()
# reset to repo directory
chdir(self.path)
return
def add_image(self,
image,
link_name,
tag=None,
registry=None,
groups=None):
"""
Add an image with a tag from a Docker registry. Defaults to the Docker
Hub if not specified. Use a Template object to write an image's
information to `plugin_manifest.cfg'
Args:
image(type): docker image
link_name(type): fill me
Kwargs:
tag(type):
registry(type):
groups(type): Group that the docker image belongs to.
Returns:
tuple(bool,str): if the function completed successfully,
(True, name of image).
If the function failed, (False, message about failure)
"""
status = (True, None)
try:
pull_name = image
org = ''
name = image
if '/' in image:
org, name = image.split('/')
else:
org = "official"
if not tag:
tag = "latest"
if not registry:
registry = "docker.io"
full_image = registry + "/" + image + ":" + tag
image = self.d_client.images.pull(full_image)
section = ':'.join([registry, org, name, '', tag])
namespace = org + '/' + name
# set template section and options for tool at version and branch
template = Template(template=self.manifest)
template.add_section(section)
template.set_option(section, "name", name)
template.set_option(section, "pull_name", pull_name)
template.set_option(section, "namespace", namespace)
template.set_option(section, "path", "")
template.set_option(section, "repo", registry + '/' + org)
template.set_option(section, "enabled", "yes")
template.set_option(section, "branch", "")
template.set_option(section, "version", tag)
template.set_option(section, "last_updated",
str(datetime.utcnow()) + " UTC")
template.set_option(section, "image_name",
image.attrs['RepoTags'][0])
template.set_option(section, "type", "registry")
template.set_option(section, "link_name", link_name)
template.set_option(section, "commit_id", "")
template.set_option(section, "built", "yes")
template.set_option(section, "image_id",
image.attrs['Id'].split(':')[1][:12])
template.set_option(section, "groups", groups)
# write out configuration to the manifest file
template.write_config()
status = (True, "Successfully added " + full_image)
except Exception as e: # pragma: no cover
self.logger.error("Couldn't add image because " + str(e))
status = (False, str(e))
return status
def __init__(self,
repo='',
tool_name='',
branch='',
version='',
next_tool=None,
just_downloaded=False,
vent_cfg=False,
from_registry=False,
new_instance=False,
*args,
**keywords):
""" Initialize EditorForm objects """
# default for any editor
self.settings = locals()
self.settings.update(keywords)
del self.settings['self']
del self.settings['args']
del self.settings['keywords']
del self.settings['parentApp']
self.p_helper = PluginHelper(plugins_dir='.internals/')
self.tool_identifier = {
'name': tool_name,
'branch': branch,
'version': version
}
self.settings.update(self.tool_identifier)
del self.settings['name']
self.settings['tool_name'] = tool_name
self.settings['next_tool'] = next_tool
self.settings['repo'] = repo
# setup checks
self.just_downloaded = ('just_downloaded' in self.settings
and self.settings['just_downloaded'])
self.vent_cfg = ('vent_cfg' in self.settings
and self.settings['vent_cfg'])
self.registry_tool = ('from_registry' in self.settings
and self.settings['from_registry'])
self.instance_cfg = ('new_instance' in self.settings
and self.settings['new_instance'])
# get manifest info for tool that will be used throughout
if not self.just_downloaded and not self.vent_cfg:
result = self.p_helper.constraint_options(self.tool_identifier, [])
tool, self.manifest = result
self.section = tool.keys()[0]
# get configuration information depending on type
if self.just_downloaded:
self.config_val = '[info]\n'
self.config_val += 'name = ' + keywords['link_name'] + '\n'
self.config_val += 'groups = ' + keywords['groups'] + '\n'
elif self.vent_cfg:
self.config_val = keywords['get_configure'](main_cfg=True)[1]
self.settings['tool_name'] = 'vent configuration'
elif self.instance_cfg:
path = self.manifest.option(self.section, 'path')[1]
# defaults in .internals
path = path.replace('.vent/plugins', '.vent/.internals')
multi_tool = self.manifest.option(self.section, 'multi_tool')
if multi_tool[0] and multi_tool[1] == 'yes':
name = self.manifest.option(self.section, 'name')[1]
if name == 'unspecified':
name = 'vent'
template_path = os.path.join(path, name + '.template')
else:
template_path = os.path.join(path, 'vent.template')
# ensure instances is in the editor and that it is the right number
template = Template(template_path)
template.add_section('settings')
template.set_option('settings', 'instances',
str(self.settings['new_instances']))
template.write_config()
with open(template_path) as vent_template:
self.config_val = vent_template.read()
else:
self.config_val = keywords['get_configure'](
**self.tool_identifier)[1]
super(EditorForm, self).__init__(*args, **keywords)
def start_containers(self, container, tool_d, s_containers, f_containers):
""" Start container that was passed in and return status """
# use section to add info to manifest
section = tool_d[container]['section']
del tool_d[container]['section']
manifest = Template(self.manifest)
try:
c = self.d_client.containers.get(container)
c.start()
s_containers.append(container)
manifest.set_option(section, 'running', 'yes')
self.logger.info('started ' + str(container) + ' with ID: ' +
str(c.short_id))
except Exception as err:
try:
gpu = 'gpu.enabled'
failed = False
if (gpu in tool_d[container]['labels']
and tool_d[container]['labels'][gpu] == 'yes'):
vent_config = Template(template=self.path_dirs.cfg_file)
port = ''
host = ''
result = vent_config.option('nvidia-docker-plugin', 'port')
if result[0]:
port = result[1]
else:
port = '3476'
result = vent_config.option('nvidia-docker-plugin', 'host')
if result[0]:
host = result[1]
else:
# now just requires ip, ifconfig
try:
route = check_output(
('ip', 'route')).decode('utf-8').split('\n')
default = ''
# grab the default network device.
for device in route:
if 'default' in device:
default = device.split()[4]
break
# grab the IP address for the default device
ip_addr = check_output(
('ifconfig', default)).decode('utf-8')
ip_addr = ip_addr.split('\n')[1].split()[1]
host = ip_addr
except Exception as e: # pragma no cover
self.logger.error('failed to grab ip. Ensure that \
ip and ifconfig are installed')
nd_url = 'http://' + host + ':' + port + '/v1.0/docker/cli'
params = {'vol': 'nvidia_driver'}
r = requests.get(nd_url, params=params)
if r.status_code == 200:
options = r.text.split()
for option in options:
if option.startswith('--volume-driver='):
tool_d[container][
'volume_driver'] = option.split('=', 1)[1]
elif option.startswith('--volume='):
vol = option.split('=', 1)[1].split(':')
if 'volumes' in tool_d[container]:
if isinstance(tool_d[container]['volumes'],
list):
if len(vol) == 2:
c_vol = vol[0] + \
':' + vol[1] + ':rw'
else:
c_vol = vol[0] + ':' + \
vol[1] + ':' + vol[2]
tool_d[container]['volumes'].append(
c_vol)
else: # Dictionary
tool_d[container]['volumes'][
vol[0]] = {
'bind': vol[1],
'mode': vol[2]
}
else:
tool_d[container]['volumes'] = {
vol[0]: {
'bind': vol[1],
'mode': vol[2]
}
}
elif option.startswith('--device='):
dev = option.split('=', 1)[1]
if 'devices' in tool_d[container]:
tool_d[container]['devices'].append(dev +
':' +
dev +
':rwm')
else:
tool_d[container]['devices'] = [
dev + ':' + dev + ':rwm'
]
else:
self.logger.error('Unable to parse ' +
'nvidia-docker option: ' +
str(option))
else:
failed = True
f_containers.append(container)
manifest.set_option(section, 'running', 'failed')
self.logger.error('failed to start ' + str(container) +
' because nvidia-docker-plugin ' +
'failed with: ' + str(r.status_code))
if not failed:
try:
self.d_client.containers.remove(container, force=True)
self.logger.info('removed old existing container: ' +
str(container))
except Exception as e:
pass
cont_id = self.d_client.containers.run(detach=True,
**tool_d[container])
s_containers.append(container)
manifest.set_option(section, 'running', 'yes')
self.logger.info('started ' + str(container) +
' with ID: ' + str(cont_id))
except Exception as e: # pragma: no cover
f_containers.append(container)
manifest.set_option(section, 'running', 'failed')
self.logger.error('failed to start ' + str(container) +
' because: ' + str(e))
# save changes made to manifest
manifest.write_config()
return s_containers, f_containers
def _build_manifest(self, matches):
""" Builds and writes the manifest for the tools being added """
# !! TODO check for pre-existing that conflict with request and disable and/or remove image
for match in matches:
template = Template(template=self.manifest)
# !! TODO check for special settings here first for the specific match
self.version = match[1]
response = self.checkout()
if response[0]:
section = self.org + ":" + self.name + ":" + match[0] + ":" + self.branch + ":" + self.version
match_path = self.path + match[0]
image_name = self.org + "-" + self.name + "-"
if match[0] != '':
# if tool is in a subdir, add that to the name of the image
image_name += '-'.join(match[0].split('/')[1:]) + "-"
image_name += self.branch + ":" + self.version
# check if the section already exists
exists, options = template.section(section)
previous_commit = None
previous_commits = None
head = False
if exists:
for option in options:
# TODO check if tool name but a different version exists - then disable/remove if set
if option[0] == 'version' and option[1] == 'HEAD':
head = True
if option[0] == 'built' and option[1] == 'yes':
# !! TODO remove pre-existing image
pass
if option[0] == 'commit_id':
previous_commit = option[1]
if option[0] == 'previous_versions':
previous_commits = option[1]
# !! TODO
# check if section should be removed from config - i.e. all tools,
# but new commit removed one that was in a previous commit
# set template section and options for tool at version and branch
template.add_section(section)
template.set_option(section, "name", match[0].split('/')[-1])
template.set_option(section, "namespace", self.org+'/'+self.name)
template.set_option(section, "path", match_path)
template.set_option(section, "repo", self.repo)
template.set_option(section, "enabled", "yes")
template.set_option(section, "branch", self.branch)
template.set_option(section, "version", self.version)
template.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC")
template.set_option(section, "image_name", image_name)
vent_template = Template(template=os.path.join(match_path, 'vent.template'))
vent_status, response = vent_template.option("info", "name")
if vent_status:
template.set_option(section, "link_name", response)
else:
template.set_option(section, "link_name", match[0].split('/')[-1])
commit_id = None
if self.version == 'HEAD':
os.chdir(match_path)
commit_id = subprocess.check_output(shlex.split("git rev-parse --short HEAD"), stderr=subprocess.STDOUT, close_fds=True).strip()
template.set_option(section, "commit_id", commit_id)
if head:
# no need to store previous commits if not HEAD, since
# the version will always be the same commit ID
if previous_commit and previous_commit != commit_id:
if previous_commits and previous_commit not in previous_commits:
previous_commits = previous_commit+','+previous_commits
elif not previous_commits:
previous_commits = previous_commit
if previous_commits and previous_commits != commit_id:
template.set_option(section, "previous_versions", previous_commits)
if self.version_alias:
template.set_option(section, "version_alias", self.version_alias)
if self.groups:
template.set_option(section, "groups", self.groups)
else:
vent_template = os.path.join(match_path, 'vent.template')
if os.path.exists(vent_template):
v_template = Template(template=vent_template)
groups = v_template.option("info", "groups")
if groups[0]:
template.set_option(section, "groups", groups[1])
template = self._build_image(template, match_path, image_name, section)
# write out configuration to the manifest file
template.write_config()
# reset to repo directory
os.chdir(self.path)
return
def cores(self, action, branch='master', version='HEAD'):
"""
Supply action (install, build, start, stop, clean) for core tools
"""
self.logger.info('Starting: cores')
status = (False, None)
try:
self.logger.info('action provided: ' + str(action))
core = self.tools_status(True, branch=branch, version=version)[1]
if action in ['install', 'build']:
tools = []
core_repo = 'https://github.com/cyberreboot/vent'
resp = self.p_helper.apply_path(core_repo)
if resp[0]:
cwd = resp[1]
else:
self.logger.info('apply_path failed. Exiting cores'
' with status ' + str(resp))
return resp
path = os.path.join(self.plugin.path_dirs.plugins_dir,
'cyberreboot/vent')
# TODO commenting out for now, should use update_repo
# response = self.p_helper.checkout(branch=branch,
# version=version)
response = (True, None)
self.logger.info('status of plugin checkout ' +
str(response))
matches = self.p_helper.available_tools(path,
version=version,
groups='core')
for match in matches:
name = match[0].rsplit('/')[-1]
constraints = {'name': name,
'repo': core_repo}
prev_installed, _ = self.p_helper. \
constraint_options(constraints, [])
if not prev_installed:
tools.append((match[0], ''))
# only add stuff not already installed or repo specification
if ((tools) or
(isinstance(matches, list) and len(matches) == 0)):
status = self.plugin.add(core_repo,
tools=tools,
branch=branch,
build=False, core=True)
self.logger.info('status of plugin add: ' + str(status))
else:
self.logger.info('no new tools to install')
status = (True, 'previously installed')
plugin_c = Template(template=self.plugin.manifest)
sections = plugin_c.sections()
for tool in core['normal']:
for section in sections[1]:
name = plugin_c.option(section, 'name')
orig_branch = plugin_c.option(section, 'branch')
namespace = plugin_c.option(section, 'namespace')
version = plugin_c.option(section, 'version')
if (name[1] == tool and
orig_branch[1] == branch and
namespace[1] == 'cyberreboot/vent' and
version[1] == 'HEAD'):
plugin_c.set_option(section,
'image_name',
'cyberreboot/vent-' +
tool.replace('_', '-') + ':' +
branch)
plugin_c.write_config()
chdir(cwd)
if action == 'build':
plugin_c = Template(template=self.plugin.manifest)
sections = plugin_c.sections()
try:
for tool in core['normal']:
for section in sections[1]:
tool = tool.replace('_', '-')
image_name = plugin_c.option(section,
'image_name')
check_image = 'cyberreboot/vent-'
check_image += tool + ':' + branch
if image_name[1] == check_image:
timestamp = str(datetime.utcnow()) + ' UTC'
try:
# currently can't use docker-py because it
# returns a 404 on pull so no way to valid
# if it worked or didn't
image_id = None
cmd = 'docker pull ' + check_image
output = check_output(shlex.split(cmd),
stderr=STDOUT).decode('utf-8')
# image_name in format of (bool, image_name)
name = image_name[1]
d_client = docker.from_env()
image_attrs = d_client.images.get(name)
image_attrs = image_attrs.attrs
image_id = image_attrs['Id'].split(':')[
1][:12]
if image_id:
plugin_c.set_option(section,
'built',
'yes')
plugin_c.set_option(section,
'image_id',
image_id)
plugin_c.set_option(section,
'last_updated',
timestamp)
status = (True, 'Pulled ' + tool)
self.logger.info(str(status))
else:
plugin_c.set_option(section,
'built',
'failed')
plugin_c.set_option(section,
'last_updated',
timestamp)
status = (False,
'Failed to pull image ' +
str(output.split('\n')[-1]))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
plugin_c.set_option(section,
'built',
'failed')
plugin_c.set_option(section,
'last_updated',
timestamp)
status = (False,
'Failed to pull image ' + str(e))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
status = (False, 'Failed to pull images ' + str(e))
self.logger.error(str(status))
plugin_c.write_config()
elif action == 'start':
status = self.api_action.prep_start(groups='core',
branch=branch)
if status[0]:
tool_d = status[1]
status = self.api_action.start(tool_d)
elif action == 'stop':
status = self.api_action.stop(groups='core', branch=branch)
elif action == 'clean':
status = self.api_action.clean(groups='core', branch=branch)
except Exception as e: # pragma: no cover
self.logger.info('core failed with error: ' + str(e))
status = (False, e)
self.logger.info('Status of core: ' + str(status[0]))
self.logger.info('Finished: core')
return status
def _build_manifest(self, matches):
""" Builds and writes the manifest for the tools being added """
# !! TODO check for pre-existing that conflict with request and disable and/or remove image
for match in matches:
template = Template(template=self.manifest)
# !! TODO check for special settings here first for the specific match
self.version = match[1]
response = self.checkout()
if response[0]:
section = self.org + ":" + self.name + ":" + match[
0] + ":" + self.branch + ":" + self.version
match_path = self.path + match[0]
image_name = self.org + "-" + self.name + "-"
if match[0] != '':
# if tool is in a subdir, add that to the name of the image
image_name += '-'.join(match[0].split('/')[1:]) + "-"
image_name += self.branch + ":" + self.version
# check if the section already exists
exists, options = template.section(section)
previous_commit = None
previous_commits = None
head = False
if exists:
for option in options:
# TODO check if tool name but a different version exists - then disable/remove if set
if option[0] == 'version' and option[1] == 'HEAD':
head = True
if option[0] == 'built' and option[1] == 'yes':
# !! TODO remove pre-existing image
pass
if option[0] == 'commit_id':
previous_commit = option[1]
if option[0] == 'previous_versions':
previous_commits = option[1]
# !! TODO
# check if section should be removed from config - i.e. all tools,
# but new commit removed one that was in a previous commit
# set template section and options for tool at version and branch
template.add_section(section)
template.set_option(section, "name", match[0].split('/')[-1])
template.set_option(section, "namespace",
self.org + '/' + self.name)
template.set_option(section, "path", match_path)
template.set_option(section, "repo", self.repo)
template.set_option(section, "enabled", "yes")
template.set_option(section, "branch", self.branch)
template.set_option(section, "version", self.version)
template.set_option(section, "last_updated",
str(datetime.datetime.utcnow()) + " UTC")
template.set_option(section, "image_name", image_name)
vent_template = Template(
template=os.path.join(match_path, 'vent.template'))
vent_status, response = vent_template.option("info", "name")
if vent_status:
template.set_option(section, "link_name", response)
else:
template.set_option(section, "link_name",
match[0].split('/')[-1])
commit_id = None
if self.version == 'HEAD':
os.chdir(match_path)
commit_id = subprocess.check_output(
shlex.split("git rev-parse --short HEAD"),
stderr=subprocess.STDOUT,
close_fds=True).strip()
template.set_option(section, "commit_id", commit_id)
if head:
# no need to store previous commits if not HEAD, since
# the version will always be the same commit ID
if previous_commit and previous_commit != commit_id:
if previous_commits and previous_commit not in previous_commits:
previous_commits = previous_commit + ',' + previous_commits
elif not previous_commits:
previous_commits = previous_commit
if previous_commits and previous_commits != commit_id:
template.set_option(section, "previous_versions",
previous_commits)
if self.version_alias:
template.set_option(section, "version_alias",
self.version_alias)
if self.groups:
template.set_option(section, "groups", self.groups)
else:
vent_template = os.path.join(match_path, 'vent.template')
if os.path.exists(vent_template):
v_template = Template(template=vent_template)
groups = v_template.option("info", "groups")
if groups[0]:
template.set_option(section, "groups", groups[1])
template = self._build_image(template, match_path, image_name,
section)
# write out configuration to the manifest file
template.write_config()
# reset to repo directory
os.chdir(self.path)
return
def cores(self, action, branch="master"):
""" Supply action (install, build, start, stop, clean) for core tools """
self.logger.info("Starting: cores")
status = (True, None)
try:
self.logger.info("action provided: " + str(action))
core = Core(branch=branch)
if action in ["install", "build"]:
tools = []
plugins = Plugin(plugins_dir=".internals/plugins")
plugins.version = 'HEAD'
plugins.branch = branch
plugins.apply_path('https://github.com/cyberreboot/vent')
response = plugins.checkout()
self.logger.info("status of plugin checkout " + str(response))
matches = plugins._available_tools(groups='core')
for match in matches:
tools.append((match[0], ''))
status = plugins.add('https://github.com/cyberreboot/vent',
tools=tools,
branch=branch,
build=False)
self.logger.info("status of plugin add: " + str(status))
plugin_config = Template(template=self.plugin.manifest)
sections = plugin_config.sections()
for tool in core['normal']:
for section in sections[1]:
name = plugin_config.option(section, "name")
orig_branch = plugin_config.option(section, "branch")
namespace = plugin_config.option(section, "namespace")
version = plugin_config.option(section, "version")
if name[1] == tool and orig_branch[
1] == branch and namespace[
1] == "cyberreboot/vent" and version[
1] == "HEAD":
plugin_config.set_option(
section, "image_name",
"cyberreboot/vent-" + tool + ":" + branch)
plugin_config.write_config()
if action == "build":
plugin_config = Template(template=self.plugin.manifest)
sections = plugin_config.sections()
try:
for tool in core['normal']:
for section in sections[1]:
image_name = plugin_config.option(
section, "image_name")
if image_name[
1] == "cyberreboot/vent-" + tool + ":" + branch:
try:
# currently can't use docker-py because it
# returns a 404 on pull so no way to valid if it
# worked or didn't
#image_id = self.d_client.images.pull('cyberreboot/vent-'+tool, tag=branch)
image_id = None
output = subprocess.check_output(
shlex.split(
"docker pull cyberreboot/vent-" +
tool + ":" + branch),
stderr=subprocess.STDOUT)
for line in output.split('\n'):
if line.startswith("Digest: sha256:"):
image_id = line.split(
"Digest: sha256:")[1][:12]
if image_id:
plugin_config.set_option(
section, "built", "yes")
plugin_config.set_option(
section, "image_id", image_id)
plugin_config.set_option(
section, "last_updated",
str(datetime.datetime.utcnow()) +
" UTC")
status = (True, "Pulled " + tool)
self.logger.info(str(status))
else:
plugin_config.set_option(
section, "built", "failed")
plugin_config.set_option(
section, "last_updated",
str(datetime.datetime.utcnow()) +
" UTC")
status = (False,
"Failed to pull image " +
str(output.split('\n')[-1]))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
plugin_config.set_option(
section, "built", "failed")
plugin_config.set_option(
section, "last_updated",
str(datetime.datetime.utcnow()) +
" UTC")
status = (False,
"Failed to pull image " + str(e))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
status = (False, "Failed to pull images " + str(e))
self.logger.error(str(status))
plugin_config.write_config()
elif action == "start":
status = self.prep_start(groups="core", branch=branch)
if status[0]:
tool_dict = status[1]
status = self.start(tool_dict)
elif action == "stop":
status = self.stop(groups="core", branch=branch)
elif action == "clean":
status = self.clean(groups="core", branch=branch)
except Exception as e:
self.logger.info("core failed with error: " + str(e))
status = (False, e)
self.logger.info("Status of core: " + str(status))
self.logger.info("Finished: core")
return status
def cores(self, action, branch="master"):
""" Supply action (install, build, start, stop, clean) for core tools """
self.logger.info("Starting: cores")
status = (True, None)
try:
self.logger.info("action provided: "+str(action))
core = Core(branch=branch)
if action in ["install", "build"]:
tools = []
plugins = Plugin(plugins_dir=".internals/plugins")
plugins.version = 'HEAD'
plugins.branch = branch
plugins.apply_path('https://github.com/cyberreboot/vent')
response = plugins.checkout()
self.logger.info("status of plugin checkout "+str(response))
matches = plugins._available_tools(groups='core')
for match in matches:
tools.append((match[0], ''))
status = plugins.add('https://github.com/cyberreboot/vent', tools=tools, branch=branch, build=False)
self.logger.info("status of plugin add: "+str(status))
plugin_config = Template(template=self.plugin.manifest)
sections = plugin_config.sections()
for tool in core['normal']:
for section in sections[1]:
name = plugin_config.option(section, "name")
orig_branch = plugin_config.option(section, "branch")
namespace = plugin_config.option(section, "namespace")
version = plugin_config.option(section, "version")
if name[1] == tool and orig_branch[1] == branch and namespace[1] == "cyberreboot/vent" and version[1] == "HEAD":
plugin_config.set_option(section, "image_name", "cyberreboot/vent-"+tool+":"+branch)
plugin_config.write_config()
if action == "build":
plugin_config = Template(template=self.plugin.manifest)
sections = plugin_config.sections()
try:
for tool in core['normal']:
for section in sections[1]:
image_name = plugin_config.option(section, "image_name")
if image_name[1] == "cyberreboot/vent-"+tool+":"+branch:
try:
# currently can't use docker-py because it
# returns a 404 on pull so no way to valid if it
# worked or didn't
#image_id = self.d_client.images.pull('cyberreboot/vent-'+tool, tag=branch)
image_id = None
output = subprocess.check_output(shlex.split("docker pull cyberreboot/vent-"+tool+":"+branch), stderr=subprocess.STDOUT)
for line in output.split('\n'):
if line.startswith("Digest: sha256:"):
image_id = line.split("Digest: sha256:")[1][:12]
if image_id:
plugin_config.set_option(section, "built", "yes")
plugin_config.set_option(section, "image_id", image_id)
plugin_config.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC")
status = (True, "Pulled "+tool)
self.logger.info(str(status))
else:
plugin_config.set_option(section, "built", "failed")
plugin_config.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC")
status = (False, "Failed to pull image "+str(output.split('\n')[-1]))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
plugin_config.set_option(section, "built", "failed")
plugin_config.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC")
status = (False, "Failed to pull image "+str(e))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
status = (False, "Failed to pull images "+str(e))
self.logger.error(str(status))
plugin_config.write_config()
elif action == "start":
status = self.prep_start(groups="core", branch=branch)
if status[0]:
tool_dict = status[1]
status = self.start(tool_dict)
elif action == "stop":
status = self.stop(groups="core", branch=branch)
elif action == "clean":
status = self.clean(groups="core", branch=branch)
except Exception as e:
self.logger.info("core failed with error: "+str(e))
status = (False, e)
self.logger.info("Status of core: "+str(status))
self.logger.info("Finished: core")
return status