def Tools(**kargs):
""" Get tools that exist in the manifest """
path_dirs = PathDirs(**kargs)
manifest = os.path.join(path_dirs.meta_dir, "plugin_manifest.cfg")
template = Template(template=manifest)
tools = template.sections()
return tools[1]
def list_tools(self):
"""
Return list of tuples of all tools
"""
tools = []
template = Template(template=self.manifest)
exists, sections = template.sections()
if exists:
for section in sections:
options = {
'section': section,
'enabled': None,
'built': None,
'version': None,
'repo': None,
'branch': None,
'name': None,
'groups': None,
'image_name': None
}
for option in options.keys():
exists, value = template.option(section, option)
if exists:
options[option] = value
tools.append(options)
return tools
def Dependencies(tools):
"""
Takes in a list of tools that are being updated and returns any tools that
depend on linking to them
"""
dependencies = []
if tools:
path_dirs = PathDirs()
man = Template(os.path.join(path_dirs.meta_dir, 'plugin_manifest.cfg'))
for section in man.sections()[1]:
# don't worry about dealing with tool if it's not running
running = man.option(section, 'running')
if not running[0] or running[1] != 'yes':
continue
t_name = man.option(section, 'name')[1]
t_branch = man.option(section, 'branch')[1]
t_version = man.option(section, 'version')[1]
t_identifier = {
'name': t_name,
'branch': t_branch,
'version': t_version
}
options = man.options(section)[1]
if 'docker' in options:
d_settings = json.loads(man.option(section, 'docker')[1])
if 'links' in d_settings:
for link in json.loads(d_settings['links']):
if link in tools:
dependencies.append(t_identifier)
return dependencies
def Core(branch="master", **kargs):
"""
Get the normal core tools, and the currently installed/built/running ones,
including custom core services
"""
# !! TODO this might need to store namespaces/branches/versions
core = {'built':[], 'running':[], 'installed':[], 'normal':[]}
# get normal core tools
plugins = Plugin(plugins_dir=".internals/plugins")
status, cwd = plugins.clone('https://github.com/cyberreboot/vent')
if status:
plugins.version = 'HEAD'
plugins.branch = branch
response = plugins.checkout()
matches = plugins._available_tools(groups='core')
for match in matches:
core['normal'].append(match[0].split('/')[-1])
else:
core['normal'] = 'failed'
# get core tools that have been installed
path_dirs = PathDirs(**kargs)
manifest = os.path.join(path_dirs.meta_dir, "plugin_manifest.cfg")
template = Template(template=manifest)
tools = template.sections()
if tools[0]:
for tool in tools[1]:
groups = template.option(tool, "groups")
if groups[0] and "core" in groups[1]:
name = template.option(tool, "name")
if name[0]:
core['installed'].append(name[1])
# get core tools that have been built and/or are running
try:
d_client = docker.from_env()
images = d_client.images.list()
for image in images:
try:
if "vent.groups" in image.attrs['Labels'] and 'core' in image.attrs['Labels']['vent.groups']:
if 'vent.name' in image.attrs['Labels']:
core['built'].append(image.attrs['Labels']['vent.name'])
except Exception as err: # pragma: no cover
pass
containers = d_client.containers.list()
for container in containers:
try:
if "vent.groups" in container.attrs['Config']['Labels'] and 'core' in container.attrs['Config']['Labels']['vent.groups']:
if 'vent.name' in container.attrs['Config']['Labels']:
core['running'].append(container.attrs['Config']['Labels']['vent.name'])
except Exception as err: # pragma: no cover
pass
except Exception as e: # pragma: no cover
pass
return core
def tools(self):
""" Return list of tuples of all tools """
tools = []
template = Template(template=self.manifest)
exists, sections = template.sections()
if exists:
for section in sections:
options = {'section':section,
'enabled':None,
'built':None,
'version':None,
'repo':None,
'branch':None,
'name':None,
'groups':None,
'image_name':None}
for option in options.keys():
exists, value = template.option(section, option)
if exists:
options[option] = value
tools.append(options)
return tools
def tools_status(self, core, branch="master", version="HEAD", **kargs):
"""
Get tools that are currently installed/built/running and also the
number of repos that those tools come from; can toggle whether looking
for core tools or plugin tools
"""
# !! TODO this might need to store namespaces/branches/versions
all_tools = {'built': [], 'running': [], 'installed': [], 'normal': []}
core_repo = 'https://github.com/cyberreboot/vent'
repos = set()
tools = Tools(**kargs)
# get manifest file
manifest = os.path.join(self.api_action.plugin.path_dirs.meta_dir,
"plugin_manifest.cfg")
template = Template(template=manifest)
tools = template.sections()
# get repos
if core:
p_helper = PluginHelper(plugins_dir='.internals/plugins/')
repos.add(core_repo)
else:
p_helper = PluginHelper(plugins_dir='plugins/')
for tool in tools[1]:
repo = template.option(tool, 'repo')
if repo[0] and repo[1] != core_repo:
repos.add(repo[1])
# get normal tools
for repo in repos:
status, _ = p_helper.clone(repo)
if status:
p_helper.apply_path(repo)
p_helper.checkout(branch=branch, version=version)
path, _, _ = p_helper.get_path(repo, core=core)
matches = None
if core:
matches = p_helper.available_tools(path, version=version,
groups='core')
else:
matches = p_helper.available_tools(path, version=version)
for match in matches:
if core:
all_tools['normal'].append(match[0].split('/')[-1].replace('_', '-'))
else:
all_tools['normal'].append(match[0].split('/')[-1])
# get tools that have been installed
for tool in tools[1]:
repo = template.option(tool, "repo")
if repo[0] and repo[1] in repos:
name = template.option(tool, "name")
if name[0]:
all_tools['installed'].append(name[1].replace('_', '-'))
# get tools that have been built and/or are running
try:
d_client = docker.from_env()
images = d_client.images.list(filters={'label': 'vent'})
for image in images:
try:
core_check = ("vent.groups" in image.attrs['Config']['Labels'] and
'core' in image.attrs['Config']['Labels']['vent.groups'])
image_check = None
if core:
image_check = core_check
else:
image_check = not core_check
if image_check:
if ('vent.name' in image.attrs['Config']['Labels'] and
'hidden' not in image.attrs['Config']['Labels']['vent.groups']):
if core:
all_tools['built'].append(image.attrs['Config']['Labels']['vent.name'].replace('_', '-'))
else:
all_tools['built'].append(image.attrs['Config']['Labels']['vent.name'])
except Exception as err: # pragma: no cover
self.logger.error("image_check went wrong " + str(err))
containers = d_client.containers.list(filters={'label': 'vent'})
for container in containers:
try:
core_check = ("vent.groups" in container.attrs['Config']['Labels'] and
'core' in container.attrs['Config']['Labels']['vent.groups'])
container_check = None
if core:
container_check = core_check
else:
container_check = not core_check
if container_check:
if ('vent.name' in container.attrs['Config']['Labels'] and
'hidden' not in image.attrs['Config']['Labels']['vent.groups']):
if core:
all_tools['running'].append(container.attrs['Config']['Labels']['vent.name'].replace('_', '-'))
else:
all_tools['running'].append(container.attrs['Config']['Labels']['vent.name'])
except Exception as err: # pragma: no cover
self.logger.error("core_check went wrong " + str(err))
except Exception as e: # pragma: no cover
self.logger.error("Something with docker went wrong " + str(e))
return (len(repos), all_tools)
def cores(self, action, branch="master", version='HEAD'):
"""
Supply action (install, build, start, stop, clean) for core tools
"""
self.logger.info("Starting: cores")
status = (False, None)
try:
self.logger.info("action provided: " + str(action))
core = self.tools_status(True, branch=branch, version=version)[1]
if action in ["install", "build"]:
tools = []
core_repo = 'https://github.com/cyberreboot/vent'
resp = self.p_helper.apply_path(core_repo)
if resp[0]:
cwd = resp[1]
else:
self.logger.info("apply_path failed. Exiting cores"
" with status " + str(resp))
return resp
path = os.path.join(self.plugin.path_dirs.plugins_dir,
'cyberreboot/vent')
response = self.p_helper.checkout(branch=branch,
version=version)
self.logger.info("status of plugin checkout " +
str(response))
matches = self.p_helper.available_tools(path,
version=version,
groups='core')
for match in matches:
name = match[0].rsplit('/')[-1]
constraints = {'name': name,
'repo': core_repo}
prev_installed, _ = self.p_helper. \
constraint_options(constraints, [])
if not prev_installed:
tools.append((match[0], ''))
# only add stuff not already installed or repo specification
if ((tools) or
(isinstance(matches, list) and len(matches) == 0)):
status = self.plugin.add(core_repo,
tools=tools,
branch=branch,
build=False, core=True)
self.logger.info("status of plugin add: " + str(status))
else:
self.logger.info("no new tools to install")
status = (True, "previously installed")
plugin_c = Template(template=self.plugin.manifest)
sections = plugin_c.sections()
for tool in core['normal']:
for section in sections[1]:
name = plugin_c.option(section, "name")
orig_branch = plugin_c.option(section, "branch")
namespace = plugin_c.option(section, "namespace")
version = plugin_c.option(section, "version")
if (name[1] == tool and
orig_branch[1] == branch and
namespace[1] == "cyberreboot/vent" and
version[1] == "HEAD"):
plugin_c.set_option(section,
"image_name",
"cyberreboot/vent-" +
tool.replace('_', '-') + ":" +
branch)
plugin_c.write_config()
chdir(cwd)
if action == "build":
plugin_c = Template(template=self.plugin.manifest)
sections = plugin_c.sections()
try:
for tool in core['normal']:
for section in sections[1]:
tool = tool.replace('_', '-')
image_name = plugin_c.option(section,
"image_name")
check_image = "cyberreboot/vent-"
check_image += tool + ":" + branch
if image_name[1] == check_image:
timestamp = str(datetime.utcnow()) + " UTC"
try:
# currently can't use docker-py because it
# returns a 404 on pull so no way to valid
# if it worked or didn't
image_id = None
cmd = "docker pull " + check_image
output = check_output(shlex.split(cmd),
stderr=STDOUT)
# image_name in format of (bool, image_name)
name = image_name[1]
d_client = docker.from_env()
image_attrs = d_client.images.get(name)
image_attrs = image_attrs.attrs
image_id = image_attrs['Id'].split(':')[1][:12]
if image_id:
plugin_c.set_option(section,
"built",
"yes")
plugin_c.set_option(section,
"image_id",
image_id)
plugin_c.set_option(section,
"last_updated",
timestamp)
status = (True, "Pulled " + tool)
self.logger.info(str(status))
else:
plugin_c.set_option(section,
"built",
"failed")
plugin_c.set_option(section,
"last_updated",
timestamp)
status = (False,
"Failed to pull image " +
str(output.split('\n')[-1]))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
plugin_c.set_option(section,
"built",
"failed")
plugin_c.set_option(section,
"last_updated",
timestamp)
status = (False,
"Failed to pull image " + str(e))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
status = (False, "Failed to pull images " + str(e))
self.logger.error(str(status))
plugin_c.write_config()
elif action == "start":
status = self.api_action.prep_start(groups="core",
branch=branch)
if status[0]:
tool_d = status[1]
status = self.api_action.start(tool_d)
elif action == "stop":
status = self.api_action.stop(groups="core", branch=branch)
elif action == "clean":
status = self.api_action.clean(groups="core", branch=branch)
except Exception as e: # pragma: no cover
self.logger.info("core failed with error: " + str(e))
status = (False, e)
self.logger.info("Status of core: " + str(status[0]))
self.logger.info("Finished: core")
return status
def auto_install(self):
"""
Automatically detects images and installs them in the manifest if they
are not there already
"""
template = Template(template=self.manifest)
sections = template.sections()
images = self.d_client.images.list(filters={'label': 'vent'})
add_sections = []
status = (True, None)
for image in images:
if ('Labels' in image.attrs
and 'vent.section' in image.attrs['Config']['Labels']
and not image.attrs['Config']['Labels']['vent.section']
in sections[1]):
section = image.attrs['Config']['Labels']['vent.section']
section_str = image.attrs['Config']['Labels'][
'vent.section'].split(":")
template.add_section(section)
if 'vent.name' in image.attrs['Config']['Labels']:
template.set_option(
section, 'name',
image.attrs['Config']['Labels']['vent.name'])
if 'vent.repo' in image.attrs['Config']['Labels']:
template.set_option(
section, 'repo',
image.attrs['Config']['Labels']['vent.repo'])
git_path = join(self.path_dirs.plugins_dir,
"/".join(section_str[:2]))
if not isdir(git_path):
# clone it down
status = self.p_helper.clone(
image.attrs['Config']['Labels']['vent.repo'])
template.set_option(section, 'path',
join(git_path, section_str[-3][1:]))
# get template settings
# TODO account for template files not named vent.template
v_template = Template(template=join(
git_path, section_str[-3][1:], 'vent.template'))
tool_sections = v_template.sections()
if tool_sections[0]:
for s in tool_sections[1]:
section_dict = {}
options = v_template.options(s)
if options[0]:
for option in options[1]:
option_name = option
if option == 'name':
# get link name
template.set_option(
section, "link_name",
v_template.option(s, option)[1])
option_name = 'link_name'
opt_val = v_template.option(s, option)[1]
section_dict[option_name] = opt_val
if section_dict:
template.set_option(section, s,
json.dumps(section_dict))
if ('vent.type' in image.attrs['Config']['Labels']
and image.attrs['Config']['Labels']['vent.type']
== 'repository'):
template.set_option(section, 'namespace',
"/".join(section_str[:2]))
template.set_option(section, 'enabled', 'yes')
template.set_option(section, 'branch', section_str[-2])
template.set_option(section, 'version', section_str[-1])
template.set_option(section, 'last_updated',
str(datetime.utcnow()) + " UTC")
template.set_option(section, 'image_name',
image.attrs['RepoTags'][0])
template.set_option(section, 'type', 'repository')
if 'vent.groups' in image.attrs['Config']['Labels']:
template.set_option(
section, 'groups',
image.attrs['Config']['Labels']['vent.groups'])
template.set_option(section, 'built', 'yes')
template.set_option(section, 'image_id',
image.attrs['Id'].split(":")[1][:12])
template.set_option(section, 'running', 'no')
# check if image is running as a container
containers = self.d_client.containers.list(
filters={'label': 'vent'})
for container in containers:
if container.attrs['Image'] == image.attrs['Id']:
template.set_option(section, 'running', 'yes')
add_sections.append(section)
template.write_config()
if status[0]:
status = (True, add_sections)
return status
def prep_start(self,
repo=None,
name=None,
groups=None,
enabled='yes',
branch='master',
version='HEAD'):
"""
Start a set of tools that match the parameters given, if no parameters
are given, start all installed tools on the master branch at verison
HEAD that are enabled
"""
args = locals()
self.logger.info('Starting: prep_start')
self.logger.info('Arguments: ' + str(args))
status = (False, None)
try:
options = [
'name', 'namespace', 'built', 'groups', 'path', 'image_name',
'branch', 'repo', 'type', 'version'
]
vent_config = Template(template=self.path_dirs.cfg_file)
manifest = Template(self.manifest)
files = vent_config.option('main', 'files')
files = (files[0], expanduser(files[1]))
s, _ = self.constraint_options(args, options)
status, tool_d = self.start_sections(s, files, groups, enabled,
branch, version)
# look out for links to delete because they're defined externally
links_to_delete = set()
# get instances for each tool
tool_instances = {}
sections = manifest.sections()[1]
for section in sections:
settings = manifest.option(section, 'settings')
if settings[0]:
settings = json.loads(settings[1])
if 'instances' in settings:
l_name = manifest.option(section, 'link_name')
if l_name[0]:
tool_instances[l_name[1]] = int(
settings['instances'])
# check and update links, volumes_from, network_mode
for container in list(tool_d.keys()):
if 'labels' not in tool_d[
container] or 'vent.groups' not in tool_d[container][
'labels'] or 'core' not in tool_d[container][
'labels']['vent.groups']:
tool_d[container]['remove'] = True
if 'links' in tool_d[container]:
for link in list(tool_d[container]['links'].keys()):
# add links to external services already running if
# necessary, by default configure local services too
configure_local = True
ext = 'external-services'
if link in vent_config.options(ext)[1]:
try:
lconf = json.loads(
vent_config.option(ext, link)[1])
if ('locally_active' not in lconf
or lconf['locally_active'] == 'no'):
ip_adr = lconf['ip_address']
port = lconf['port']
tool_d[container]['extra_hosts'] = {}
# containers use lowercase names for
# connections
tool_d[container]['extra_hosts'][
link.lower()] = ip_adr
# create an environment variable for container
# to access port later
env_variable = link.upper() + \
'_CUSTOM_PORT=' + port
if 'environment' not in tool_d[container]:
tool_d[container]['environment'] = []
tool_d[container]['environment'].append(
env_variable)
# remove the entry from links because no
# longer connecting to local container
links_to_delete.add(link)
configure_local = False
except Exception as e: # pragma: no cover
self.logger.error("couldn't load external"
' settings because: ' +
str(e))
configure_local = True
status = False
if configure_local:
for c in list(tool_d.keys()):
if ('tmp_name' in tool_d[c]
and tool_d[c]['tmp_name'] == link):
tool_d[container]['links'][
tool_d[c]['name']] = tool_d[container][
'links'].pop(link)
if link in tool_instances and tool_instances[
link] > 1:
for i in range(
2, tool_instances[link] + 1):
tool_d[container]['links'][
tool_d[c]['name'] +
str(i)] = tool_d[container][
'links'][tool_d[c]
['name']] + str(i)
if 'volumes_from' in tool_d[container]:
tmp_volumes_from = tool_d[container]['volumes_from']
tool_d[container]['volumes_from'] = []
for volumes_from in list(tmp_volumes_from):
for c in list(tool_d.keys()):
if ('tmp_name' in tool_d[c]
and tool_d[c]['tmp_name'] == volumes_from):
tool_d[container]['volumes_from'].append(
tool_d[c]['name'])
tmp_volumes_from.remove(volumes_from)
tool_d[container]['volumes_from'] += tmp_volumes_from
if 'network_mode' in tool_d[container]:
if tool_d[container]['network_mode'].startswith(
'container:'):
network_c_name = tool_d[container][
'network_mode'].split('container:')[1]
for c in list(tool_d.keys()):
if ('tmp_name' in tool_d[c] and
tool_d[c]['tmp_name'] == network_c_name):
tool_d[container]['network_mode'] = 'container:' + \
tool_d[c]['name']
# remove tmp_names
for c in list(tool_d.keys()):
if 'tmp_name' in tool_d[c]:
del tool_d[c]['tmp_name']
# remove links section if all were externally configured
for c in list(tool_d.keys()):
if 'links' in tool_d[c]:
for link in links_to_delete:
if link in tool_d[c]['links']:
del tool_d[c]['links'][link]
# delete links if no more defined
if not tool_d[c]['links']:
del tool_d[c]['links']
# remove containers that shouldn't be started
for c in list(tool_d.keys()):
deleted = False
if 'start' in tool_d[c] and not tool_d[c]['start']:
del tool_d[c]
deleted = True
if not deleted:
# look for tools services that are being done externally
# tools are capitalized in vent.cfg, so make them lowercase
# for comparison
ext = 'external-services'
external_tools = vent_config.section(ext)[1]
name = tool_d[c]['labels']['vent.name']
for tool in external_tools:
if name == tool[0].lower():
try:
tool_config = json.loads(tool[1])
if ('locally_active' in tool_config and
tool_config['locally_active'] == 'no'):
del tool_d[c]
except Exception as e: # pragma: no cover
self.logger.warning(
'Locally running container ' + name +
' may be redundant')
if status:
status = (True, tool_d)
else:
status = (False, tool_d)
except Exception as e: # pragma: no cover
self.logger.error('prep_start failed with error: ' + str(e))
status = (False, e)
self.logger.info('Status of prep_start: ' + str(status[0]))
self.logger.info('Finished: prep_start')
return status
def start_sections(self, s, files, groups, enabled, branch, version):
""" Run through sections for prep_start """
tool_d = {}
status = (True, None)
for section in s:
# initialize needed vars
c_name = s[section]['image_name'].replace(':', '-')
c_name = c_name.replace('/', '-')
instance_num = re.search(r'\d+$', s[section]['name'])
if instance_num:
c_name += instance_num.group()
image_name = s[section]['image_name']
# checkout the right version and branch of the repo
cwd = getcwd()
self.logger.info('current directory is: ' + str(cwd))
# images built from registry won't have path
if s[section]['path'] != '':
chdir(join(s[section]['path']))
# TODO commenting out for now, should use update_repo
#status = self.checkout(branch=branch, version=version)
status = (True, None)
self.logger.info(status)
chdir(cwd)
tool_d[c_name] = {'image': image_name, 'name': c_name}
# get rid of all commented sections in various runtime
# configurations
manifest = Template(self.manifest)
overall_dict = {}
for setting in ['info', 'docker', 'gpu', 'settings', 'service']:
option = manifest.option(section, setting)
if option[0]:
overall_dict[setting] = {}
settings_dict = json.loads(option[1])
for opt in settings_dict:
if not opt.startswith('#'):
overall_dict[setting][opt] = \
settings_dict[opt]
if 'docker' in overall_dict:
options_dict = overall_dict['docker']
for option in options_dict:
options = options_dict[option]
# check for commands to evaluate
if '`' in options:
cmds = options.split('`')
if len(cmds) > 2:
i = 1
while i < len(cmds):
try:
cmds[i] = check_output(
shlex.split(cmds[i]),
stderr=STDOUT,
close_fds=True).strip().decode('utf-8')
except Exception as e: # pragma: no cover
self.logger.error(
'unable to evaluate command specified in vent.template: '
+ str(e))
i += 2
options = ''.join(cmds)
# check for commands to evaluate
# store options set for docker
try:
tool_d[c_name][option] = literal_eval(options)
except Exception as e: # pragma: no cover
self.logger.info('unable to literal_eval: ' +
str(options))
tool_d[c_name][option] = options
if 'labels' not in tool_d[c_name]:
tool_d[c_name]['labels'] = {}
# get the service uri info
if 'service' in overall_dict:
try:
options_dict = overall_dict['service']
for option in options_dict:
tool_d[c_name]['labels'][option] = options_dict[option]
except Exception as e: # pragma: no cover
self.logger.error('unable to store service options for '
'docker: ' + str(e))
# check for gpu settings
if 'gpu' in overall_dict:
try:
options_dict = json.loads(status[1])
for option in options_dict:
tool_d[c_name]['labels']['gpu.' +
option] = options_dict[option]
except Exception as e: # pragma: no cover
self.logger.error('unable to store gpu options for '
'docker: ' + str(e))
# get temporary name for links, etc.
plugin_c = Template(template=self.manifest)
status, plugin_sections = plugin_c.sections()
self.logger.info(status)
for plugin_section in plugin_sections:
status = plugin_c.option(plugin_section, 'link_name')
self.logger.info(status)
image_status = plugin_c.option(plugin_section, 'image_name')
self.logger.info(image_status)
if status[0] and image_status[0]:
cont_name = image_status[1].replace(':', '-')
cont_name = cont_name.replace('/', '-')
if cont_name not in tool_d:
tool_d[cont_name] = {
'image': image_status[1],
'name': cont_name,
'start': False
}
tool_d[cont_name]['tmp_name'] = status[1]
# add extra labels
tool_d[c_name]['labels']['vent'] = Version()
tool_d[c_name]['labels']['vent.namespace'] = s[section][
'namespace']
tool_d[c_name]['labels']['vent.branch'] = branch
tool_d[c_name]['labels']['vent.version'] = version
tool_d[c_name]['labels']['vent.name'] = s[section]['name']
tool_d[c_name]['labels']['vent.section'] = section
tool_d[c_name]['labels']['vent.repo'] = s[section]['repo']
tool_d[c_name]['labels']['vent.type'] = s[section]['type']
# check for log_config settings in external-services
externally_configured = False
vent_config = Template(self.path_dirs.cfg_file)
for ext_tool in vent_config.section('external-services')[1]:
if ext_tool[0].lower() == 'syslog':
try:
log_dict = json.loads(ext_tool[1])
# configure if not locally active
if ('locally_active' not in log_dict
or log_dict['locally_active'] == 'no'):
del log_dict['locally_active']
log_config = {}
log_config['type'] = 'syslog'
log_config['config'] = {}
ip_address = ''
port = ''
for option in log_dict:
if option == 'ip_address':
ip_address = log_dict[option]
elif option == 'port':
port = log_dict['port']
syslog_address = 'tcp://' + ip_address + ':' + port
syslog_config = {
'syslog-address': syslog_address,
'syslog-facility': 'daemon',
'tag': '{{.Name}}'
}
log_config['config'].update(syslog_config)
externally_configured = True
except Exception as e: # pragma: no cover
self.logger.error('external settings for log_config'
" couldn't be stored because: " +
str(e))
externally_configured = False
if not externally_configured:
log_config = {
'type': 'syslog',
'config': {
'syslog-address': 'tcp://0.0.0.0:514',
'syslog-facility': 'daemon',
'tag': '{{.Name}}'
}
}
if 'groups' in s[section]:
# add labels for groups
tool_d[c_name]['labels']['vent.groups'] = s[section]['groups']
# add restart=always to core containers
if 'core' in s[section]['groups']:
tool_d[c_name]['restart_policy'] = {'Name': 'always'}
# map network names to environment variables
if 'network' in s[section]['groups']:
vent_config = Template(template=self.path_dirs.cfg_file)
nic_mappings = vent_config.section('network-mapping')
nics = ''
if nic_mappings[0]:
for nic in nic_mappings[1]:
nics += nic[0] + ':' + nic[1] + ','
nics = nics[:-1]
if nics:
if 'environment' in tool_d[c_name]:
tool_d[c_name]['environment'].append('VENT_NICS=' +
nics)
else:
tool_d[c_name]['environment'] = [
'VENT_NICS=' + nics
]
# send logs to syslog
if ('syslog' not in s[section]['groups']
and 'core' in s[section]['groups']):
log_config['config']['tag'] = '{{.Name}}'
tool_d[c_name]['log_config'] = log_config
if 'syslog' not in s[section]['groups']:
tool_d[c_name]['log_config'] = log_config
# mount necessary directories
if 'files' in s[section]['groups']:
ulimits = []
ulimits.append(
docker.types.Ulimit(name='nofile',
soft=1048576,
hard=1048576))
tool_d[c_name]['ulimits'] = ulimits
# check if running in a docker container
if 'VENT_CONTAINERIZED' in environ and environ[
'VENT_CONTAINERIZED'] == 'true':
if 'volumes_from' in tool_d[c_name]:
tool_d[c_name]['volumes_from'].append(
environ['HOSTNAME'])
else:
tool_d[c_name]['volumes_from'] = [
environ['HOSTNAME']
]
else:
if 'volumes' in tool_d[c_name]:
tool_d[c_name]['volumes'][
self.path_dirs.base_dir[:-1]] = {
'bind': '/vent',
'mode': 'ro'
}
else:
tool_d[c_name]['volumes'] = {
self.path_dirs.base_dir[:-1]: {
'bind': '/vent',
'mode': 'ro'
}
}
if files[0]:
if 'volumes' in tool_d[c_name]:
tool_d[c_name]['volumes'][files[1]] = {
'bind': '/files',
'mode': 'rw'
}
else:
tool_d[c_name]['volumes'] = {
files[1]: {
'bind': '/files',
'mode': 'rw'
}
}
else:
tool_d[c_name]['log_config'] = log_config
# add label for priority
if 'settings' in overall_dict:
try:
options_dict = overall_dict['settings']
for option in options_dict:
if option == 'priority':
tool_d[c_name]['labels'][
'vent.priority'] = options_dict[option]
except Exception as e: # pragma: no cover
self.logger.error('unable to store settings options '
'for docker ' + str(e))
# only start tools that have been built
if s[section]['built'] != 'yes':
del tool_d[c_name]
# store section information for adding info to manifest later
else:
tool_d[c_name]['section'] = section
return status, tool_d
def cores(self, action, branch='master', version='HEAD'):
"""
Supply action (install, build, start, stop, clean) for core tools
"""
self.logger.info('Starting: cores')
status = (False, None)
try:
self.logger.info('action provided: ' + str(action))
core = self.tools_status(True, branch=branch, version=version)[1]
if action in ['install', 'build']:
tools = []
core_repo = 'https://github.com/cyberreboot/vent'
resp = self.p_helper.apply_path(core_repo)
if resp[0]:
cwd = resp[1]
else:
self.logger.info('apply_path failed. Exiting cores'
' with status ' + str(resp))
return resp
path = os.path.join(self.plugin.path_dirs.plugins_dir,
'cyberreboot/vent')
# TODO commenting out for now, should use update_repo
# response = self.p_helper.checkout(branch=branch,
# version=version)
response = (True, None)
self.logger.info('status of plugin checkout ' +
str(response))
matches = self.p_helper.available_tools(path,
version=version,
groups='core')
for match in matches:
name = match[0].rsplit('/')[-1]
constraints = {'name': name,
'repo': core_repo}
prev_installed, _ = self.p_helper. \
constraint_options(constraints, [])
if not prev_installed:
tools.append((match[0], ''))
# only add stuff not already installed or repo specification
if ((tools) or
(isinstance(matches, list) and len(matches) == 0)):
status = self.plugin.add(core_repo,
tools=tools,
branch=branch,
build=False, core=True)
self.logger.info('status of plugin add: ' + str(status))
else:
self.logger.info('no new tools to install')
status = (True, 'previously installed')
plugin_c = Template(template=self.plugin.manifest)
sections = plugin_c.sections()
for tool in core['normal']:
for section in sections[1]:
name = plugin_c.option(section, 'name')
orig_branch = plugin_c.option(section, 'branch')
namespace = plugin_c.option(section, 'namespace')
version = plugin_c.option(section, 'version')
if (name[1] == tool and
orig_branch[1] == branch and
namespace[1] == 'cyberreboot/vent' and
version[1] == 'HEAD'):
plugin_c.set_option(section,
'image_name',
'cyberreboot/vent-' +
tool.replace('_', '-') + ':' +
branch)
plugin_c.write_config()
chdir(cwd)
if action == 'build':
plugin_c = Template(template=self.plugin.manifest)
sections = plugin_c.sections()
try:
for tool in core['normal']:
for section in sections[1]:
tool = tool.replace('_', '-')
image_name = plugin_c.option(section,
'image_name')
check_image = 'cyberreboot/vent-'
check_image += tool + ':' + branch
if image_name[1] == check_image:
timestamp = str(datetime.utcnow()) + ' UTC'
try:
# currently can't use docker-py because it
# returns a 404 on pull so no way to valid
# if it worked or didn't
image_id = None
cmd = 'docker pull ' + check_image
output = check_output(shlex.split(cmd),
stderr=STDOUT).decode('utf-8')
# image_name in format of (bool, image_name)
name = image_name[1]
d_client = docker.from_env()
image_attrs = d_client.images.get(name)
image_attrs = image_attrs.attrs
image_id = image_attrs['Id'].split(':')[
1][:12]
if image_id:
plugin_c.set_option(section,
'built',
'yes')
plugin_c.set_option(section,
'image_id',
image_id)
plugin_c.set_option(section,
'last_updated',
timestamp)
status = (True, 'Pulled ' + tool)
self.logger.info(str(status))
else:
plugin_c.set_option(section,
'built',
'failed')
plugin_c.set_option(section,
'last_updated',
timestamp)
status = (False,
'Failed to pull image ' +
str(output.split('\n')[-1]))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
plugin_c.set_option(section,
'built',
'failed')
plugin_c.set_option(section,
'last_updated',
timestamp)
status = (False,
'Failed to pull image ' + str(e))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
status = (False, 'Failed to pull images ' + str(e))
self.logger.error(str(status))
plugin_c.write_config()
elif action == 'start':
status = self.api_action.prep_start(groups='core',
branch=branch)
if status[0]:
tool_d = status[1]
status = self.api_action.start(tool_d)
elif action == 'stop':
status = self.api_action.stop(groups='core', branch=branch)
elif action == 'clean':
status = self.api_action.clean(groups='core', branch=branch)
except Exception as e: # pragma: no cover
self.logger.info('core failed with error: ' + str(e))
status = (False, e)
self.logger.info('Status of core: ' + str(status[0]))
self.logger.info('Finished: core')
return status
def prep_start(self,
repo=None,
name=None,
groups=None,
enabled="yes",
branch="master",
version="HEAD",
run_build=False):
"""
Start a set of tools that match the parameters given, if no parameters
are given, start all installed tools on the master branch at verison
HEAD that are enabled
"""
args = locals()
self.logger.info("Starting: prep_start")
self.logger.info("Arguments: " + str(args))
status = (True, None)
tool_dict = {}
try:
del args['run_build']
options = [
'name', 'namespace', 'built', 'groups', 'path', 'image_name',
'branch', 'version'
]
vent_config = Template(template=self.vent_config)
files = vent_config.option('main', 'files')
sections, template = self.plugin.constraint_options(args, options)
for section in sections:
# initialize needed vars
template_path = os.path.join(sections[section]['path'],
'vent.template')
container_name = sections[section]['image_name'].replace(
':', '-')
container_name = container_name.replace('/', '-')
image_name = sections[section]['image_name']
# checkout the right version and branch of the repo
self.plugin.branch = branch
self.plugin.version = version
cwd = os.getcwd()
self.logger.info("current directory is: " + str(cwd))
os.chdir(os.path.join(sections[section]['path']))
status = self.plugin.checkout()
self.logger.info(status)
os.chdir(cwd)
if run_build:
status = self.build(name=sections[section]['name'],
groups=groups,
enabled=enabled,
branch=branch,
version=version)
self.logger.info(status)
# set docker settings for container
vent_template = Template(template_path)
status = vent_template.section('docker')
self.logger.info(status)
tool_dict[container_name] = {
'image': image_name,
'name': container_name
}
if status[0]:
for option in status[1]:
options = option[1]
# check for commands to evaluate
if '`' in options:
cmds = options.split('`')
# TODO this probably needs better error checking to handle mismatched ``
if len(cmds) > 2:
i = 1
while i < len(cmds):
try:
cmds[i] = subprocess.check_output(
shlex.split(cmds[i]),
stderr=subprocess.STDOUT,
close_fds=True).strip()
except Exception as e: # pragma: no cover
self.logger.error(
"unable to evaluate command specified in vent.template: "
+ str(e))
i += 2
options = "".join(cmds)
# store options set for docker
try:
tool_dict[container_name][
option[0]] = ast.literal_eval(options)
except Exception as e: # pragma: no cover
self.logger.error(
"unable to store the options set for docker: "
+ str(e))
tool_dict[container_name][option[0]] = options
# get temporary name for links, etc.
status = vent_template.section('info')
self.logger.info(status)
plugin_config = Template(template=self.plugin.manifest)
status, plugin_sections = plugin_config.sections()
self.logger.info(status)
for plugin_section in plugin_sections:
status = plugin_config.option(plugin_section, "link_name")
self.logger.info(status)
image_status = plugin_config.option(
plugin_section, "image_name")
self.logger.info(image_status)
if status[0] and image_status[0]:
cont_name = image_status[1].replace(':', '-')
cont_name = cont_name.replace('/', '-')
if cont_name not in tool_dict:
tool_dict[cont_name] = {
'image': image_status[1],
'name': cont_name,
'start': False
}
tool_dict[cont_name]['tmp_name'] = status[1]
# add extra labels
if 'labels' not in tool_dict[container_name]:
tool_dict[container_name]['labels'] = {}
tool_dict[container_name]['labels']['vent'] = Version()
tool_dict[container_name]['labels'][
'vent.namespace'] = sections[section]['namespace']
tool_dict[container_name]['labels']['vent.branch'] = branch
tool_dict[container_name]['labels']['vent.version'] = version
tool_dict[container_name]['labels']['vent.name'] = sections[
section]['name']
if 'groups' in sections[section]:
# add labels for groups
tool_dict[container_name]['labels'][
'vent.groups'] = sections[section]['groups']
# send logs to syslog
if 'syslog' not in sections[section][
'groups'] and 'core' in sections[section]['groups']:
tool_dict[container_name]['log_config'] = {
'type': 'syslog',
'config': {
'syslog-address': 'tcp://0.0.0.0:514',
'syslog-facility': 'daemon',
'tag': 'core'
}
}
if 'syslog' not in sections[section]['groups']:
tool_dict[container_name]['log_config'] = {
'type': 'syslog',
'config': {
'syslog-address': 'tcp://0.0.0.0:514',
'syslog-facility': 'daemon',
'tag': 'plugin'
}
}
# mount necessary directories
if 'files' in sections[section]['groups']:
if 'volumes' in tool_dict[container_name]:
tool_dict[container_name]['volumes'][
self.plugin.path_dirs.base_dir[:-1]] = {
'bind': '/vent',
'mode': 'ro'
}
else:
tool_dict[container_name]['volumes'] = {
self.plugin.path_dirs.base_dir[:-1]: {
'bind': '/vent',
'mode': 'ro'
}
}
if files[0]:
tool_dict[container_name]['volumes'][files[1]] = {
'bind': '/files',
'mode': 'ro'
}
else:
tool_dict[container_name]['log_config'] = {
'type': 'syslog',
'config': {
'syslog-address': 'tcp://0.0.0.0:514',
'syslog-facility': 'daemon',
'tag': 'plugin'
}
}
# add label for priority
status = vent_template.section('settings')
self.logger.info(status)
if status[0]:
for option in status[1]:
if option[0] == 'priority':
tool_dict[container_name]['labels'][
'vent.priority'] = option[1]
# only start tools that have been built
if sections[section]['built'] != 'yes':
del tool_dict[container_name]
# check and update links, volumes_from, network_mode
for container in tool_dict.keys():
if 'links' in tool_dict[container]:
for link in tool_dict[container]['links']:
for c in tool_dict.keys():
if 'tmp_name' in tool_dict[c] and tool_dict[c][
'tmp_name'] == link:
tool_dict[container]['links'][
tool_dict[c]['name']] = tool_dict[
container]['links'].pop(link)
if 'volumes_from' in tool_dict[container]:
tmp_volumes_from = tool_dict[container]['volumes_from']
tool_dict[container]['volumes_from'] = []
for volumes_from in list(tmp_volumes_from):
for c in tool_dict.keys():
if 'tmp_name' in tool_dict[c] and tool_dict[c][
'tmp_name'] == volumes_from:
tool_dict[container]['volumes_from'].append(
tool_dict[c]['name'])
tmp_volumes_from.remove(volumes_from)
tool_dict[container]['volumes_from'] += tmp_volumes_from
if 'network_mode' in tool_dict[container]:
if tool_dict[container]['network_mode'].startswith(
'container:'):
network_c_name = tool_dict[container][
'network_mode'].split('container:')[1]
for c in tool_dict.keys():
if 'tmp_name' in tool_dict[c] and tool_dict[c][
'tmp_name'] == network_c_name:
tool_dict[container][
'network_mode'] = 'container:' + tool_dict[
c]['name']
# remove tmp_names
for c in tool_dict.keys():
if 'tmp_name' in tool_dict[c]:
del tool_dict[c]['tmp_name']
# remove containers that shouldn't be started
for c in tool_dict.keys():
if 'start' in tool_dict[c] and not tool_dict[c]['start']:
del tool_dict[c]
except Exception as e:
self.logger.error("prep_start failed with error: " + str(e))
status = (False, e)
status = (True, tool_dict)
self.logger.info("Status of prep_start: " + str(status))
self.logger.info("Finished: prep_start")
return status
def cores(self, action, branch="master"):
""" Supply action (install, build, start, stop, clean) for core tools """
self.logger.info("Starting: cores")
status = (True, None)
try:
self.logger.info("action provided: " + str(action))
core = Core(branch=branch)
if action in ["install", "build"]:
tools = []
plugins = Plugin(plugins_dir=".internals/plugins")
plugins.version = 'HEAD'
plugins.branch = branch
plugins.apply_path('https://github.com/cyberreboot/vent')
response = plugins.checkout()
self.logger.info("status of plugin checkout " + str(response))
matches = plugins._available_tools(groups='core')
for match in matches:
tools.append((match[0], ''))
status = plugins.add('https://github.com/cyberreboot/vent',
tools=tools,
branch=branch,
build=False)
self.logger.info("status of plugin add: " + str(status))
plugin_config = Template(template=self.plugin.manifest)
sections = plugin_config.sections()
for tool in core['normal']:
for section in sections[1]:
name = plugin_config.option(section, "name")
orig_branch = plugin_config.option(section, "branch")
namespace = plugin_config.option(section, "namespace")
version = plugin_config.option(section, "version")
if name[1] == tool and orig_branch[
1] == branch and namespace[
1] == "cyberreboot/vent" and version[
1] == "HEAD":
plugin_config.set_option(
section, "image_name",
"cyberreboot/vent-" + tool + ":" + branch)
plugin_config.write_config()
if action == "build":
plugin_config = Template(template=self.plugin.manifest)
sections = plugin_config.sections()
try:
for tool in core['normal']:
for section in sections[1]:
image_name = plugin_config.option(
section, "image_name")
if image_name[
1] == "cyberreboot/vent-" + tool + ":" + branch:
try:
# currently can't use docker-py because it
# returns a 404 on pull so no way to valid if it
# worked or didn't
#image_id = self.d_client.images.pull('cyberreboot/vent-'+tool, tag=branch)
image_id = None
output = subprocess.check_output(
shlex.split(
"docker pull cyberreboot/vent-" +
tool + ":" + branch),
stderr=subprocess.STDOUT)
for line in output.split('\n'):
if line.startswith("Digest: sha256:"):
image_id = line.split(
"Digest: sha256:")[1][:12]
if image_id:
plugin_config.set_option(
section, "built", "yes")
plugin_config.set_option(
section, "image_id", image_id)
plugin_config.set_option(
section, "last_updated",
str(datetime.datetime.utcnow()) +
" UTC")
status = (True, "Pulled " + tool)
self.logger.info(str(status))
else:
plugin_config.set_option(
section, "built", "failed")
plugin_config.set_option(
section, "last_updated",
str(datetime.datetime.utcnow()) +
" UTC")
status = (False,
"Failed to pull image " +
str(output.split('\n')[-1]))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
plugin_config.set_option(
section, "built", "failed")
plugin_config.set_option(
section, "last_updated",
str(datetime.datetime.utcnow()) +
" UTC")
status = (False,
"Failed to pull image " + str(e))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
status = (False, "Failed to pull images " + str(e))
self.logger.error(str(status))
plugin_config.write_config()
elif action == "start":
status = self.prep_start(groups="core", branch=branch)
if status[0]:
tool_dict = status[1]
status = self.start(tool_dict)
elif action == "stop":
status = self.stop(groups="core", branch=branch)
elif action == "clean":
status = self.clean(groups="core", branch=branch)
except Exception as e:
self.logger.info("core failed with error: " + str(e))
status = (False, e)
self.logger.info("Status of core: " + str(status))
self.logger.info("Finished: core")
return status
def prep_start(self,
repo=None,
name=None,
groups=None,
enabled="yes",
branch="master",
version="HEAD",
run_build=False):
"""
Start a set of tools that match the parameters given, if no parameters
are given, start all installed tools on the master branch at verison
HEAD that are enabled
"""
args = locals()
self.logger.info("Starting: prep_start")
self.logger.info("Arguments: "+str(args))
status = (True, None)
tool_dict = {}
try:
del args['run_build']
options = ['name',
'namespace',
'built',
'groups',
'path',
'image_name',
'branch',
'version']
vent_config = Template(template=self.vent_config)
files = vent_config.option('main', 'files')
sections, template = self.plugin.constraint_options(args, options)
for section in sections:
# initialize needed vars
template_path = os.path.join(sections[section]['path'], 'vent.template')
container_name = sections[section]['image_name'].replace(':','-')
container_name = container_name.replace('/','-')
image_name = sections[section]['image_name']
# checkout the right version and branch of the repo
self.plugin.branch = branch
self.plugin.version = version
cwd = os.getcwd()
self.logger.info("current directory is: "+str(cwd))
os.chdir(os.path.join(sections[section]['path']))
status = self.plugin.checkout()
self.logger.info(status)
os.chdir(cwd)
if run_build:
status = self.build(name=sections[section]['name'],
groups=groups,
enabled=enabled,
branch=branch,
version=version)
self.logger.info(status)
# set docker settings for container
vent_template = Template(template_path)
status = vent_template.section('docker')
self.logger.info(status)
tool_dict[container_name] = {'image':image_name, 'name':container_name}
if status[0]:
for option in status[1]:
options = option[1]
# check for commands to evaluate
if '`' in options:
cmds = options.split('`')
# TODO this probably needs better error checking to handle mismatched ``
if len(cmds) > 2:
i = 1
while i < len(cmds):
try:
cmds[i] = subprocess.check_output(shlex.split(cmds[i]), stderr=subprocess.STDOUT, close_fds=True).strip()
except Exception as e: # pragma: no cover
self.logger.error("unable to evaluate command specified in vent.template: "+str(e))
i += 2
options = "".join(cmds)
# store options set for docker
try:
tool_dict[container_name][option[0]] = ast.literal_eval(options)
except Exception as e: # pragma: no cover
self.logger.error("unable to store the options set for docker: "+str(e))
tool_dict[container_name][option[0]] = options
# get temporary name for links, etc.
status = vent_template.section('info')
self.logger.info(status)
plugin_config = Template(template=self.plugin.manifest)
status, plugin_sections = plugin_config.sections()
self.logger.info(status)
for plugin_section in plugin_sections:
status = plugin_config.option(plugin_section, "link_name")
self.logger.info(status)
image_status = plugin_config.option(plugin_section, "image_name")
self.logger.info(image_status)
if status[0] and image_status[0]:
cont_name = image_status[1].replace(':','-')
cont_name = cont_name.replace('/','-')
if cont_name not in tool_dict:
tool_dict[cont_name] = {'image':image_status[1], 'name':cont_name, 'start':False}
tool_dict[cont_name]['tmp_name'] = status[1]
# add extra labels
if 'labels' not in tool_dict[container_name]:
tool_dict[container_name]['labels'] = {}
tool_dict[container_name]['labels']['vent'] = Version()
tool_dict[container_name]['labels']['vent.namespace'] = sections[section]['namespace']
tool_dict[container_name]['labels']['vent.branch'] = branch
tool_dict[container_name]['labels']['vent.version'] = version
tool_dict[container_name]['labels']['vent.name'] = sections[section]['name']
if 'groups' in sections[section]:
# add labels for groups
tool_dict[container_name]['labels']['vent.groups'] = sections[section]['groups']
# send logs to syslog
if 'syslog' not in sections[section]['groups'] and 'core' in sections[section]['groups']:
tool_dict[container_name]['log_config'] = {'type':'syslog', 'config': {'syslog-address':'tcp://0.0.0.0:514', 'syslog-facility':'daemon', 'tag':'core'}}
if 'syslog' not in sections[section]['groups']:
tool_dict[container_name]['log_config'] = {'type':'syslog', 'config': {'syslog-address':'tcp://0.0.0.0:514', 'syslog-facility':'daemon', 'tag':'plugin'}}
# mount necessary directories
if 'files' in sections[section]['groups']:
if 'volumes' in tool_dict[container_name]:
tool_dict[container_name]['volumes'][self.plugin.path_dirs.base_dir[:-1]] = {'bind': '/vent', 'mode': 'ro'}
else:
tool_dict[container_name]['volumes'] = {self.plugin.path_dirs.base_dir[:-1]: {'bind': '/vent', 'mode': 'ro'}}
if files[0]:
tool_dict[container_name]['volumes'][files[1]] = {'bind': '/files', 'mode': 'ro'}
else:
tool_dict[container_name]['log_config'] = {'type':'syslog', 'config': {'syslog-address':'tcp://0.0.0.0:514', 'syslog-facility':'daemon', 'tag':'plugin'}}
# add label for priority
status = vent_template.section('settings')
self.logger.info(status)
if status[0]:
for option in status[1]:
if option[0] == 'priority':
tool_dict[container_name]['labels']['vent.priority'] = option[1]
# only start tools that have been built
if sections[section]['built'] != 'yes':
del tool_dict[container_name]
# check and update links, volumes_from, network_mode
for container in tool_dict.keys():
if 'links' in tool_dict[container]:
for link in tool_dict[container]['links']:
for c in tool_dict.keys():
if 'tmp_name' in tool_dict[c] and tool_dict[c]['tmp_name'] == link:
tool_dict[container]['links'][tool_dict[c]['name']] = tool_dict[container]['links'].pop(link)
if 'volumes_from' in tool_dict[container]:
tmp_volumes_from = tool_dict[container]['volumes_from']
tool_dict[container]['volumes_from'] = []
for volumes_from in list(tmp_volumes_from):
for c in tool_dict.keys():
if 'tmp_name' in tool_dict[c] and tool_dict[c]['tmp_name'] == volumes_from:
tool_dict[container]['volumes_from'].append(tool_dict[c]['name'])
tmp_volumes_from.remove(volumes_from)
tool_dict[container]['volumes_from'] += tmp_volumes_from
if 'network_mode' in tool_dict[container]:
if tool_dict[container]['network_mode'].startswith('container:'):
network_c_name = tool_dict[container]['network_mode'].split('container:')[1]
for c in tool_dict.keys():
if 'tmp_name' in tool_dict[c] and tool_dict[c]['tmp_name'] == network_c_name:
tool_dict[container]['network_mode'] = 'container:'+tool_dict[c]['name']
# remove tmp_names
for c in tool_dict.keys():
if 'tmp_name' in tool_dict[c]:
del tool_dict[c]['tmp_name']
# remove containers that shouldn't be started
for c in tool_dict.keys():
if 'start' in tool_dict[c] and not tool_dict[c]['start']:
del tool_dict[c]
except Exception as e:
self.logger.error("prep_start failed with error: "+str(e))
status = (False, e)
status = (True, tool_dict)
self.logger.info("Status of prep_start: "+str(status))
self.logger.info("Finished: prep_start")
return status
def cores(self, action, branch="master"):
""" Supply action (install, build, start, stop, clean) for core tools """
self.logger.info("Starting: cores")
status = (True, None)
try:
self.logger.info("action provided: "+str(action))
core = Core(branch=branch)
if action in ["install", "build"]:
tools = []
plugins = Plugin(plugins_dir=".internals/plugins")
plugins.version = 'HEAD'
plugins.branch = branch
plugins.apply_path('https://github.com/cyberreboot/vent')
response = plugins.checkout()
self.logger.info("status of plugin checkout "+str(response))
matches = plugins._available_tools(groups='core')
for match in matches:
tools.append((match[0], ''))
status = plugins.add('https://github.com/cyberreboot/vent', tools=tools, branch=branch, build=False)
self.logger.info("status of plugin add: "+str(status))
plugin_config = Template(template=self.plugin.manifest)
sections = plugin_config.sections()
for tool in core['normal']:
for section in sections[1]:
name = plugin_config.option(section, "name")
orig_branch = plugin_config.option(section, "branch")
namespace = plugin_config.option(section, "namespace")
version = plugin_config.option(section, "version")
if name[1] == tool and orig_branch[1] == branch and namespace[1] == "cyberreboot/vent" and version[1] == "HEAD":
plugin_config.set_option(section, "image_name", "cyberreboot/vent-"+tool+":"+branch)
plugin_config.write_config()
if action == "build":
plugin_config = Template(template=self.plugin.manifest)
sections = plugin_config.sections()
try:
for tool in core['normal']:
for section in sections[1]:
image_name = plugin_config.option(section, "image_name")
if image_name[1] == "cyberreboot/vent-"+tool+":"+branch:
try:
# currently can't use docker-py because it
# returns a 404 on pull so no way to valid if it
# worked or didn't
#image_id = self.d_client.images.pull('cyberreboot/vent-'+tool, tag=branch)
image_id = None
output = subprocess.check_output(shlex.split("docker pull cyberreboot/vent-"+tool+":"+branch), stderr=subprocess.STDOUT)
for line in output.split('\n'):
if line.startswith("Digest: sha256:"):
image_id = line.split("Digest: sha256:")[1][:12]
if image_id:
plugin_config.set_option(section, "built", "yes")
plugin_config.set_option(section, "image_id", image_id)
plugin_config.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC")
status = (True, "Pulled "+tool)
self.logger.info(str(status))
else:
plugin_config.set_option(section, "built", "failed")
plugin_config.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC")
status = (False, "Failed to pull image "+str(output.split('\n')[-1]))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
plugin_config.set_option(section, "built", "failed")
plugin_config.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC")
status = (False, "Failed to pull image "+str(e))
self.logger.error(str(status))
except Exception as e: # pragma: no cover
status = (False, "Failed to pull images "+str(e))
self.logger.error(str(status))
plugin_config.write_config()
elif action == "start":
status = self.prep_start(groups="core", branch=branch)
if status[0]:
tool_dict = status[1]
status = self.start(tool_dict)
elif action == "stop":
status = self.stop(groups="core", branch=branch)
elif action == "clean":
status = self.clean(groups="core", branch=branch)
except Exception as e:
self.logger.info("core failed with error: "+str(e))
status = (False, e)
self.logger.info("Status of core: "+str(status))
self.logger.info("Finished: core")
return status