def host_config(self): """ Ensure the host configuration file exists """ if platform.system() == 'Darwin': default_file_dir = os.path.join(os.path.expanduser('~'), 'vent_files') else: default_file_dir = '/opt/vent_files' status = self.ensure_dir(default_file_dir) config = Template(template=self.cfg_file) sections = { 'main': { 'files': default_file_dir }, 'network-mapping': {}, 'nvidia-docker-plugin': { 'port': '3476' } } for s in sections: if sections[s]: for option in sections[s]: config.add_option(s, option, sections[s][option]) else: config.add_section(s) config.write_config() return status
def auto_install(self): """ Automatically detects images and installs them in the manifest if they are not there already """ template = Template(template=self.manifest) sections = template.sections() images = self.d_client.images.list(filters={'label': 'vent'}) add_sections = [] status = (True, None) for image in images: if ('Labels' in image.attrs and 'vent.section' in image.attrs['Config']['Labels'] and not image.attrs['Config']['Labels']['vent.section'] in sections[1]): section = image.attrs['Config']['Labels']['vent.section'] section_str = image.attrs['Config']['Labels'][ 'vent.section'].split(":") template.add_section(section) if 'vent.name' in image.attrs['Config']['Labels']: template.set_option( section, 'name', image.attrs['Config']['Labels']['vent.name']) if 'vent.repo' in image.attrs['Config']['Labels']: template.set_option( section, 'repo', image.attrs['Config']['Labels']['vent.repo']) git_path = join(self.path_dirs.plugins_dir, "/".join(section_str[:2])) if not isdir(git_path): # clone it down status = self.p_helper.clone( image.attrs['Config']['Labels']['vent.repo']) template.set_option(section, 'path', join(git_path, section_str[-3][1:])) # get template settings # TODO account for template files not named vent.template v_template = Template(template=join( git_path, section_str[-3][1:], 'vent.template')) tool_sections = v_template.sections() if tool_sections[0]: for s in tool_sections[1]: section_dict = {} options = v_template.options(s) if options[0]: for option in options[1]: option_name = option if option == 'name': # get link name template.set_option( section, "link_name", v_template.option(s, option)[1]) option_name = 'link_name' opt_val = v_template.option(s, option)[1] section_dict[option_name] = opt_val if section_dict: template.set_option(section, s, json.dumps(section_dict)) if ('vent.type' in image.attrs['Config']['Labels'] and image.attrs['Config']['Labels']['vent.type'] == 'repository'): template.set_option(section, 'namespace', "/".join(section_str[:2])) template.set_option(section, 'enabled', 'yes') template.set_option(section, 'branch', section_str[-2]) template.set_option(section, 'version', section_str[-1]) template.set_option(section, 'last_updated', str(datetime.utcnow()) + " UTC") template.set_option(section, 'image_name', image.attrs['RepoTags'][0]) template.set_option(section, 'type', 'repository') if 'vent.groups' in image.attrs['Config']['Labels']: template.set_option( section, 'groups', image.attrs['Config']['Labels']['vent.groups']) template.set_option(section, 'built', 'yes') template.set_option(section, 'image_id', image.attrs['Id'].split(":")[1][:12]) template.set_option(section, 'running', 'no') # check if image is running as a container containers = self.d_client.containers.list( filters={'label': 'vent'}) for container in containers: if container.attrs['Image'] == image.attrs['Id']: template.set_option(section, 'running', 'yes') add_sections.append(section) template.write_config() if status[0]: status = (True, add_sections) return status
def _build_manifest(self, matches): """ Builds and writes the manifest for the tools being added """ # !! TODO check for pre-existing that conflict with request and # disable and/or remove image for match in matches: # keep track of whether or not to write an additional manifest # entry for multiple instances, and how many additional entries # to write addtl_entries = 0 # remove the .git for adding repo info to manifest if self.repo.endswith('.git'): self.repo = self.repo[:-4] # remove @ in match for template setting purposes if match[0].find('@') >= 0: true_name = match[0].split('@')[1] else: true_name = match[0] template = Template(template=self.manifest) # TODO check for special settings here first for the specific match self.version = match[1] response = self.p_helper.checkout(branch=self.branch, version=self.version) if response[0]: section = self.org + ":" + self.name + ":" + true_name + ":" section += self.branch + ":" + self.version # need to get rid of temp identifiers for tools in same repo match_path = self.path + match[0].split('@')[0] if not self.core: image_name = self.org + "-" + self.name + "-" if match[0] != '': # if tool is in a subdir, add that to the name of the # image image_name += '-'.join(match[0].split('/')[1:]) + "-" image_name += self.branch + ":" + self.version else: image_name = ('cyberreboot/vent-' + match[0].split('/')[-1] + ':' + self.branch) image_name = image_name.replace('_', '-') # check if the section already exists exists, options = template.section(section) previous_commit = None previous_commits = None head = False if exists: for option in options: # TODO check if tool name but a different version # exists - then disable/remove if set if option[0] == 'version' and option[1] == 'HEAD': head = True if option[0] == 'built' and option[1] == 'yes': # !! TODO remove pre-existing image pass if option[0] == 'commit_id': previous_commit = option[1] if option[0] == 'previous_versions': previous_commits = option[1] # check if tool comes from multi directory multi_tool = "no" if match[0].find('@') >= 0: multi_tool = "yes" # !! TODO # check if section should be removed from config i.e. all tools # but new commit removed one that was in a previous commit image_name = image_name.lower() if image_name.endswith(":head"): image_name = image_name.split(":head")[0] + ":HEAD" # set template section & options for tool at version and branch template.add_section(section) template.set_option(section, "name", true_name.split('/')[-1]) template.set_option(section, "namespace", self.org + '/' + self.name) template.set_option(section, "path", match_path) template.set_option(section, "repo", self.repo) template.set_option(section, "enabled", "yes") template.set_option(section, "multi_tool", multi_tool) template.set_option(section, "branch", self.branch) template.set_option(section, "version", self.version) template.set_option(section, "last_updated", str(datetime.utcnow()) + " UTC") template.set_option(section, "image_name", image_name.replace('@', '-')) template.set_option(section, "type", "repository") # save settings in vent.template to plugin_manifest # watch for multiple tools in same directory # just wanted to store match path with @ for path for use in # other actions tool_template = 'vent.template' if match[0].find('@') >= 0: tool_template = match[0].split('@')[1] + '.template' vent_template_path = join(match_path, tool_template) if os.path.exists(vent_template_path): with open(vent_template_path) as f: vent_template_val = f.read() else: vent_template_val = '' settings_dict = ParsedSections(vent_template_val) for setting in settings_dict: template.set_option(section, setting, json.dumps(settings_dict[setting])) # TODO do we need this if we save as a dictionary? vent_template = Template(vent_template_path) vent_status, response = vent_template.option("info", "name") if vent_status: template.set_option(section, "link_name", response) else: template.set_option(section, "link_name", true_name.split('/')[-1]) commit_id = None if self.version == 'HEAD': # remove @ in multi-tools chdir(match_path) cmd = "git rev-parse --short HEAD" commit_id = check_output(shlex.split(cmd), stderr=STDOUT, close_fds=True).strip() template.set_option(section, "commit_id", commit_id) if head: # no need to store previous commits if not HEAD, since # the version will always be the same commit ID if previous_commit and previous_commit != commit_id: if (previous_commits and previous_commit not in previous_commits): previous_commits = (previous_commit + ',' + previous_commits) elif not previous_commits: previous_commits = previous_commit if previous_commits and previous_commits != commit_id: template.set_option(section, "previous_versions", previous_commits) if self.version_alias: template.set_option(section, "version_alias", self.version_alias) if self.groups: template.set_option(section, "groups", self.groups) else: groups = vent_template.option("info", "groups") if groups[0]: template.set_option(section, "groups", groups[1]) # set groups to empty string if no groups defined for tool else: template.set_option(section, "groups", '') template = self._build_image(template, match_path, image_name, section) # write additional entries for multiple instances if addtl_entries > 0: # add 2 for naming conventions for i in range(2, addtl_entries + 2): addtl_section = section.rsplit(':', 2) addtl_section[0] += str(i) addtl_section = ':'.join(addtl_section) template.add_section(addtl_section) orig_vals = template.section(section)[1] for val in orig_vals: template.set_option(addtl_section, val[0], val[1]) template.set_option(addtl_section, "name", true_name.split('/')[-1] + str(i)) # write out configuration to the manifest file template.write_config() # reset to repo directory chdir(self.path) return
def add_image(self, image, link_name, tag=None, registry=None, groups=None): """ Add an image with a tag from a Docker registry. Defaults to the Docker Hub if not specified. Use a Template object to write an image's information to `plugin_manifest.cfg' Args: image(type): docker image link_name(type): fill me Kwargs: tag(type): registry(type): groups(type): Group that the docker image belongs to. Returns: tuple(bool,str): if the function completed successfully, (True, name of image). If the function failed, (False, message about failure) """ status = (True, None) try: pull_name = image org = '' name = image if '/' in image: org, name = image.split('/') else: org = "official" if not tag: tag = "latest" if not registry: registry = "docker.io" full_image = registry + "/" + image + ":" + tag image = self.d_client.images.pull(full_image) section = ':'.join([registry, org, name, '', tag]) namespace = org + '/' + name # set template section and options for tool at version and branch template = Template(template=self.manifest) template.add_section(section) template.set_option(section, "name", name) template.set_option(section, "pull_name", pull_name) template.set_option(section, "namespace", namespace) template.set_option(section, "path", "") template.set_option(section, "repo", registry + '/' + org) template.set_option(section, "enabled", "yes") template.set_option(section, "branch", "") template.set_option(section, "version", tag) template.set_option(section, "last_updated", str(datetime.utcnow()) + " UTC") template.set_option(section, "image_name", image.attrs['RepoTags'][0]) template.set_option(section, "type", "registry") template.set_option(section, "link_name", link_name) template.set_option(section, "commit_id", "") template.set_option(section, "built", "yes") template.set_option(section, "image_id", image.attrs['Id'].split(':')[1][:12]) template.set_option(section, "groups", groups) # write out configuration to the manifest file template.write_config() status = (True, "Successfully added " + full_image) except Exception as e: # pragma: no cover self.logger.error("Couldn't add image because " + str(e)) status = (False, str(e)) return status
def __init__(self, repo='', tool_name='', branch='', version='', next_tool=None, just_downloaded=False, vent_cfg=False, from_registry=False, new_instance=False, *args, **keywords): """ Initialize EditorForm objects """ # default for any editor self.settings = locals() self.settings.update(keywords) del self.settings['self'] del self.settings['args'] del self.settings['keywords'] del self.settings['parentApp'] self.p_helper = PluginHelper(plugins_dir='.internals/') self.tool_identifier = { 'name': tool_name, 'branch': branch, 'version': version } self.settings.update(self.tool_identifier) del self.settings['name'] self.settings['tool_name'] = tool_name self.settings['next_tool'] = next_tool self.settings['repo'] = repo # setup checks self.just_downloaded = ('just_downloaded' in self.settings and self.settings['just_downloaded']) self.vent_cfg = ('vent_cfg' in self.settings and self.settings['vent_cfg']) self.registry_tool = ('from_registry' in self.settings and self.settings['from_registry']) self.instance_cfg = ('new_instance' in self.settings and self.settings['new_instance']) # get manifest info for tool that will be used throughout if not self.just_downloaded and not self.vent_cfg: result = self.p_helper.constraint_options(self.tool_identifier, []) tool, self.manifest = result self.section = tool.keys()[0] # get configuration information depending on type if self.just_downloaded: self.config_val = '[info]\n' self.config_val += 'name = ' + keywords['link_name'] + '\n' self.config_val += 'groups = ' + keywords['groups'] + '\n' elif self.vent_cfg: self.config_val = keywords['get_configure'](main_cfg=True)[1] self.settings['tool_name'] = 'vent configuration' elif self.instance_cfg: path = self.manifest.option(self.section, 'path')[1] # defaults in .internals path = path.replace('.vent/plugins', '.vent/.internals') multi_tool = self.manifest.option(self.section, 'multi_tool') if multi_tool[0] and multi_tool[1] == 'yes': name = self.manifest.option(self.section, 'name')[1] if name == 'unspecified': name = 'vent' template_path = os.path.join(path, name + '.template') else: template_path = os.path.join(path, 'vent.template') # ensure instances is in the editor and that it is the right number template = Template(template_path) template.add_section('settings') template.set_option('settings', 'instances', str(self.settings['new_instances'])) template.write_config() with open(template_path) as vent_template: self.config_val = vent_template.read() else: self.config_val = keywords['get_configure']( **self.tool_identifier)[1] super(EditorForm, self).__init__(*args, **keywords)
def _build_manifest(self, matches): """ Builds and writes the manifest for the tools being added """ # !! TODO check for pre-existing that conflict with request and disable and/or remove image for match in matches: template = Template(template=self.manifest) # !! TODO check for special settings here first for the specific match self.version = match[1] response = self.checkout() if response[0]: section = self.org + ":" + self.name + ":" + match[0] + ":" + self.branch + ":" + self.version match_path = self.path + match[0] image_name = self.org + "-" + self.name + "-" if match[0] != '': # if tool is in a subdir, add that to the name of the image image_name += '-'.join(match[0].split('/')[1:]) + "-" image_name += self.branch + ":" + self.version # check if the section already exists exists, options = template.section(section) previous_commit = None previous_commits = None head = False if exists: for option in options: # TODO check if tool name but a different version exists - then disable/remove if set if option[0] == 'version' and option[1] == 'HEAD': head = True if option[0] == 'built' and option[1] == 'yes': # !! TODO remove pre-existing image pass if option[0] == 'commit_id': previous_commit = option[1] if option[0] == 'previous_versions': previous_commits = option[1] # !! TODO # check if section should be removed from config - i.e. all tools, # but new commit removed one that was in a previous commit # set template section and options for tool at version and branch template.add_section(section) template.set_option(section, "name", match[0].split('/')[-1]) template.set_option(section, "namespace", self.org+'/'+self.name) template.set_option(section, "path", match_path) template.set_option(section, "repo", self.repo) template.set_option(section, "enabled", "yes") template.set_option(section, "branch", self.branch) template.set_option(section, "version", self.version) template.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") template.set_option(section, "image_name", image_name) vent_template = Template(template=os.path.join(match_path, 'vent.template')) vent_status, response = vent_template.option("info", "name") if vent_status: template.set_option(section, "link_name", response) else: template.set_option(section, "link_name", match[0].split('/')[-1]) commit_id = None if self.version == 'HEAD': os.chdir(match_path) commit_id = subprocess.check_output(shlex.split("git rev-parse --short HEAD"), stderr=subprocess.STDOUT, close_fds=True).strip() template.set_option(section, "commit_id", commit_id) if head: # no need to store previous commits if not HEAD, since # the version will always be the same commit ID if previous_commit and previous_commit != commit_id: if previous_commits and previous_commit not in previous_commits: previous_commits = previous_commit+','+previous_commits elif not previous_commits: previous_commits = previous_commit if previous_commits and previous_commits != commit_id: template.set_option(section, "previous_versions", previous_commits) if self.version_alias: template.set_option(section, "version_alias", self.version_alias) if self.groups: template.set_option(section, "groups", self.groups) else: vent_template = os.path.join(match_path, 'vent.template') if os.path.exists(vent_template): v_template = Template(template=vent_template) groups = v_template.option("info", "groups") if groups[0]: template.set_option(section, "groups", groups[1]) template = self._build_image(template, match_path, image_name, section) # write out configuration to the manifest file template.write_config() # reset to repo directory os.chdir(self.path) return
def _build_manifest(self, matches): """ Builds and writes the manifest for the tools being added """ # !! TODO check for pre-existing that conflict with request and disable and/or remove image for match in matches: template = Template(template=self.manifest) # !! TODO check for special settings here first for the specific match self.version = match[1] response = self.checkout() if response[0]: section = self.org + ":" + self.name + ":" + match[ 0] + ":" + self.branch + ":" + self.version match_path = self.path + match[0] image_name = self.org + "-" + self.name + "-" if match[0] != '': # if tool is in a subdir, add that to the name of the image image_name += '-'.join(match[0].split('/')[1:]) + "-" image_name += self.branch + ":" + self.version # check if the section already exists exists, options = template.section(section) previous_commit = None previous_commits = None head = False if exists: for option in options: # TODO check if tool name but a different version exists - then disable/remove if set if option[0] == 'version' and option[1] == 'HEAD': head = True if option[0] == 'built' and option[1] == 'yes': # !! TODO remove pre-existing image pass if option[0] == 'commit_id': previous_commit = option[1] if option[0] == 'previous_versions': previous_commits = option[1] # !! TODO # check if section should be removed from config - i.e. all tools, # but new commit removed one that was in a previous commit # set template section and options for tool at version and branch template.add_section(section) template.set_option(section, "name", match[0].split('/')[-1]) template.set_option(section, "namespace", self.org + '/' + self.name) template.set_option(section, "path", match_path) template.set_option(section, "repo", self.repo) template.set_option(section, "enabled", "yes") template.set_option(section, "branch", self.branch) template.set_option(section, "version", self.version) template.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") template.set_option(section, "image_name", image_name) vent_template = Template( template=os.path.join(match_path, 'vent.template')) vent_status, response = vent_template.option("info", "name") if vent_status: template.set_option(section, "link_name", response) else: template.set_option(section, "link_name", match[0].split('/')[-1]) commit_id = None if self.version == 'HEAD': os.chdir(match_path) commit_id = subprocess.check_output( shlex.split("git rev-parse --short HEAD"), stderr=subprocess.STDOUT, close_fds=True).strip() template.set_option(section, "commit_id", commit_id) if head: # no need to store previous commits if not HEAD, since # the version will always be the same commit ID if previous_commit and previous_commit != commit_id: if previous_commits and previous_commit not in previous_commits: previous_commits = previous_commit + ',' + previous_commits elif not previous_commits: previous_commits = previous_commit if previous_commits and previous_commits != commit_id: template.set_option(section, "previous_versions", previous_commits) if self.version_alias: template.set_option(section, "version_alias", self.version_alias) if self.groups: template.set_option(section, "groups", self.groups) else: vent_template = os.path.join(match_path, 'vent.template') if os.path.exists(vent_template): v_template = Template(template=vent_template) groups = v_template.option("info", "groups") if groups[0]: template.set_option(section, "groups", groups[1]) template = self._build_image(template, match_path, image_name, section) # write out configuration to the manifest file template.write_config() # reset to repo directory os.chdir(self.path) return