def Checkout(path, branch='master', version='HEAD', **kargs): status = (True, None) path_dirs = PathDirs(**kargs) status = path_dirs.apply_path(path) git_stdout = '' if status[0]: try: git_stdout += check_output(shlex.split('git checkout ' + branch), stderr=STDOUT, close_fds=True).decode('utf-8') git_stdout += check_output(shlex.split('git pull origin ' + version), stderr=STDOUT, close_fds=True).decode('utf-8') if version: git_stdout += check_output(shlex.split('git reset --hard ' + version), stderr=STDOUT, close_fds=True).decode('utf-8') chdir(status[1]) except Exception as e: # pragma: no cover if str(e).endswith('exit status 128.'): status = (True, 'Repo has already been checked out.') else: logger.error('Checkout failed with error: {0} {1}'.format( str(e), git_stdout)) status = (False, str(e)) return status
def Checkout(path, branch='master', version='HEAD', **kargs): status = (True, None) path_dirs = PathDirs(**kargs) status = path_dirs.apply_path(path) if status[0]: try: check_output(shlex.split('git checkout ' + branch), stderr=STDOUT, close_fds=True).decode('utf-8') check_output(shlex.split('git pull origin ' + version), stderr=STDOUT, close_fds=True).decode('utf-8') if version: check_output(shlex.split('git reset --hard ' + version), stderr=STDOUT, close_fds=True).decode('utf-8') chdir(status[1]) except Exception as e: # pragma: no cover logger.error( 'Checkout failed with error: {0}'.format(str(e))) status = (False, str(e)) return status
class Repository: def __init__(self, manifest, *args, **kwargs): self.path_dirs = PathDirs(**kwargs) self.manifest = manifest self.d_client = docker.from_env() self.logger = Logger(__name__) def add(self, repo, tools=None, overrides=None, version='HEAD', core=False, image_name=None, branch='master', build=True, user=None, pw=None): status = (True, None) self.repo = repo.lower() self.tools = tools self.overrides = overrides self.branch = branch self.version = version self.image_name = image_name self.core = core status = self._clone(user=user, pw=pw) if status[0] and build: status = self._build() return status def _build(self): status = (True, None) status = self._get_tools() matches = status[1] status = self.path_dirs.apply_path(self.repo) original_path = status[1] if status[0] and len(matches) > 0: repo, org, name = self.path_dirs.get_path(self.repo) cmd = 'git rev-parse --short ' + self.version commit_id = '' try: commit_id = check_output(shlex.split(cmd), stderr=STDOUT, close_fds=True).strip().decode('utf-8') except Exception as e: # pragma: no cover self.logger.error( 'Unable to get commit ID because: {0}'.format(str(e))) template = Template(template=self.manifest) for match in matches: status, template, match_path, image_name, section = self._build_manifest( match, template, repo, org, name, commit_id) if not status[0]: break status, template = self._build_image(template, match_path, image_name, section) if not status[0]: break if status[0]: # write out configuration to the manifest file template.write_config() chdir(original_path) return status def _get_tools(self): status = (True, None) matches = [] path, _, _ = self.path_dirs.get_path(self.repo) status = Checkout(path, branch=self.branch, version=self.version) if status[0]: if self.tools is None and self.overrides is None: # get all tools matches = AvailableTools( path, branch=self.branch, version=self.version, core=self.core) elif self.tools is None: # there's only something in overrides # grab all the tools then apply overrides matches = AvailableTools( path, branch=self.branch, version=self.version, core=self.core) # !! TODO apply overrides to matches elif self.overrides is None: # there's only something in tools # only grab the tools specified matches = ToolMatches(tools=self.tools, version=self.version) else: # both tools and overrides were specified # grab only the tools specified, with the overrides applied o_matches = ToolMatches(tools=self.tools, version=self.version) matches = o_matches for override in self.overrides: override_t = None if override[0] == '.': override_t = ('', override[1]) else: override_t = override for match in o_matches: if override_t[0] == match[0]: matches.remove(match) matches.append(override_t) status = (True, matches) return status def _build_manifest(self, match, template, repo, org, name, commit_id): status = (True, None) # keep track of whether or not to write an additional manifest # entry for multiple instances, and how many additional entries # to write addtl_entries = 1 # remove @ in match for template setting purposes if match[0].find('@') >= 0: true_name = match[0].split('@')[1] else: true_name = match[0] # TODO check for special settings here first for the specific match self.version = match[1] section = org + ':' + name + ':' + true_name + ':' section += self.branch + ':' + self.version # need to get rid of temp identifiers for tools in same repo match_path = repo + match[0].split('@')[0] if self.image_name: image_name = self.image_name elif not self.core: image_name = org + '/' + name if match[0] != '': # if tool is in a subdir, add that to the name of the # image image_name += '-' + '-'.join(match[0].split('/')[1:]) image_name += ':' + self.branch else: image_name = ('cyberreboot/vent-' + match[0].split('/')[-1] + ':' + self.branch) image_name = image_name.replace('_', '-') # check if the section already exists is_there, options = template.section(section) previous_commit = None previous_commits = None head = False if is_there: for option in options: # TODO check if tool name but a different version # exists - then disable/remove if set if option[0] == 'version' and option[1] == 'HEAD': head = True if option[0] == 'built' and option[1] == 'yes': # !! TODO remove pre-existing image pass if option[0] == 'commit_id': previous_commit = option[1] if option[0] == 'previous_versions': previous_commits = option[1] # check if tool comes from multi directory multi_tool = 'no' if match[0].find('@') >= 0: multi_tool = 'yes' # !! TODO # check if section should be removed from config i.e. all tools # but new commit removed one that was in a previous commit image_name = image_name.lower() image_name = image_name.replace('@', '-') # special case for vent images if image_name.startswith('cyberreboot/vent'): image_name = image_name.replace('vent-vent-core-', 'vent-') image_name = image_name.replace('vent-vent-extras-', 'vent-') # set template section & options for tool at version and branch template.add_section(section) template.set_option(section, 'name', true_name.split('/')[-1]) template.set_option(section, 'namespace', org + '/' + name) template.set_option(section, 'path', match_path) template.set_option(section, 'repo', self.repo) template.set_option(section, 'multi_tool', multi_tool) template.set_option(section, 'branch', self.branch) template.set_option(section, 'version', self.version) template.set_option(section, 'last_updated', str(datetime.utcnow()) + ' UTC') template.set_option(section, 'image_name', image_name) template.set_option(section, 'type', 'repository') # save settings in vent.template to plugin_manifest # watch for multiple tools in same directory # just wanted to store match path with @ for path for use in # other actions tool_template = 'vent.template' if match[0].find('@') >= 0: tool_template = match[0].split('@')[1] + '.template' vent_template_path = join(match_path, tool_template) if exists(vent_template_path): with open(vent_template_path, 'r') as f: vent_template_val = f.read() else: vent_template_val = '' settings_dict = ParsedSections(vent_template_val) for setting in settings_dict: template.set_option(section, setting, json.dumps(settings_dict[setting])) # TODO do we need this if we save as a dictionary? vent_template = Template(vent_template_path) vent_status, response = vent_template.option('info', 'name') instances = vent_template.option('settings', 'instances') if instances[0]: addtl_entries = int(instances[1]) if vent_status: template.set_option(section, 'link_name', response) else: template.set_option(section, 'link_name', true_name.split('/')[-1]) if self.version == 'HEAD': template.set_option(section, 'commit_id', commit_id) if head: # no need to store previous commits if not HEAD, since # the version will always be the same commit ID if previous_commit and previous_commit != commit_id: if (previous_commits and previous_commit not in previous_commits): previous_commits = (previous_commit + ',' + previous_commits) elif not previous_commits: previous_commits = previous_commit if previous_commits and previous_commits != commit_id: template.set_option(section, 'previous_versions', previous_commits) groups = vent_template.option('info', 'groups') if groups[0]: template.set_option(section, 'groups', groups[1]) # set groups to empty string if no groups defined for tool else: template.set_option(section, 'groups', '') # write additional entries for multiple instances if addtl_entries > 1: # add 2 for naming conventions for i in range(2, addtl_entries + 1): addtl_section = section.rsplit(':', 2) addtl_section[0] += str(i) addtl_section = ':'.join(addtl_section) template.add_section(addtl_section) orig_vals = template.section(section)[1] for val in orig_vals: template.set_option(addtl_section, val[0], val[1]) template.set_option(addtl_section, 'name', true_name.split('/')[-1]+str(i)) return status, template, match_path, image_name, section def _build_image(self, template, match_path, image_name, section, build_local=False): status = (True, None) output = '' def set_instances(template, section, built, image_id=None): """ Set build information for multiple instances """ i = 2 while True: addtl_section = section.rsplit(':', 2) addtl_section[0] += str(i) addtl_section = ':'.join(addtl_section) if template.section(addtl_section)[0]: template.set_option(addtl_section, 'built', built) if image_id: template.set_option(addtl_section, 'image_id', image_id) template.set_option(addtl_section, 'last_updated', Timestamp()) else: break i += 1 # determine whether a tool should be considered a multi instance multi_instance = False try: settings = template.option(section, 'settings') if settings[0]: settings_dict = json.loads(settings[1]) if 'instances' in settings_dict and int(settings_dict['instances']) > 1: multi_instance = True except Exception as e: # pragma: no cover self.logger.error( 'Failed to check for multi instance because: {0}'.format(str(e))) status = (False, str(e)) cwd = getcwd() chdir(match_path) try: name = template.option(section, 'name') groups = template.option(section, 'groups') t_type = template.option(section, 'type') path = template.option(section, 'path') status, config_override = self.path_dirs.override_config(path[1]) if groups[1] == '' or not groups[0]: groups = (True, 'none') if not name[0]: name = (True, image_name) pull = False image_exists = False cfg_template = Template(template=self.path_dirs.cfg_file) use_existing_image = False result = cfg_template.option('build-options', 'use_existing_images') if result[0]: use_existing_image = result[1] if use_existing_image == 'yes' and not config_override: try: self.d_client.images.get(image_name) i_attrs = self.d_client.images.get(image_name).attrs image_id = i_attrs['Id'].split(':')[1][:12] template.set_option(section, 'built', 'yes') template.set_option(section, 'image_id', image_id) template.set_option(section, 'last_updated', str(datetime.utcnow()) + ' UTC') # set other instances too if multi_instance: set_instances(template, section, 'yes', image_id) status = (True, 'Found {0}'.format(image_name)) self.logger.info(str(status)) image_exists = True except docker.errors.ImageNotFound: image_exists = False except Exception as e: # pragma: no cover self.logger.warning( 'Failed to query Docker for images because: {0}'.format(str(e))) if not image_exists: # pull if '/' in image_name, fallback to build if '/' in image_name and not build_local and not config_override: try: image = self.d_client.images.pull(image_name) i_attrs = self.d_client.images.get( image_name).attrs image_id = i_attrs['Id'].split(':')[1][:12] if image_id: template.set_option(section, 'built', 'yes') template.set_option(section, 'image_id', image_id) template.set_option(section, 'last_updated', str(datetime.utcnow()) + ' UTC') # set other instances too if multi_instance: set_instances(template, section, 'yes', image_id) status = (True, 'Pulled {0}'.format(image_name)) self.logger.info(str(status)) else: template.set_option(section, 'built', 'failed') template.set_option(section, 'last_updated', str(datetime.utcnow()) + ' UTC') # set other instances too if multi_instance: set_instances(template, section, 'failed') status = (False, 'Failed to pull image {0}'.format( str(output.split('\n')[-1]))) self.logger.warning(str(status)) pull = True except Exception as e: # pragma: no cover self.logger.warning( 'Failed to pull image, going to build instead: {0}'.format(str(e))) status = ( False, 'Failed to pull image because: {0}'.format(str(e))) if not pull and not image_exists: # get username to label built image with username = getpass.getuser() # see if additional file arg needed for building multiple # images from same directory file_tag = 'Dockerfile' multi_tool = template.option(section, 'multi_tool') if multi_tool[0] and multi_tool[1] == 'yes': specific_file = template.option(section, 'name')[1] if specific_file != 'unspecified': file_tag = 'Dockerfile.' + specific_file # update image name with new version for update image_name = image_name.rsplit(':', 1)[0]+':'+self.branch labels = {} labels['vent'] = '' labels['vent.section'] = section labels['vent.repo'] = self.repo labels['vent.type'] = t_type[1] labels['vent.name'] = name[1] labels['vent.groups'] = groups[1] labels['built-by'] = username image = self.d_client.images.build(path='.', dockerfile=file_tag, tag=image_name, labels=labels, rm=True) image_id = image[0].id.split(':')[1][:12] template.set_option(section, 'built', 'yes') template.set_option(section, 'image_id', image_id) template.set_option(section, 'last_updated', str(datetime.utcnow()) + ' UTC') # set other instances too if multi_instance: set_instances(template, section, 'yes', image_id) status = (True, 'Built {0}'.format(image_name)) except Exception as e: # pragma: no cover self.logger.error('Unable to build image {0} because: {1} | {2}'.format( str(image_name), str(e), str(output))) template.set_option(section, 'built', 'failed') template.set_option(section, 'last_updated', str(datetime.utcnow()) + ' UTC') if multi_instance: set_instances(template, section, 'failed') status = ( False, 'Failed to build image because: {0}'.format(str(e))) chdir(cwd) template.set_option(section, 'running', 'no') return status, template def _clone(self, user=None, pw=None): status = (True, None) try: # if path already exists, ignore try: path, _, _ = self.path_dirs.get_path(self.repo) chdir(path) return status except Exception as e: # pragma: no cover self.logger.debug("Repo doesn't exist, attempting to clone.") status = self.path_dirs.apply_path(self.repo) if not status[0]: self.logger.error( 'Unable to clone because: {0}'.format(str(status[1]))) return status repo = self.repo # check if user and pw were supplied, typically for private repos if user and pw: # only https is supported when using user/pw auth_repo = 'https://' + user + ':' + pw + '@' repo = auth_repo + repo.split('https://')[-1] # clone repo check_output(shlex.split('env GIT_SSL_NO_VERIFY=true git clone --recursive ' + repo + ' .'), stderr=STDOUT, close_fds=True).decode('utf-8') chdir(status[1]) status = (True, 'Successfully cloned: {0}'.format(self.repo)) except Exception as e: # pragma: no cover e_str = str(e) # scrub username and password from error message if e_str.find('@') >= 0: e_str = e_str[:e_str.find('//') + 2] + \ e_str[e_str.find('@') + 1:] self.logger.error('Clone failed with error: {0}'.format(e_str)) status = (False, e_str) return status def update(self, repo, tools=None): # TODO return
class Tools: def __init__(self, version='HEAD', branch='master', user=None, pw=None, *args, **kwargs): self.version = version self.branch = branch self.user = user self.pw = pw self.d_client = docker.from_env() self.path_dirs = PathDirs(**kwargs) self.path_dirs.host_config() self.manifest = join(self.path_dirs.meta_dir, 'plugin_manifest.cfg') self.logger = Logger(__name__) def new(self, tool_type, uri, tools=None, link_name=None, image_name=None, overrides=None, tag=None, registry=None, groups=None): try: # remove tools that are already installed from being added if isinstance(tools, list): i = len(tools) - 1 while i >= 0: tool = tools[i] if tool[0].find('@') >= 0: tool_name = tool[0].split('@')[-1] else: tool_name = tool[0].rsplit('/', 1)[-1] constraints = {'name': tool_name, 'repo': uri.split('.git')[0]} prev_installed, _ = Template(template=self.manifest).constrain_opts(constraints, []) # don't reinstall if prev_installed: tools.remove(tool) i -= 1 if len(tools) == 0: tools = None except Exception as e: # pragma: no cover self.logger.error('Add failed with error: {0}'.format(str(e))) return (False, str(e)) if tool_type == 'image': status = Image(self.manifest).add(uri, link_name, tag=tag, registry=registry, groups=groups) else: if tool_type == 'core': uri = 'https://github.com/cyberreboot/vent' core = True else: core = False status = Repository(self.manifest).add(uri, tools, overrides=overrides, version=self.version, image_name=image_name, branch=self.branch, user=self.user, pw=self.pw, core=core) return status def configure(self, tool): # TODO return def inventory(self, choices=None): """ Return a dictionary of the inventory items and status """ status = (True, None) if not choices: return (False, 'No choices made') try: # choices: repos, tools, images, built, running, enabled items = {'repos': [], 'tools': {}, 'images': {}, 'built': {}, 'running': {}, 'enabled': {}} tools = Template(self.manifest).list_tools() for choice in choices: for tool in tools: try: if choice == 'repos': if 'repo' in tool: if (tool['repo'] and tool['repo'] not in items[choice]): items[choice].append(tool['repo']) elif choice == 'tools': items[choice][tool['section']] = tool['name'] elif choice == 'images': # TODO also check against docker items[choice][tool['section']] = tool['image_name'] elif choice == 'built': items[choice][tool['section']] = tool['built'] elif choice == 'running': containers = Containers() status = 'not running' for container in containers: image_name = tool['image_name'] \ .rsplit(':' + tool['version'], 1)[0] image_name = image_name.replace(':', '-') image_name = image_name.replace('/', '-') self.logger.info('image_name: ' + image_name) if container[0] == image_name: status = container[1] elif container[0] == image_name + \ '-' + tool['version']: status = container[1] items[choice][tool['section']] = status elif choice == 'enabled': items[choice][tool['section']] = tool['enabled'] else: # unknown choice pass except Exception as e: # pragma: no cover self.logger.error('Unable to grab info about tool: ' + str(tool) + ' because: ' + str(e)) status = (True, items) except Exception as e: # pragma: no cover self.logger.error( 'Inventory failed with error: {0}'.format(str(e))) status = (False, str(e)) return status def remove(self, repo, name): args = locals() status = (True, None) # get resulting dict of sections with options that match constraints template = Template(template=self.manifest) results, _ = template.constrain_opts(args, []) for result in results: response, image_name = template.option(result, 'image_name') name = template.option(result, 'name')[1] try: settings_dict = json.loads(template.option(result, 'settings')[1]) instances = int(settings_dict['instances']) except Exception: instances = 1 try: # check for container and remove c_name = image_name.replace(':', '-').replace('/', '-') for i in range(1, instances + 1): container_name = c_name + str(i) if i != 1 else c_name container = self.d_client.containers.get(container_name) response = container.remove(v=True, force=True) self.logger.info( 'Removing container: {0}'.format(container_name)) except Exception as e: # pragma: no cover self.logger.warning('Unable to remove the container: ' + container_name + ' because: ' + str(e)) # check for image and remove try: response = None image_id = template.option(result, 'image_id')[1] response = self.d_client.images.remove(image_id, force=True) self.logger.info('Removing image: ' + image_name) except Exception as e: # pragma: no cover self.logger.warning('Unable to remove the image: ' + image_name + ' because: ' + str(e)) # remove tool from the manifest for i in range(1, instances + 1): res = result.rsplit(':', 2) res[0] += str(i) if i != 1 else '' res = ':'.join(res) if template.section(res)[0]: status = template.del_section(res) self.logger.info('Removing tool: ' + res) # TODO if all tools from a repo have been removed, remove the repo template.write_config() return status def start(self, repo, name, is_tool_d=False): if is_tool_d: tool_d = repo else: args = locals() del args['self'] del args['is_tool_d'] tool_d = {} tool_d.update(self._prep_start(**args)[1]) status = (True, None) try: # check start priorities (priority of groups alphabetical for now) group_orders = {} groups = [] containers_remaining = [] username = getpass.getuser() # remove tools that have the hidden label tool_d_copy = copy.deepcopy(tool_d) for container in tool_d_copy: if 'labels' in tool_d_copy[container] and 'vent.groups' in tool_d_copy[container]['labels']: groups_copy = tool_d_copy[container]['labels']['vent.groups'].split( ',') if 'hidden' in groups_copy: del tool_d[container] for container in tool_d: containers_remaining.append(container) self.logger.info( "User: '******' starting container: {1}".format(username, container)) if 'labels' in tool_d[container]: if 'vent.groups' in tool_d[container]['labels']: groups += tool_d[container]['labels']['vent.groups'].split( ',') if 'vent.priority' in tool_d[container]['labels']: priorities = tool_d[container]['labels']['vent.priority'].split( ',') container_groups = tool_d[container]['labels']['vent.groups'].split( ',') for i, priority in enumerate(priorities): if container_groups[i] not in group_orders: group_orders[container_groups[i]] = [] group_orders[container_groups[i]].append( (int(priority), container)) containers_remaining.remove(container) tool_d[container]['labels'].update( {'started-by': username} ) else: tool_d[container].update( {'labels': {'started-by': username}} ) # start containers based on priorities p_results = self._start_priority_containers(groups, group_orders, tool_d) # start the rest of the containers that didn't have any priorities r_results = self._start_remaining_containers( containers_remaining, tool_d) results = (p_results[0] + r_results[0], p_results[1] + r_results[1]) if len(results[1]) > 0: status = (False, results) else: status = (True, results) except Exception as e: # pragma: no cover self.logger.error('Start failed with error: {0}'.format(str(e))) status = (False, str(e)) return status def _prep_start(self, repo, name): args = locals() status = (True, None) try: options = ['name', 'namespace', 'built', 'groups', 'path', 'image_name', 'branch', 'repo', 'type', 'version'] vent_config = Template(template=self.path_dirs.cfg_file) manifest = Template(self.manifest) files = vent_config.option('main', 'files') files = (files[0], expanduser(files[1])) s, _ = manifest.constrain_opts(args, options) status, tool_d = self._start_sections(s, files) # look out for links to delete because they're defined externally links_to_delete = set() # get instances for each tool tool_instances = {} sections = manifest.sections()[1] for section in sections: settings = manifest.option(section, 'settings') if settings[0]: settings = json.loads(settings[1]) if 'instances' in settings: l_name = manifest.option(section, 'link_name') if l_name[0]: tool_instances[l_name[1]] = int( settings['instances']) # check and update links, volumes_from, network_mode for container in list(tool_d.keys()): if 'labels' not in tool_d[container] or 'vent.groups' not in tool_d[container]['labels'] or 'core' not in tool_d[container]['labels']['vent.groups']: tool_d[container]['remove'] = True if 'links' in tool_d[container]: for link in list(tool_d[container]['links'].keys()): # add links to external services already running if # necessary, by default configure local services too configure_local = True ext = 'external-services' if link in vent_config.options(ext)[1]: try: lconf = json.loads(vent_config.option(ext, link)[1]) if ('locally_active' not in lconf or lconf['locally_active'] == 'no'): ip_adr = lconf['ip_address'] port = lconf['port'] tool_d[container]['extra_hosts'] = {} # containers use lowercase names for # connections tool_d[container]['extra_hosts'][link.lower() ] = ip_adr # create an environment variable for container # to access port later env_variable = link.upper() + \ '_CUSTOM_PORT=' + port if 'environment' not in tool_d[container]: tool_d[container]['environment'] = [] tool_d[container]['environment'].append( env_variable) # remove the entry from links because no # longer connecting to local container links_to_delete.add(link) configure_local = False except Exception as e: # pragma: no cover self.logger.error( 'Could not load external settings because: {0}'.format(str(e))) configure_local = True status = False if configure_local: for c in list(tool_d.keys()): if ('tmp_name' in tool_d[c] and tool_d[c]['tmp_name'] == link): tool_d[container]['links'][tool_d[c]['name'] ] = tool_d[container]['links'].pop(link) if link in tool_instances and tool_instances[link] > 1: for i in range(2, tool_instances[link] + 1): tool_d[container]['links'][tool_d[c]['name'] + str( i)] = tool_d[container]['links'][tool_d[c]['name']] + str(i) if 'volumes_from' in tool_d[container]: tmp_volumes_from = tool_d[container]['volumes_from'] tool_d[container]['volumes_from'] = [] for volumes_from in list(tmp_volumes_from): for c in list(tool_d.keys()): if ('tmp_name' in tool_d[c] and tool_d[c]['tmp_name'] == volumes_from): tool_d[container]['volumes_from'].append( tool_d[c]['name']) tmp_volumes_from.remove(volumes_from) tool_d[container]['volumes_from'] += tmp_volumes_from if 'network_mode' in tool_d[container]: if tool_d[container]['network_mode'].startswith('container:'): network_c_name = tool_d[container]['network_mode'].split('container:')[ 1] for c in list(tool_d.keys()): if ('tmp_name' in tool_d[c] and tool_d[c]['tmp_name'] == network_c_name): tool_d[container]['network_mode'] = 'container:' + \ tool_d[c]['name'] # remove tmp_names for c in list(tool_d.keys()): if 'tmp_name' in tool_d[c]: del tool_d[c]['tmp_name'] # remove links section if all were externally configured for c in list(tool_d.keys()): if 'links' in tool_d[c]: for link in links_to_delete: if link in tool_d[c]['links']: del tool_d[c]['links'][link] # delete links if no more defined if not tool_d[c]['links']: del tool_d[c]['links'] # remove containers that shouldn't be started for c in list(tool_d.keys()): deleted = False if 'start' in tool_d[c] and not tool_d[c]['start']: del tool_d[c] deleted = True if not deleted: # look for tools services that are being done externally # tools are capitalized in vent.cfg, so make them lowercase # for comparison ext = 'external-services' external_tools = vent_config.section(ext)[1] name = tool_d[c]['labels']['vent.name'] for tool in external_tools: if name == tool[0].lower(): try: tool_config = json.loads(tool[1]) if ('locally_active' in tool_config and tool_config['locally_active'] == 'no'): del tool_d[c] except Exception as e: # pragma: no cover self.logger.warning('Locally running container ' + name + ' may be redundant') if status: status = (True, tool_d) else: status = (False, tool_d) except Exception as e: # pragma: no cover self.logger.error('_prep_start failed with error: '+str(e)) status = (False, e) return status def _start_sections(self, s, files): tool_d = {} status = (True, None) for section in s: # initialize needed vars c_name = s[section]['image_name'].replace(':', '-') c_name = c_name.replace('/', '-') instance_num = re.search(r'\d+$', s[section]['name']) if instance_num: c_name += instance_num.group() image_name = s[section]['image_name'] # checkout the right version and branch of the repo tool_d[c_name] = {'image': image_name, 'name': c_name} # get rid of all commented sections in various runtime # configurations manifest = Template(self.manifest) overall_dict = {} for setting in ['info', 'docker', 'gpu', 'settings', 'service']: option = manifest.option(section, setting) if option[0]: overall_dict[setting] = {} settings_dict = json.loads(option[1]) for opt in settings_dict: if not opt.startswith('#'): overall_dict[setting][opt] = settings_dict[opt] if 'docker' in overall_dict: options_dict = overall_dict['docker'] for option in options_dict: options = options_dict[option] # check for commands to evaluate if '`' in options: cmds = options.split('`') if len(cmds) > 2: i = 1 while i < len(cmds): try: cmds[i] = check_output(shlex.split(cmds[i]), stderr=STDOUT, close_fds=True).strip().decode('utf-8') except Exception as e: # pragma: no cover self.logger.error( 'unable to evaluate command specified in vent.template: ' + str(e)) i += 2 options = ''.join(cmds) # check for commands to evaluate # store options set for docker try: tool_d[c_name][option] = ast.literal_eval(options) except Exception as e: # pragma: no cover self.logger.debug( 'Unable to literal_eval: {0}'.format(str(options))) tool_d[c_name][option] = options if 'labels' not in tool_d[c_name]: tool_d[c_name]['labels'] = {} # get the service uri info if 'service' in overall_dict: try: options_dict = overall_dict['service'] for option in options_dict: tool_d[c_name]['labels'][option] = options_dict[option] except Exception as e: # pragma: no cover self.logger.error('unable to store service options for ' 'docker: ' + str(e)) # check for gpu settings if 'gpu' in overall_dict: try: options_dict = json.loads(status[1]) for option in options_dict: tool_d[c_name]['labels']['gpu.' + option] = options_dict[option] except Exception as e: # pragma: no cover self.logger.error('unable to store gpu options for ' 'docker: ' + str(e)) # get temporary name for links, etc. plugin_c = Template(template=self.manifest) status, plugin_sections = plugin_c.sections() for plugin_section in plugin_sections: status = plugin_c.option(plugin_section, 'link_name') image_status = plugin_c.option(plugin_section, 'image_name') if status[0] and image_status[0]: cont_name = image_status[1].replace(':', '-') cont_name = cont_name.replace('/', '-') if cont_name not in tool_d: tool_d[cont_name] = {'image': image_status[1], 'name': cont_name, 'start': False} tool_d[cont_name]['tmp_name'] = status[1] # add extra labels tool_d[c_name]['labels']['vent'] = Version() tool_d[c_name]['labels']['vent.namespace'] = s[section]['namespace'] tool_d[c_name]['labels']['vent.branch'] = s[section]['branch'] tool_d[c_name]['labels']['vent.version'] = s[section]['version'] tool_d[c_name]['labels']['vent.name'] = s[section]['name'] tool_d[c_name]['labels']['vent.section'] = section tool_d[c_name]['labels']['vent.repo'] = s[section]['repo'] tool_d[c_name]['labels']['vent.type'] = s[section]['type'] # check for log_config settings in external-services externally_configured = False vent_config = Template(self.path_dirs.cfg_file) for ext_tool in vent_config.section('external-services')[1]: if ext_tool[0].lower() == 'syslog': try: log_dict = json.loads(ext_tool[1]) # configure if not locally active if ('locally_active' not in log_dict or log_dict['locally_active'] == 'no'): del log_dict['locally_active'] log_config = {} log_config['type'] = 'syslog' log_config['config'] = {} ip_address = '' port = '' for option in log_dict: if option == 'ip_address': ip_address = log_dict[option] elif option == 'port': port = log_dict['port'] syslog_address = 'tcp://' + ip_address + ':' + port syslog_config = {'syslog-address': syslog_address, 'syslog-facility': 'daemon', 'tag': '{{.Name}}'} log_config['config'].update(syslog_config) externally_configured = True except Exception as e: # pragma: no cover self.logger.error('external settings for log_config' " couldn't be stored because: " + str(e)) externally_configured = False if not externally_configured: log_config = {'type': 'syslog', 'config': {'syslog-address': 'tcp://0.0.0.0:514', 'syslog-facility': 'daemon', 'tag': '{{.Name}}'}} if 'groups' in s[section]: # add labels for groups tool_d[c_name]['labels']['vent.groups'] = s[section]['groups'] # add restart=always to core containers if 'core' in s[section]['groups']: tool_d[c_name]['restart_policy'] = {'Name': 'always'} # map network names to environment variables if 'network' in s[section]['groups']: vent_config = Template(template=self.path_dirs.cfg_file) nic_mappings = vent_config.section('network-mapping') nics = '' if nic_mappings[0]: for nic in nic_mappings[1]: nics += nic[0] + ':' + nic[1] + ',' nics = nics[:-1] if nics: if 'environment' in tool_d[c_name]: tool_d[c_name]['environment'].append( 'VENT_NICS='+nics) else: tool_d[c_name]['environment'] = ['VENT_NICS='+nics] # send logs to syslog if ('syslog' not in s[section]['groups'] and 'core' in s[section]['groups']): log_config['config']['tag'] = '{{.Name}}' tool_d[c_name]['log_config'] = log_config if 'syslog' not in s[section]['groups']: tool_d[c_name]['log_config'] = log_config # mount necessary directories if 'files' in s[section]['groups']: ulimits = [] ulimits.append(docker.types.Ulimit( name='nofile', soft=1048576, hard=1048576)) tool_d[c_name]['ulimits'] = ulimits # check if running in a docker container if 'VENT_CONTAINERIZED' in environ and environ['VENT_CONTAINERIZED'] == 'true': if 'volumes_from' in tool_d[c_name]: tool_d[c_name]['volumes_from'].append( environ['HOSTNAME']) else: tool_d[c_name]['volumes_from'] = [ environ['HOSTNAME']] else: if 'volumes' in tool_d[c_name]: tool_d[c_name]['volumes'][self.path_dirs.base_dir[:-1] ] = {'bind': '/vent', 'mode': 'ro'} else: tool_d[c_name]['volumes'] = { self.path_dirs.base_dir[:-1]: {'bind': '/vent', 'mode': 'ro'}} if files[0]: if 'volumes' in tool_d[c_name]: tool_d[c_name]['volumes'][files[1]] = { 'bind': '/files', 'mode': 'rw'} else: tool_d[c_name]['volumes'] = { files[1]: {'bind': '/files', 'mode': 'rw'}} else: tool_d[c_name]['log_config'] = log_config # add label for priority if 'settings' in overall_dict: try: options_dict = overall_dict['settings'] for option in options_dict: if option == 'priority': tool_d[c_name]['labels']['vent.priority'] = options_dict[option] except Exception as e: # pragma: no cover self.logger.error('unable to store settings options ' 'for docker ' + str(e)) # only start tools that have been built if s[section]['built'] != 'yes': del tool_d[c_name] # store section information for adding info to manifest later else: tool_d[c_name]['section'] = section return status, tool_d def _start_priority_containers(self, groups, group_orders, tool_d): """ Select containers based on priorities to start """ vent_cfg = Template(self.path_dirs.cfg_file) cfg_groups = vent_cfg.option('groups', 'start_order') if cfg_groups[0]: cfg_groups = cfg_groups[1].split(',') else: cfg_groups = [] all_groups = sorted(set(groups)) s_conts = [] f_conts = [] # start tools in order of group defined in vent.cfg for group in cfg_groups: # remove from all_groups because already checked out if group in all_groups: all_groups.remove(group) if group in group_orders: for cont_t in sorted(group_orders[group]): if cont_t[1] not in s_conts: s_conts, f_conts = self._start_container(cont_t[1], tool_d, s_conts, f_conts) # start tools that haven't been specified in the vent.cfg, if any for group in all_groups: if group in group_orders: for cont_t in sorted(group_orders[group]): if cont_t[1] not in s_conts: s_conts, f_conts = self._start_container(cont_t[1], tool_d, s_conts, f_conts) return (s_conts, f_conts) def _start_remaining_containers(self, containers_remaining, tool_d): """ Select remaining containers that didn't have priorities to start """ s_containers = [] f_containers = [] for container in containers_remaining: s_containers, f_containers = self._start_container(container, tool_d, s_containers, f_containers) return (s_containers, f_containers) def _start_container(self, container, tool_d, s_containers, f_containers): """ Start container that was passed in and return status """ # use section to add info to manifest section = tool_d[container]['section'] del tool_d[container]['section'] manifest = Template(self.manifest) try: # try to start an existing container first c = self.d_client.containers.get(container) c.start() s_containers.append(container) manifest.set_option(section, 'running', 'yes') self.logger.info('started ' + str(container) + ' with ID: ' + str(c.short_id)) except Exception as err: s_containers, f_containers = self._run_container( container, tool_d, section, s_containers, f_containers) # save changes made to manifest manifest.write_config() return s_containers, f_containers def _run_container(self, container, tool_d, section, s_containers, f_containers): manifest = Template(self.manifest) try: gpu = 'gpu.enabled' failed = False if (gpu in tool_d[container]['labels'] and tool_d[container]['labels'][gpu] == 'yes'): vent_config = Template(template=self.path_dirs.cfg_file) port = '' host = '' result = vent_config.option('nvidia-docker-plugin', 'port') if result[0]: port = result[1] else: port = '3476' result = vent_config.option('nvidia-docker-plugin', 'host') if result[0]: host = result[1] else: # now just requires ip, ifconfig try: route = check_output(('ip', 'route')).decode( 'utf-8').split('\n') default = '' # grab the default network device. for device in route: if 'default' in device: default = device.split()[4] break # grab the IP address for the default device ip_addr = check_output( ('ifconfig', default)).decode('utf-8') ip_addr = ip_addr.split('\n')[1].split()[1] host = ip_addr except Exception as e: # pragma no cover self.logger.error('failed to grab ip. Ensure that \ ip and ifconfig are installed') nd_url = 'http://' + host + ':' + port + '/v1.0/docker/cli' params = {'vol': 'nvidia_driver'} r = requests.get(nd_url, params=params) if r.status_code == 200: options = r.text.split() for option in options: if option.startswith('--volume-driver='): tool_d[container]['volume_driver'] = option.split('=', 1)[ 1] elif option.startswith('--volume='): vol = option.split('=', 1)[1].split(':') if 'volumes' in tool_d[container]: if isinstance(tool_d[container]['volumes'], list): if len(vol) == 2: c_vol = vol[0] + \ ':' + vol[1] + ':rw' else: c_vol = vol[0] + ':' + \ vol[1] + ':' + vol[2] tool_d[container]['volumes'].append( c_vol) else: # Dictionary tool_d[container]['volumes'][vol[0]] = {'bind': vol[1], 'mode': vol[2]} else: tool_d[container]['volumes'] = {vol[0]: {'bind': vol[1], 'mode': vol[2]}} elif option.startswith('--device='): dev = option.split('=', 1)[1] if 'devices' in tool_d[container]: tool_d[container]['devices'].append(dev + ':' + dev + ':rwm') else: tool_d[container]['devices'] = [ dev + ':' + dev + ':rwm'] else: self.logger.error('Unable to parse ' + 'nvidia-docker option: ' + str(option)) else: failed = True f_containers.append(container) manifest.set_option(section, 'running', 'failed') self.logger.error('failed to start ' + str(container) + ' because nvidia-docker-plugin ' + 'failed with: ' + str(r.status_code)) if not failed: try: self.d_client.containers.remove(container, force=True) self.logger.info( 'removed old existing container: ' + str(container)) except Exception as e: pass cont_id = self.d_client.containers.run(detach=True, **tool_d[container]) s_containers.append(container) manifest.set_option(section, 'running', 'yes') self.logger.info('started ' + str(container) + ' with ID: ' + str(cont_id)) except Exception as e: # pragma: no cover f_containers.append(container) manifest.set_option(section, 'running', 'failed') self.logger.error('failed to start ' + str(container) + ' because: ' + str(e)) return s_containers, f_containers def stop(self, repo, name): args = locals() status = (True, None) try: # !! TODO need to account for plugin containers that have random # names, use labels perhaps options = ['name', 'namespace', 'built', 'groups', 'path', 'image_name', 'branch', 'version'] s, _ = Template(template=self.manifest).constrain_opts(args, options) for section in s: container_name = s[section]['image_name'].replace(':', '-') container_name = container_name.replace('/', '-') try: container = self.d_client.containers.get(container_name) container.stop() self.logger.info('Stopped {0}'.format(str(container_name))) except Exception as e: # pragma: no cover self.logger.error('Failed to stop ' + str(container_name) + ' because: ' + str(e)) except Exception as e: # pragma: no cover self.logger.error('Stop failed with error: ' + str(e)) status = (False, e) return status def repo_commits(self, repo): """ Get the commit IDs for all of the branches of a repository """ commits = [] try: status = self.path_dirs.apply_path(repo) # switch to directory where repo will be cloned to if status[0]: cwd = status[1] else: self.logger.error('apply_path failed. Exiting repo_commits with' ' status: ' + str(status)) return status status = self.repo_branches(repo) if status[0]: branches = status[1] for branch in branches: try: branch_output = check_output(shlex .split('git rev-list origin/' + branch), stderr=STDOUT, close_fds=True).decode('utf-8') branch_output = branch_output.split('\n')[:-1] branch_output += ['HEAD'] commits.append((branch, branch_output)) except Exception as e: # pragma: no cover self.logger.error('repo_commits failed with error: ' + str(e) + ' on branch: ' + str(branch)) status = (False, e) return status else: self.logger.error('repo_branches failed. Exiting repo_commits' ' with status: ' + str(status)) return status chdir(cwd) status = (True, commits) except Exception as e: # pragma: no cover self.logger.error('repo_commits failed with error: ' + str(e)) status = (False, e) return status def repo_branches(self, repo): """ Get the branches of a repository """ branches = [] try: # switch to directory where repo will be cloned to status = self.path_dirs.apply_path(repo) if status[0]: cwd = status[1] else: self.logger.error('apply_path failed. Exiting repo_branches' ' with status ' + str(status)) return status branch_output = check_output(shlex.split('git branch -a'), stderr=STDOUT, close_fds=True) branch_output = branch_output.split(b'\n') for branch in branch_output: br = branch.strip() if br.startswith(b'*'): br = br[2:] if b'/' in br: branches.append(br.rsplit(b'/', 1)[1].decode('utf-8')) elif br: branches.append(br.decode('utf-8')) branches = list(set(branches)) for branch in branches: try: check_output(shlex.split('git checkout ' + branch), stderr=STDOUT, close_fds=True) except Exception as e: # pragma: no cover self.logger.error('repo_branches failed with error: ' + str(e) + ' on branch: ' + str(branch)) status = (False, e) return status chdir(cwd) status = (True, branches) except Exception as e: # pragma: no cover self.logger.error('repo_branches failed with error: ' + str(e)) status = (False, e) return status def repo_tools(self, repo, branch, version): """ Get available tools for a repository branch at a version """ try: tools = [] status = self.path_dirs.apply_path(repo) # switch to directory where repo will be cloned to if status[0]: cwd = status[1] else: self.logger.error('apply_path failed. Exiting repo_tools with' ' status: ' + str(status)) return status # TODO commenting out for now, should use update_repo #status = self.p_helper.checkout(branch=branch, version=version) status = (True, None) if status[0]: path, _, _ = self.path_dirs.get_path(repo) tools = AvailableTools(path, version=version) else: self.logger.error('checkout failed. Exiting repo_tools with' ' status: ' + str(status)) return status chdir(cwd) status = (True, tools) except Exception as e: # pragma: no cover self.logger.error('repo_tools failed with error: ' + str(e)) status = (False, e) return status
class Tools: def __init__(self, version='HEAD', branch='master', user=None, pw=None, *args, **kwargs): self.version = version self.branch = branch self.user = user self.pw = pw self.d_client = docker.from_env() self.path_dirs = PathDirs(**kwargs) self.path_dirs.host_config() self.manifest = join(self.path_dirs.meta_dir, 'plugin_manifest.cfg') self.logger = Logger(__name__) def new(self, tool_type, uri, tools=None, link_name=None, image_name=None, overrides=None, tag=None, registry=None, groups=None): try: # remove tools that are already installed from being added if isinstance(tools, list): i = len(tools) - 1 while i >= 0: tool = tools[i] if tool[0].find('@') >= 0: tool_name = tool[0].split('@')[-1] else: tool_name = tool[0].rsplit('/', 1)[-1] constraints = { 'name': tool_name, 'repo': uri.split('.git')[0] } prev_installed, _ = Template( template=self.manifest).constrain_opts( constraints, []) # don't reinstall if prev_installed: tools.remove(tool) i -= 1 if len(tools) == 0: tools = None except Exception as e: # pragma: no cover self.logger.error('Add failed with error: {0}'.format(str(e))) return (False, str(e)) if tool_type == 'image': status = Image(self.manifest).add(uri, link_name, tag=tag, registry=registry, groups=groups) else: if tool_type == 'core': uri = 'https://github.com/cyberreboot/vent' core = True else: core = False status = Repository(self.manifest).add(uri, tools, overrides=overrides, version=self.version, image_name=image_name, branch=self.branch, user=self.user, pw=self.pw, core=core) return status def configure(self, tool): # TODO return def inventory(self, choices=None): """ Return a dictionary of the inventory items and status """ status = (True, None) if not choices: return (False, 'No choices made') try: # choices: repos, tools, images, built, running, enabled items = { 'repos': [], 'tools': {}, 'images': {}, 'built': {}, 'running': {}, 'enabled': {} } tools = Template(self.manifest).list_tools() for choice in choices: for tool in tools: try: if choice == 'repos': if 'repo' in tool: if (tool['repo'] and tool['repo'] not in items[choice]): items[choice].append(tool['repo']) elif choice == 'tools': items[choice][tool['section']] = tool['name'] elif choice == 'images': # TODO also check against docker items[choice][tool['section']] = tool['image_name'] elif choice == 'built': items[choice][tool['section']] = tool['built'] elif choice == 'running': containers = Containers() status = 'not running' for container in containers: image_name = tool['image_name'] \ .rsplit(':' + tool['version'], 1)[0] image_name = image_name.replace(':', '-') image_name = image_name.replace('/', '-') self.logger.info('image_name: ' + image_name) if container[0] == image_name: status = container[1] elif container[0] == image_name + \ '-' + tool['version']: status = container[1] items[choice][tool['section']] = status elif choice == 'enabled': items[choice][tool['section']] = tool['enabled'] else: # unknown choice pass except Exception as e: # pragma: no cover self.logger.error('Unable to grab info about tool: ' + str(tool) + ' because: ' + str(e)) status = (True, items) except Exception as e: # pragma: no cover self.logger.error('Inventory failed with error: {0}'.format( str(e))) status = (False, str(e)) return status def remove(self, repo, name): args = locals() status = (True, None) # get resulting dict of sections with options that match constraints template = Template(template=self.manifest) results, _ = template.constrain_opts(args, []) for result in results: response, image_name = template.option(result, 'image_name') name = template.option(result, 'name')[1] try: settings_dict = json.loads( template.option(result, 'settings')[1]) instances = int(settings_dict['instances']) except Exception: instances = 1 try: # check for container and remove c_name = image_name.replace(':', '-').replace('/', '-') for i in range(1, instances + 1): container_name = c_name + str(i) if i != 1 else c_name container = self.d_client.containers.get(container_name) response = container.remove(v=True, force=True) self.logger.info( 'Removing container: {0}'.format(container_name)) except Exception as e: # pragma: no cover self.logger.warning('Unable to remove the container: ' + container_name + ' because: ' + str(e)) # check for image and remove try: response = None image_id = template.option(result, 'image_id')[1] response = self.d_client.images.remove(image_id, force=True) self.logger.info('Removing image: ' + image_name) except Exception as e: # pragma: no cover self.logger.warning('Unable to remove the image: ' + image_name + ' because: ' + str(e)) # remove tool from the manifest for i in range(1, instances + 1): res = result.rsplit(':', 2) res[0] += str(i) if i != 1 else '' res = ':'.join(res) if template.section(res)[0]: status = template.del_section(res) self.logger.info('Removing tool: ' + res) # TODO if all tools from a repo have been removed, remove the repo template.write_config() return status def start(self, repo, name, is_tool_d=False): if is_tool_d: tool_d = repo else: args = locals() del args['self'] del args['is_tool_d'] tool_d = {} tool_d.update(self._prep_start(**args)[1]) status = (True, None) try: # check start priorities (priority of groups alphabetical for now) group_orders = {} groups = [] containers_remaining = [] username = getpass.getuser() # remove tools that have the hidden label tool_d_copy = copy.deepcopy(tool_d) for container in tool_d_copy: if 'labels' in tool_d_copy[ container] and 'vent.groups' in tool_d_copy[container][ 'labels']: groups_copy = tool_d_copy[container]['labels'][ 'vent.groups'].split(',') if 'hidden' in groups_copy: del tool_d[container] for container in tool_d: containers_remaining.append(container) self.logger.info("User: '******' starting container: {1}".format( username, container)) if 'labels' in tool_d[container]: if 'vent.groups' in tool_d[container]['labels']: groups += tool_d[container]['labels'][ 'vent.groups'].split(',') if 'vent.priority' in tool_d[container]['labels']: priorities = tool_d[container]['labels'][ 'vent.priority'].split(',') container_groups = tool_d[container]['labels'][ 'vent.groups'].split(',') for i, priority in enumerate(priorities): if container_groups[i] not in group_orders: group_orders[container_groups[i]] = [] group_orders[container_groups[i]].append( (int(priority), container)) containers_remaining.remove(container) tool_d[container]['labels'].update( {'started-by': username}) else: tool_d[container].update( {'labels': { 'started-by': username }}) # start containers based on priorities p_results = self._start_priority_containers( groups, group_orders, tool_d) # start the rest of the containers that didn't have any priorities r_results = self._start_remaining_containers( containers_remaining, tool_d) results = (p_results[0] + r_results[0], p_results[1] + r_results[1]) if len(results[1]) > 0: status = (False, results) else: status = (True, results) except Exception as e: # pragma: no cover self.logger.error('Start failed with error: {0}'.format(str(e))) status = (False, str(e)) return status def _prep_start(self, repo, name): args = locals() status = (True, None) try: options = [ 'name', 'namespace', 'built', 'groups', 'path', 'image_name', 'branch', 'repo', 'type', 'version' ] vent_config = Template(template=self.path_dirs.cfg_file) manifest = Template(self.manifest) files = vent_config.option('main', 'files') files = (files[0], expanduser(files[1])) s, _ = manifest.constrain_opts(args, options) status, tool_d = self._start_sections(s, files) # look out for links to delete because they're defined externally links_to_delete = set() # get instances for each tool tool_instances = {} sections = manifest.sections()[1] for section in sections: settings = manifest.option(section, 'settings') if settings[0]: settings = json.loads(settings[1]) if 'instances' in settings: l_name = manifest.option(section, 'link_name') if l_name[0]: tool_instances[l_name[1]] = int( settings['instances']) # check and update links, volumes_from, network_mode for container in list(tool_d.keys()): if 'labels' not in tool_d[ container] or 'vent.groups' not in tool_d[container][ 'labels'] or 'core' not in tool_d[container][ 'labels']['vent.groups']: tool_d[container]['remove'] = True if 'links' in tool_d[container]: for link in list(tool_d[container]['links'].keys()): # add links to external services already running if # necessary, by default configure local services too configure_local = True ext = 'external-services' if link in vent_config.options(ext)[1]: try: lconf = json.loads( vent_config.option(ext, link)[1]) if ('locally_active' not in lconf or lconf['locally_active'] == 'no'): ip_adr = lconf['ip_address'] port = lconf['port'] tool_d[container]['extra_hosts'] = {} # containers use lowercase names for # connections tool_d[container]['extra_hosts'][ link.lower()] = ip_adr # create an environment variable for container # to access port later env_variable = link.upper() + \ '_CUSTOM_PORT=' + port if 'environment' not in tool_d[container]: tool_d[container]['environment'] = [] tool_d[container]['environment'].append( env_variable) # remove the entry from links because no # longer connecting to local container links_to_delete.add(link) configure_local = False except Exception as e: # pragma: no cover self.logger.error( 'Could not load external settings because: {0}' .format(str(e))) configure_local = True status = False if configure_local: for c in list(tool_d.keys()): if ('tmp_name' in tool_d[c] and tool_d[c]['tmp_name'] == link): tool_d[container]['links'][ tool_d[c]['name']] = tool_d[container][ 'links'].pop(link) if link in tool_instances and tool_instances[ link] > 1: for i in range( 2, tool_instances[link] + 1): tool_d[container]['links'][ tool_d[c]['name'] + str(i)] = tool_d[container][ 'links'][tool_d[c] ['name']] + str(i) if 'volumes_from' in tool_d[container]: tmp_volumes_from = tool_d[container]['volumes_from'] tool_d[container]['volumes_from'] = [] for volumes_from in list(tmp_volumes_from): for c in list(tool_d.keys()): if ('tmp_name' in tool_d[c] and tool_d[c]['tmp_name'] == volumes_from): tool_d[container]['volumes_from'].append( tool_d[c]['name']) tmp_volumes_from.remove(volumes_from) tool_d[container]['volumes_from'] += tmp_volumes_from if 'network_mode' in tool_d[container]: if tool_d[container]['network_mode'].startswith( 'container:'): network_c_name = tool_d[container][ 'network_mode'].split('container:')[1] for c in list(tool_d.keys()): if ('tmp_name' in tool_d[c] and tool_d[c]['tmp_name'] == network_c_name): tool_d[container]['network_mode'] = 'container:' + \ tool_d[c]['name'] # remove tmp_names for c in list(tool_d.keys()): if 'tmp_name' in tool_d[c]: del tool_d[c]['tmp_name'] # remove links section if all were externally configured for c in list(tool_d.keys()): if 'links' in tool_d[c]: for link in links_to_delete: if link in tool_d[c]['links']: del tool_d[c]['links'][link] # delete links if no more defined if not tool_d[c]['links']: del tool_d[c]['links'] # remove containers that shouldn't be started for c in list(tool_d.keys()): deleted = False if 'start' in tool_d[c] and not tool_d[c]['start']: del tool_d[c] deleted = True if not deleted: # look for tools services that are being done externally # tools are capitalized in vent.cfg, so make them lowercase # for comparison ext = 'external-services' external_tools = vent_config.section(ext)[1] name = tool_d[c]['labels']['vent.name'] for tool in external_tools: if name == tool[0].lower(): try: tool_config = json.loads(tool[1]) if ('locally_active' in tool_config and tool_config['locally_active'] == 'no'): del tool_d[c] except Exception as e: # pragma: no cover self.logger.warning( 'Locally running container ' + name + ' may be redundant') if status: status = (True, tool_d) else: status = (False, tool_d) except Exception as e: # pragma: no cover self.logger.error('_prep_start failed with error: ' + str(e)) status = (False, e) return status def _start_sections(self, s, files): tool_d = {} status = (True, None) for section in s: # initialize needed vars c_name = s[section]['image_name'].replace(':', '-') c_name = c_name.replace('/', '-') instance_num = re.search(r'\d+$', s[section]['name']) if instance_num: c_name += instance_num.group() image_name = s[section]['image_name'] # checkout the right version and branch of the repo tool_d[c_name] = {'image': image_name, 'name': c_name} # get rid of all commented sections in various runtime # configurations manifest = Template(self.manifest) overall_dict = {} for setting in ['info', 'docker', 'gpu', 'settings', 'service']: option = manifest.option(section, setting) if option[0]: overall_dict[setting] = {} settings_dict = json.loads(option[1]) for opt in settings_dict: if not opt.startswith('#'): overall_dict[setting][opt] = settings_dict[opt] if 'docker' in overall_dict: options_dict = overall_dict['docker'] for option in options_dict: options = options_dict[option] # check for commands to evaluate if '`' in options: cmds = options.split('`') if len(cmds) > 2: i = 1 while i < len(cmds): try: cmds[i] = check_output( shlex.split(cmds[i]), stderr=STDOUT, close_fds=True).strip().decode('utf-8') except Exception as e: # pragma: no cover self.logger.error( 'unable to evaluate command specified in vent.template: ' + str(e)) i += 2 options = ''.join(cmds) # check for commands to evaluate # store options set for docker try: tool_d[c_name][option] = ast.literal_eval(options) except Exception as e: # pragma: no cover self.logger.debug('Unable to literal_eval: {0}'.format( str(options))) tool_d[c_name][option] = options if 'labels' not in tool_d[c_name]: tool_d[c_name]['labels'] = {} # get the service uri info if 'service' in overall_dict: try: options_dict = overall_dict['service'] for option in options_dict: tool_d[c_name]['labels'][option] = options_dict[option] except Exception as e: # pragma: no cover self.logger.error('unable to store service options for ' 'docker: ' + str(e)) # check for gpu settings if 'gpu' in overall_dict: try: options_dict = json.loads(status[1]) for option in options_dict: tool_d[c_name]['labels']['gpu.' + option] = options_dict[option] except Exception as e: # pragma: no cover self.logger.error('unable to store gpu options for ' 'docker: ' + str(e)) # get temporary name for links, etc. plugin_c = Template(template=self.manifest) status, plugin_sections = plugin_c.sections() for plugin_section in plugin_sections: status = plugin_c.option(plugin_section, 'link_name') image_status = plugin_c.option(plugin_section, 'image_name') if status[0] and image_status[0]: cont_name = image_status[1].replace(':', '-') cont_name = cont_name.replace('/', '-') if cont_name not in tool_d: tool_d[cont_name] = { 'image': image_status[1], 'name': cont_name, 'start': False } tool_d[cont_name]['tmp_name'] = status[1] # add extra labels tool_d[c_name]['labels']['vent'] = Version() tool_d[c_name]['labels']['vent.namespace'] = s[section][ 'namespace'] tool_d[c_name]['labels']['vent.branch'] = s[section]['branch'] tool_d[c_name]['labels']['vent.version'] = s[section]['version'] tool_d[c_name]['labels']['vent.name'] = s[section]['name'] tool_d[c_name]['labels']['vent.section'] = section tool_d[c_name]['labels']['vent.repo'] = s[section]['repo'] tool_d[c_name]['labels']['vent.type'] = s[section]['type'] # check for log_config settings in external-services externally_configured = False vent_config = Template(self.path_dirs.cfg_file) for ext_tool in vent_config.section('external-services')[1]: if ext_tool[0].lower() == 'syslog': try: log_dict = json.loads(ext_tool[1]) # configure if not locally active if ('locally_active' not in log_dict or log_dict['locally_active'] == 'no'): del log_dict['locally_active'] log_config = {} log_config['type'] = 'syslog' log_config['config'] = {} ip_address = '' port = '' for option in log_dict: if option == 'ip_address': ip_address = log_dict[option] elif option == 'port': port = log_dict['port'] syslog_address = 'tcp://' + ip_address + ':' + port syslog_config = { 'syslog-address': syslog_address, 'syslog-facility': 'daemon', 'tag': '{{.Name}}' } log_config['config'].update(syslog_config) externally_configured = True except Exception as e: # pragma: no cover self.logger.error('external settings for log_config' " couldn't be stored because: " + str(e)) externally_configured = False if not externally_configured: log_config = { 'type': 'syslog', 'config': { 'syslog-address': 'tcp://0.0.0.0:514', 'syslog-facility': 'daemon', 'tag': '{{.Name}}' } } if 'groups' in s[section]: # add labels for groups tool_d[c_name]['labels']['vent.groups'] = s[section]['groups'] # add restart=always to core containers if 'core' in s[section]['groups']: tool_d[c_name]['restart_policy'] = {'Name': 'always'} # map network names to environment variables if 'network' in s[section]['groups']: vent_config = Template(template=self.path_dirs.cfg_file) nic_mappings = vent_config.section('network-mapping') nics = '' if nic_mappings[0]: for nic in nic_mappings[1]: nics += nic[0] + ':' + nic[1] + ',' nics = nics[:-1] if nics: if 'environment' in tool_d[c_name]: tool_d[c_name]['environment'].append('VENT_NICS=' + nics) else: tool_d[c_name]['environment'] = [ 'VENT_NICS=' + nics ] # send logs to syslog if ('syslog' not in s[section]['groups'] and 'core' in s[section]['groups']): log_config['config']['tag'] = '{{.Name}}' tool_d[c_name]['log_config'] = log_config if 'syslog' not in s[section]['groups']: tool_d[c_name]['log_config'] = log_config # mount necessary directories if 'files' in s[section]['groups']: ulimits = [] ulimits.append( docker.types.Ulimit(name='nofile', soft=1048576, hard=1048576)) tool_d[c_name]['ulimits'] = ulimits # check if running in a docker container if 'VENT_CONTAINERIZED' in environ and environ[ 'VENT_CONTAINERIZED'] == 'true': if 'volumes_from' in tool_d[c_name]: tool_d[c_name]['volumes_from'].append( environ['HOSTNAME']) else: tool_d[c_name]['volumes_from'] = [ environ['HOSTNAME'] ] else: if 'volumes' in tool_d[c_name]: tool_d[c_name]['volumes'][ self.path_dirs.base_dir[:-1]] = { 'bind': '/vent', 'mode': 'ro' } else: tool_d[c_name]['volumes'] = { self.path_dirs.base_dir[:-1]: { 'bind': '/vent', 'mode': 'ro' } } if files[0]: if 'volumes' in tool_d[c_name]: tool_d[c_name]['volumes'][files[1]] = { 'bind': '/files', 'mode': 'rw' } else: tool_d[c_name]['volumes'] = { files[1]: { 'bind': '/files', 'mode': 'rw' } } else: tool_d[c_name]['log_config'] = log_config # add label for priority if 'settings' in overall_dict: try: options_dict = overall_dict['settings'] for option in options_dict: if option == 'priority': tool_d[c_name]['labels'][ 'vent.priority'] = options_dict[option] except Exception as e: # pragma: no cover self.logger.error('unable to store settings options ' 'for docker ' + str(e)) # only start tools that have been built if s[section]['built'] != 'yes': del tool_d[c_name] # store section information for adding info to manifest later else: tool_d[c_name]['section'] = section return status, tool_d def _start_priority_containers(self, groups, group_orders, tool_d): """ Select containers based on priorities to start """ vent_cfg = Template(self.path_dirs.cfg_file) cfg_groups = vent_cfg.option('groups', 'start_order') if cfg_groups[0]: cfg_groups = cfg_groups[1].split(',') else: cfg_groups = [] all_groups = sorted(set(groups)) s_conts = [] f_conts = [] # start tools in order of group defined in vent.cfg for group in cfg_groups: # remove from all_groups because already checked out if group in all_groups: all_groups.remove(group) if group in group_orders: for cont_t in sorted(group_orders[group]): if cont_t[1] not in s_conts: s_conts, f_conts = self._start_container( cont_t[1], tool_d, s_conts, f_conts) # start tools that haven't been specified in the vent.cfg, if any for group in all_groups: if group in group_orders: for cont_t in sorted(group_orders[group]): if cont_t[1] not in s_conts: s_conts, f_conts = self._start_container( cont_t[1], tool_d, s_conts, f_conts) return (s_conts, f_conts) def _start_remaining_containers(self, containers_remaining, tool_d): """ Select remaining containers that didn't have priorities to start """ s_containers = [] f_containers = [] for container in containers_remaining: s_containers, f_containers = self._start_container( container, tool_d, s_containers, f_containers) return (s_containers, f_containers) def _start_container(self, container, tool_d, s_containers, f_containers): """ Start container that was passed in and return status """ # use section to add info to manifest section = tool_d[container]['section'] del tool_d[container]['section'] manifest = Template(self.manifest) try: # try to start an existing container first c = self.d_client.containers.get(container) c.start() s_containers.append(container) manifest.set_option(section, 'running', 'yes') self.logger.info('started ' + str(container) + ' with ID: ' + str(c.short_id)) except Exception as err: s_containers, f_containers = self._run_container( container, tool_d, section, s_containers, f_containers) # save changes made to manifest manifest.write_config() return s_containers, f_containers def _run_container(self, container, tool_d, section, s_containers, f_containers): manifest = Template(self.manifest) try: gpu = 'gpu.enabled' failed = False if (gpu in tool_d[container]['labels'] and tool_d[container]['labels'][gpu] == 'yes'): vent_config = Template(template=self.path_dirs.cfg_file) port = '' host = '' result = vent_config.option('nvidia-docker-plugin', 'port') if result[0]: port = result[1] else: port = '3476' result = vent_config.option('nvidia-docker-plugin', 'host') if result[0]: host = result[1] else: # now just requires ip, ifconfig try: route = check_output( ('ip', 'route')).decode('utf-8').split('\n') default = '' # grab the default network device. for device in route: if 'default' in device: default = device.split()[4] break # grab the IP address for the default device ip_addr = check_output( ('ifconfig', default)).decode('utf-8') ip_addr = ip_addr.split('\n')[1].split()[1] host = ip_addr except Exception as e: # pragma no cover self.logger.error('failed to grab ip. Ensure that \ ip and ifconfig are installed') nd_url = 'http://' + host + ':' + port + '/v1.0/docker/cli' params = {'vol': 'nvidia_driver'} r = requests.get(nd_url, params=params) if r.status_code == 200: options = r.text.split() for option in options: if option.startswith('--volume-driver='): tool_d[container]['volume_driver'] = option.split( '=', 1)[1] elif option.startswith('--volume='): vol = option.split('=', 1)[1].split(':') if 'volumes' in tool_d[container]: if isinstance(tool_d[container]['volumes'], list): if len(vol) == 2: c_vol = vol[0] + \ ':' + vol[1] + ':rw' else: c_vol = vol[0] + ':' + \ vol[1] + ':' + vol[2] tool_d[container]['volumes'].append(c_vol) else: # Dictionary tool_d[container]['volumes'][vol[0]] = { 'bind': vol[1], 'mode': vol[2] } else: tool_d[container]['volumes'] = { vol[0]: { 'bind': vol[1], 'mode': vol[2] } } elif option.startswith('--device='): dev = option.split('=', 1)[1] if 'devices' in tool_d[container]: tool_d[container]['devices'].append(dev + ':' + dev + ':rwm') else: tool_d[container]['devices'] = [ dev + ':' + dev + ':rwm' ] else: self.logger.error('Unable to parse ' + 'nvidia-docker option: ' + str(option)) else: failed = True f_containers.append(container) manifest.set_option(section, 'running', 'failed') self.logger.error('failed to start ' + str(container) + ' because nvidia-docker-plugin ' + 'failed with: ' + str(r.status_code)) if not failed: try: self.d_client.containers.remove(container, force=True) self.logger.info('removed old existing container: ' + str(container)) except Exception as e: pass cont_id = self.d_client.containers.run(detach=True, **tool_d[container]) s_containers.append(container) manifest.set_option(section, 'running', 'yes') self.logger.info('started ' + str(container) + ' with ID: ' + str(cont_id)) except Exception as e: # pragma: no cover f_containers.append(container) manifest.set_option(section, 'running', 'failed') self.logger.error('failed to start ' + str(container) + ' because: ' + str(e)) return s_containers, f_containers def stop(self, repo, name): args = locals() status = (True, None) try: # !! TODO need to account for plugin containers that have random # names, use labels perhaps options = [ 'name', 'namespace', 'built', 'groups', 'path', 'image_name', 'branch', 'version' ] s, _ = Template(template=self.manifest).constrain_opts( args, options) for section in s: container_name = s[section]['image_name'].replace(':', '-') container_name = container_name.replace('/', '-') try: container = self.d_client.containers.get(container_name) container.stop() self.logger.info('Stopped {0}'.format(str(container_name))) except Exception as e: # pragma: no cover self.logger.error('Failed to stop ' + str(container_name) + ' because: ' + str(e)) except Exception as e: # pragma: no cover self.logger.error('Stop failed with error: ' + str(e)) status = (False, e) return status def repo_commits(self, repo): """ Get the commit IDs for all of the branches of a repository """ commits = [] try: status = self.path_dirs.apply_path(repo) # switch to directory where repo will be cloned to if status[0]: cwd = status[1] else: self.logger.error( 'apply_path failed. Exiting repo_commits with' ' status: ' + str(status)) return status status = self.repo_branches(repo) if status[0]: branches = status[1] for branch in branches: try: branch_output = check_output( shlex.split('git rev-list origin/' + branch), stderr=STDOUT, close_fds=True).decode('utf-8') branch_output = branch_output.split('\n')[:-1] branch_output += ['HEAD'] commits.append((branch, branch_output)) except Exception as e: # pragma: no cover self.logger.error('repo_commits failed with error: ' + str(e) + ' on branch: ' + str(branch)) status = (False, e) return status else: self.logger.error('repo_branches failed. Exiting repo_commits' ' with status: ' + str(status)) return status chdir(cwd) status = (True, commits) except Exception as e: # pragma: no cover self.logger.error('repo_commits failed with error: ' + str(e)) status = (False, e) return status def repo_branches(self, repo): """ Get the branches of a repository """ branches = [] try: # switch to directory where repo will be cloned to status = self.path_dirs.apply_path(repo) if status[0]: cwd = status[1] else: self.logger.error('apply_path failed. Exiting repo_branches' ' with status ' + str(status)) return status branch_output = check_output(shlex.split('git branch -a'), stderr=STDOUT, close_fds=True) branch_output = branch_output.split(b'\n') for branch in branch_output: br = branch.strip() if br.startswith(b'*'): br = br[2:] if b'/' in br: branches.append(br.rsplit(b'/', 1)[1].decode('utf-8')) elif br: branches.append(br.decode('utf-8')) branches = list(set(branches)) for branch in branches: try: check_output(shlex.split('git checkout ' + branch), stderr=STDOUT, close_fds=True) except Exception as e: # pragma: no cover self.logger.error('repo_branches failed with error: ' + str(e) + ' on branch: ' + str(branch)) status = (False, e) return status chdir(cwd) status = (True, branches) except Exception as e: # pragma: no cover self.logger.error('repo_branches failed with error: ' + str(e)) status = (False, e) return status def repo_tools(self, repo, branch, version): """ Get available tools for a repository branch at a version """ try: tools = [] status = self.path_dirs.apply_path(repo) # switch to directory where repo will be cloned to if status[0]: cwd = status[1] else: self.logger.error('apply_path failed. Exiting repo_tools with' ' status: ' + str(status)) return status # TODO commenting out for now, should use update_repo #status = self.p_helper.checkout(branch=branch, version=version) status = (True, None) if status[0]: path, _, _ = self.path_dirs.get_path(repo) tools = AvailableTools(path, version=version) else: self.logger.error('checkout failed. Exiting repo_tools with' ' status: ' + str(status)) return status chdir(cwd) status = (True, tools) except Exception as e: # pragma: no cover self.logger.error('repo_tools failed with error: ' + str(e)) status = (False, e) return status
class Repository: def __init__(self, manifest, *args, **kwargs): self.path_dirs = PathDirs(**kwargs) self.manifest = manifest self.d_client = docker.from_env() self.logger = Logger(__name__) def add(self, repo, tools=None, overrides=None, version='HEAD', core=False, image_name=None, branch='master', build=True, user=None, pw=None): status = (True, None) self.repo = repo.lower() self.tools = tools self.overrides = overrides self.branch = branch self.version = version self.image_name = image_name self.core = core status = self._clone(user=user, pw=pw) if status[0] and build: status = self._build() return status def _build(self): status = (True, None) status = self._get_tools() matches = status[1] status = self.path_dirs.apply_path(self.repo) original_path = status[1] if status[0] and len(matches) > 0: repo, org, name = self.path_dirs.get_path(self.repo) cmd = 'git rev-parse --short ' + self.version commit_id = '' try: commit_id = check_output( shlex.split(cmd), stderr=STDOUT, close_fds=True).strip().decode('utf-8') except Exception as e: # pragma: no cover self.logger.error( 'Unable to get commit ID because: {0}'.format(str(e))) template = Template(template=self.manifest) for match in matches: status, template, match_path, image_name, section = self._build_manifest( match, template, repo, org, name, commit_id) if not status[0]: break status, template = self._build_image(template, match_path, image_name, section) if not status[0]: break if status[0]: # write out configuration to the manifest file template.write_config() chdir(original_path) return status def _get_tools(self): status = (True, None) matches = [] path, _, _ = self.path_dirs.get_path(self.repo) status = Checkout(path, branch=self.branch, version=self.version) if status[0]: if self.tools is None and self.overrides is None: # get all tools matches = AvailableTools(path, branch=self.branch, version=self.version, core=self.core) elif self.tools is None: # there's only something in overrides # grab all the tools then apply overrides matches = AvailableTools(path, branch=self.branch, version=self.version, core=self.core) # !! TODO apply overrides to matches elif self.overrides is None: # there's only something in tools # only grab the tools specified matches = ToolMatches(tools=self.tools, version=self.version) else: # both tools and overrides were specified # grab only the tools specified, with the overrides applied o_matches = ToolMatches(tools=self.tools, version=self.version) matches = o_matches for override in self.overrides: override_t = None if override[0] == '.': override_t = ('', override[1]) else: override_t = override for match in o_matches: if override_t[0] == match[0]: matches.remove(match) matches.append(override_t) status = (True, matches) return status def _build_manifest(self, match, template, repo, org, name, commit_id): status = (True, None) # keep track of whether or not to write an additional manifest # entry for multiple instances, and how many additional entries # to write addtl_entries = 1 # remove @ in match for template setting purposes if match[0].find('@') >= 0: true_name = match[0].split('@')[1] else: true_name = match[0] # TODO check for special settings here first for the specific match self.version = match[1] section = org + ':' + name + ':' + true_name + ':' section += self.branch + ':' + self.version # need to get rid of temp identifiers for tools in same repo match_path = repo + match[0].split('@')[0] if self.image_name: image_name = self.image_name elif not self.core: image_name = org + '/' + name if match[0] != '': # if tool is in a subdir, add that to the name of the # image image_name += '-' + '-'.join(match[0].split('/')[1:]) image_name += ':' + self.branch else: image_name = ('cyberreboot/vent-' + match[0].split('/')[-1] + ':' + self.branch) image_name = image_name.replace('_', '-') # check if the section already exists is_there, options = template.section(section) previous_commit = None previous_commits = None head = False if is_there: for option in options: # TODO check if tool name but a different version # exists - then disable/remove if set if option[0] == 'version' and option[1] == 'HEAD': head = True if option[0] == 'built' and option[1] == 'yes': # !! TODO remove pre-existing image pass if option[0] == 'commit_id': previous_commit = option[1] if option[0] == 'previous_versions': previous_commits = option[1] # check if tool comes from multi directory multi_tool = 'no' if match[0].find('@') >= 0: multi_tool = 'yes' # !! TODO # check if section should be removed from config i.e. all tools # but new commit removed one that was in a previous commit image_name = image_name.lower() image_name = image_name.replace('@', '-') # special case for vent images if image_name.startswith('cyberreboot/vent'): image_name = image_name.replace('vent-vent-core-', 'vent-') image_name = image_name.replace('vent-vent-extras-', 'vent-') # set template section & options for tool at version and branch template.add_section(section) template.set_option(section, 'name', true_name.split('/')[-1]) template.set_option(section, 'namespace', org + '/' + name) template.set_option(section, 'path', match_path) template.set_option(section, 'repo', self.repo) template.set_option(section, 'multi_tool', multi_tool) template.set_option(section, 'branch', self.branch) template.set_option(section, 'version', self.version) template.set_option(section, 'last_updated', str(datetime.utcnow()) + ' UTC') template.set_option(section, 'image_name', image_name) template.set_option(section, 'type', 'repository') # save settings in vent.template to plugin_manifest # watch for multiple tools in same directory # just wanted to store match path with @ for path for use in # other actions tool_template = 'vent.template' if match[0].find('@') >= 0: tool_template = match[0].split('@')[1] + '.template' vent_template_path = join(match_path, tool_template) if exists(vent_template_path): with open(vent_template_path, 'r') as f: vent_template_val = f.read() else: vent_template_val = '' settings_dict = ParsedSections(vent_template_val) for setting in settings_dict: template.set_option(section, setting, json.dumps(settings_dict[setting])) # TODO do we need this if we save as a dictionary? vent_template = Template(vent_template_path) vent_status, response = vent_template.option('info', 'name') instances = vent_template.option('settings', 'instances') if instances[0]: addtl_entries = int(instances[1]) if vent_status: template.set_option(section, 'link_name', response) else: template.set_option(section, 'link_name', true_name.split('/')[-1]) if self.version == 'HEAD': template.set_option(section, 'commit_id', commit_id) if head: # no need to store previous commits if not HEAD, since # the version will always be the same commit ID if previous_commit and previous_commit != commit_id: if (previous_commits and previous_commit not in previous_commits): previous_commits = (previous_commit + ',' + previous_commits) elif not previous_commits: previous_commits = previous_commit if previous_commits and previous_commits != commit_id: template.set_option(section, 'previous_versions', previous_commits) groups = vent_template.option('info', 'groups') if groups[0]: template.set_option(section, 'groups', groups[1]) # set groups to empty string if no groups defined for tool else: template.set_option(section, 'groups', '') # write additional entries for multiple instances if addtl_entries > 1: # add 2 for naming conventions for i in range(2, addtl_entries + 1): addtl_section = section.rsplit(':', 2) addtl_section[0] += str(i) addtl_section = ':'.join(addtl_section) template.add_section(addtl_section) orig_vals = template.section(section)[1] for val in orig_vals: template.set_option(addtl_section, val[0], val[1]) template.set_option(addtl_section, 'name', true_name.split('/')[-1] + str(i)) return status, template, match_path, image_name, section def _build_image(self, template, match_path, image_name, section, build_local=False): status = (True, None) output = '' def set_instances(template, section, built, image_id=None): """ Set build information for multiple instances """ i = 2 while True: addtl_section = section.rsplit(':', 2) addtl_section[0] += str(i) addtl_section = ':'.join(addtl_section) if template.section(addtl_section)[0]: template.set_option(addtl_section, 'built', built) if image_id: template.set_option(addtl_section, 'image_id', image_id) template.set_option(addtl_section, 'last_updated', Timestamp()) else: break i += 1 # determine whether a tool should be considered a multi instance multi_instance = False try: settings = template.option(section, 'settings') if settings[0]: settings_dict = json.loads(settings[1]) if 'instances' in settings_dict and int( settings_dict['instances']) > 1: multi_instance = True except Exception as e: # pragma: no cover self.logger.error( 'Failed to check for multi instance because: {0}'.format( str(e))) status = (False, str(e)) cwd = getcwd() chdir(match_path) try: name = template.option(section, 'name') groups = template.option(section, 'groups') t_type = template.option(section, 'type') path = template.option(section, 'path') status, config_override = self.path_dirs.override_config(path[1]) if groups[1] == '' or not groups[0]: groups = (True, 'none') if not name[0]: name = (True, image_name) pull = False image_exists = False cfg_template = Template(template=self.path_dirs.cfg_file) use_existing_image = False result = cfg_template.option('build-options', 'use_existing_images') if result[0]: use_existing_image = result[1] if use_existing_image == 'yes' and not config_override: try: self.d_client.images.get(image_name) i_attrs = self.d_client.images.get(image_name).attrs image_id = i_attrs['Id'].split(':')[1][:12] template.set_option(section, 'built', 'yes') template.set_option(section, 'image_id', image_id) template.set_option(section, 'last_updated', str(datetime.utcnow()) + ' UTC') # set other instances too if multi_instance: set_instances(template, section, 'yes', image_id) status = (True, 'Found {0}'.format(image_name)) self.logger.info(str(status)) image_exists = True except docker.errors.ImageNotFound: image_exists = False except Exception as e: # pragma: no cover self.logger.warning( 'Failed to query Docker for images because: {0}'. format(str(e))) if not image_exists: # pull if '/' in image_name, fallback to build if '/' in image_name and not build_local and not config_override: try: image = self.d_client.images.pull(image_name) i_attrs = self.d_client.images.get(image_name).attrs image_id = i_attrs['Id'].split(':')[1][:12] if image_id: template.set_option(section, 'built', 'yes') template.set_option(section, 'image_id', image_id) template.set_option( section, 'last_updated', str(datetime.utcnow()) + ' UTC') # set other instances too if multi_instance: set_instances(template, section, 'yes', image_id) status = (True, 'Pulled {0}'.format(image_name)) self.logger.info(str(status)) else: template.set_option(section, 'built', 'failed') template.set_option( section, 'last_updated', str(datetime.utcnow()) + ' UTC') # set other instances too if multi_instance: set_instances(template, section, 'failed') status = (False, 'Failed to pull image {0}'.format( str(output.split('\n')[-1]))) self.logger.warning(str(status)) pull = True except Exception as e: # pragma: no cover self.logger.warning( 'Failed to pull image, going to build instead: {0}' .format(str(e))) status = (False, 'Failed to pull image because: {0}'.format( str(e))) if not pull and not image_exists: # get username to label built image with username = getpass.getuser() # see if additional file arg needed for building multiple # images from same directory file_tag = 'Dockerfile' multi_tool = template.option(section, 'multi_tool') if multi_tool[0] and multi_tool[1] == 'yes': specific_file = template.option(section, 'name')[1] if specific_file != 'unspecified': file_tag = 'Dockerfile.' + specific_file # update image name with new version for update image_name = image_name.rsplit(':', 1)[0] + ':' + self.branch labels = {} labels['vent'] = '' labels['vent.section'] = section labels['vent.repo'] = self.repo labels['vent.type'] = t_type[1] labels['vent.name'] = name[1] labels['vent.groups'] = groups[1] labels['built-by'] = username image = self.d_client.images.build(path='.', dockerfile=file_tag, tag=image_name, labels=labels, rm=True) image_id = image[0].id.split(':')[1][:12] template.set_option(section, 'built', 'yes') template.set_option(section, 'image_id', image_id) template.set_option(section, 'last_updated', str(datetime.utcnow()) + ' UTC') # set other instances too if multi_instance: set_instances(template, section, 'yes', image_id) status = (True, 'Built {0}'.format(image_name)) except Exception as e: # pragma: no cover self.logger.error( 'Unable to build image {0} because: {1} | {2}'.format( str(image_name), str(e), str(output))) template.set_option(section, 'built', 'failed') template.set_option(section, 'last_updated', str(datetime.utcnow()) + ' UTC') if multi_instance: set_instances(template, section, 'failed') status = (False, 'Failed to build image because: {0}'.format(str(e))) chdir(cwd) template.set_option(section, 'running', 'no') return status, template def _clone(self, user=None, pw=None): status = (True, None) try: # if path already exists, ignore try: path, _, _ = self.path_dirs.get_path(self.repo) chdir(path) return status except Exception as e: # pragma: no cover self.logger.debug("Repo doesn't exist, attempting to clone.") status = self.path_dirs.apply_path(self.repo) if not status[0]: self.logger.error('Unable to clone because: {0}'.format( str(status[1]))) return status repo = self.repo # check if user and pw were supplied, typically for private repos if user and pw: # only https is supported when using user/pw auth_repo = 'https://' + user + ':' + pw + '@' repo = auth_repo + repo.split('https://')[-1] # clone repo check_output(shlex.split( 'env GIT_SSL_NO_VERIFY=true git clone --recursive ' + repo + ' .'), stderr=STDOUT, close_fds=True).decode('utf-8') chdir(status[1]) status = (True, 'Successfully cloned: {0}'.format(self.repo)) except Exception as e: # pragma: no cover e_str = str(e) # scrub username and password from error message if e_str.find('@') >= 0: e_str = e_str[:e_str.find('//') + 2] + \ e_str[e_str.find('@') + 1:] self.logger.error('Clone failed with error: {0}'.format(e_str)) status = (False, e_str) return status def update(self, repo, tools=None): # TODO return