def host_config(self): """ Ensure the host configuration file exists """ default_file_dir = "/tmp/vent_files" config = Template(template=os.path.join(self.base_dir, "vent.cfg")) resp = config.section("main") if resp[0]: resp = config.option("main", "files") if not resp[0]: config.add_option("main", "files", default_file_dir) self.ensure_dir(default_file_dir) else: config.add_option("main", "files", default_file_dir) self.ensure_dir(default_file_dir) config.write_config() return
def _build_manifest(self, matches): """ Builds and writes the manifest for the tools being added """ # !! TODO check for pre-existing that conflict with request and # disable and/or remove image for match in matches: # keep track of whether or not to write an additional manifest # entry for multiple instances, and how many additional entries # to write addtl_entries = 0 # remove the .git for adding repo info to manifest if self.repo.endswith('.git'): self.repo = self.repo[:-4] # remove @ in match for template setting purposes if match[0].find('@') >= 0: true_name = match[0].split('@')[1] else: true_name = match[0] template = Template(template=self.manifest) # TODO check for special settings here first for the specific match self.version = match[1] response = self.p_helper.checkout(branch=self.branch, version=self.version) if response[0]: section = self.org + ":" + self.name + ":" + true_name + ":" section += self.branch + ":" + self.version # need to get rid of temp identifiers for tools in same repo match_path = self.path + match[0].split('@')[0] if not self.core: image_name = self.org + "-" + self.name + "-" if match[0] != '': # if tool is in a subdir, add that to the name of the # image image_name += '-'.join(match[0].split('/')[1:]) + "-" image_name += self.branch + ":" + self.version else: image_name = ('cyberreboot/vent-' + match[0].split('/')[-1] + ':' + self.branch) image_name = image_name.replace('_', '-') # check if the section already exists exists, options = template.section(section) previous_commit = None previous_commits = None head = False if exists: for option in options: # TODO check if tool name but a different version # exists - then disable/remove if set if option[0] == 'version' and option[1] == 'HEAD': head = True if option[0] == 'built' and option[1] == 'yes': # !! TODO remove pre-existing image pass if option[0] == 'commit_id': previous_commit = option[1] if option[0] == 'previous_versions': previous_commits = option[1] # check if tool comes from multi directory multi_tool = "no" if match[0].find('@') >= 0: multi_tool = "yes" # !! TODO # check if section should be removed from config i.e. all tools # but new commit removed one that was in a previous commit image_name = image_name.lower() if image_name.endswith(":head"): image_name = image_name.split(":head")[0] + ":HEAD" # set template section & options for tool at version and branch template.add_section(section) template.set_option(section, "name", true_name.split('/')[-1]) template.set_option(section, "namespace", self.org + '/' + self.name) template.set_option(section, "path", match_path) template.set_option(section, "repo", self.repo) template.set_option(section, "enabled", "yes") template.set_option(section, "multi_tool", multi_tool) template.set_option(section, "branch", self.branch) template.set_option(section, "version", self.version) template.set_option(section, "last_updated", str(datetime.utcnow()) + " UTC") template.set_option(section, "image_name", image_name.replace('@', '-')) template.set_option(section, "type", "repository") # save settings in vent.template to plugin_manifest # watch for multiple tools in same directory # just wanted to store match path with @ for path for use in # other actions tool_template = 'vent.template' if match[0].find('@') >= 0: tool_template = match[0].split('@')[1] + '.template' vent_template_path = join(match_path, tool_template) if os.path.exists(vent_template_path): with open(vent_template_path) as f: vent_template_val = f.read() else: vent_template_val = '' settings_dict = ParsedSections(vent_template_val) for setting in settings_dict: template.set_option(section, setting, json.dumps(settings_dict[setting])) # TODO do we need this if we save as a dictionary? vent_template = Template(vent_template_path) vent_status, response = vent_template.option("info", "name") if vent_status: template.set_option(section, "link_name", response) else: template.set_option(section, "link_name", true_name.split('/')[-1]) commit_id = None if self.version == 'HEAD': # remove @ in multi-tools chdir(match_path) cmd = "git rev-parse --short HEAD" commit_id = check_output(shlex.split(cmd), stderr=STDOUT, close_fds=True).strip() template.set_option(section, "commit_id", commit_id) if head: # no need to store previous commits if not HEAD, since # the version will always be the same commit ID if previous_commit and previous_commit != commit_id: if (previous_commits and previous_commit not in previous_commits): previous_commits = (previous_commit + ',' + previous_commits) elif not previous_commits: previous_commits = previous_commit if previous_commits and previous_commits != commit_id: template.set_option(section, "previous_versions", previous_commits) if self.version_alias: template.set_option(section, "version_alias", self.version_alias) if self.groups: template.set_option(section, "groups", self.groups) else: groups = vent_template.option("info", "groups") if groups[0]: template.set_option(section, "groups", groups[1]) # set groups to empty string if no groups defined for tool else: template.set_option(section, "groups", '') template = self._build_image(template, match_path, image_name, section) # write additional entries for multiple instances if addtl_entries > 0: # add 2 for naming conventions for i in range(2, addtl_entries + 2): addtl_section = section.rsplit(':', 2) addtl_section[0] += str(i) addtl_section = ':'.join(addtl_section) template.add_section(addtl_section) orig_vals = template.section(section)[1] for val in orig_vals: template.set_option(addtl_section, val[0], val[1]) template.set_option(addtl_section, "name", true_name.split('/')[-1] + str(i)) # write out configuration to the manifest file template.write_config() # reset to repo directory chdir(self.path) return
def prep_start(self, repo=None, name=None, groups=None, enabled='yes', branch='master', version='HEAD'): """ Start a set of tools that match the parameters given, if no parameters are given, start all installed tools on the master branch at verison HEAD that are enabled """ args = locals() self.logger.info('Starting: prep_start') self.logger.info('Arguments: ' + str(args)) status = (False, None) try: options = [ 'name', 'namespace', 'built', 'groups', 'path', 'image_name', 'branch', 'repo', 'type', 'version' ] vent_config = Template(template=self.path_dirs.cfg_file) manifest = Template(self.manifest) files = vent_config.option('main', 'files') files = (files[0], expanduser(files[1])) s, _ = self.constraint_options(args, options) status, tool_d = self.start_sections(s, files, groups, enabled, branch, version) # look out for links to delete because they're defined externally links_to_delete = set() # get instances for each tool tool_instances = {} sections = manifest.sections()[1] for section in sections: settings = manifest.option(section, 'settings') if settings[0]: settings = json.loads(settings[1]) if 'instances' in settings: l_name = manifest.option(section, 'link_name') if l_name[0]: tool_instances[l_name[1]] = int( settings['instances']) # check and update links, volumes_from, network_mode for container in list(tool_d.keys()): if 'labels' not in tool_d[ container] or 'vent.groups' not in tool_d[container][ 'labels'] or 'core' not in tool_d[container][ 'labels']['vent.groups']: tool_d[container]['remove'] = True if 'links' in tool_d[container]: for link in list(tool_d[container]['links'].keys()): # add links to external services already running if # necessary, by default configure local services too configure_local = True ext = 'external-services' if link in vent_config.options(ext)[1]: try: lconf = json.loads( vent_config.option(ext, link)[1]) if ('locally_active' not in lconf or lconf['locally_active'] == 'no'): ip_adr = lconf['ip_address'] port = lconf['port'] tool_d[container]['extra_hosts'] = {} # containers use lowercase names for # connections tool_d[container]['extra_hosts'][ link.lower()] = ip_adr # create an environment variable for container # to access port later env_variable = link.upper() + \ '_CUSTOM_PORT=' + port if 'environment' not in tool_d[container]: tool_d[container]['environment'] = [] tool_d[container]['environment'].append( env_variable) # remove the entry from links because no # longer connecting to local container links_to_delete.add(link) configure_local = False except Exception as e: # pragma: no cover self.logger.error("couldn't load external" ' settings because: ' + str(e)) configure_local = True status = False if configure_local: for c in list(tool_d.keys()): if ('tmp_name' in tool_d[c] and tool_d[c]['tmp_name'] == link): tool_d[container]['links'][ tool_d[c]['name']] = tool_d[container][ 'links'].pop(link) if link in tool_instances and tool_instances[ link] > 1: for i in range( 2, tool_instances[link] + 1): tool_d[container]['links'][ tool_d[c]['name'] + str(i)] = tool_d[container][ 'links'][tool_d[c] ['name']] + str(i) if 'volumes_from' in tool_d[container]: tmp_volumes_from = tool_d[container]['volumes_from'] tool_d[container]['volumes_from'] = [] for volumes_from in list(tmp_volumes_from): for c in list(tool_d.keys()): if ('tmp_name' in tool_d[c] and tool_d[c]['tmp_name'] == volumes_from): tool_d[container]['volumes_from'].append( tool_d[c]['name']) tmp_volumes_from.remove(volumes_from) tool_d[container]['volumes_from'] += tmp_volumes_from if 'network_mode' in tool_d[container]: if tool_d[container]['network_mode'].startswith( 'container:'): network_c_name = tool_d[container][ 'network_mode'].split('container:')[1] for c in list(tool_d.keys()): if ('tmp_name' in tool_d[c] and tool_d[c]['tmp_name'] == network_c_name): tool_d[container]['network_mode'] = 'container:' + \ tool_d[c]['name'] # remove tmp_names for c in list(tool_d.keys()): if 'tmp_name' in tool_d[c]: del tool_d[c]['tmp_name'] # remove links section if all were externally configured for c in list(tool_d.keys()): if 'links' in tool_d[c]: for link in links_to_delete: if link in tool_d[c]['links']: del tool_d[c]['links'][link] # delete links if no more defined if not tool_d[c]['links']: del tool_d[c]['links'] # remove containers that shouldn't be started for c in list(tool_d.keys()): deleted = False if 'start' in tool_d[c] and not tool_d[c]['start']: del tool_d[c] deleted = True if not deleted: # look for tools services that are being done externally # tools are capitalized in vent.cfg, so make them lowercase # for comparison ext = 'external-services' external_tools = vent_config.section(ext)[1] name = tool_d[c]['labels']['vent.name'] for tool in external_tools: if name == tool[0].lower(): try: tool_config = json.loads(tool[1]) if ('locally_active' in tool_config and tool_config['locally_active'] == 'no'): del tool_d[c] except Exception as e: # pragma: no cover self.logger.warning( 'Locally running container ' + name + ' may be redundant') if status: status = (True, tool_d) else: status = (False, tool_d) except Exception as e: # pragma: no cover self.logger.error('prep_start failed with error: ' + str(e)) status = (False, e) self.logger.info('Status of prep_start: ' + str(status[0])) self.logger.info('Finished: prep_start') return status
def start_sections(self, s, files, groups, enabled, branch, version): """ Run through sections for prep_start """ tool_d = {} status = (True, None) for section in s: # initialize needed vars c_name = s[section]['image_name'].replace(':', '-') c_name = c_name.replace('/', '-') instance_num = re.search(r'\d+$', s[section]['name']) if instance_num: c_name += instance_num.group() image_name = s[section]['image_name'] # checkout the right version and branch of the repo cwd = getcwd() self.logger.info('current directory is: ' + str(cwd)) # images built from registry won't have path if s[section]['path'] != '': chdir(join(s[section]['path'])) # TODO commenting out for now, should use update_repo #status = self.checkout(branch=branch, version=version) status = (True, None) self.logger.info(status) chdir(cwd) tool_d[c_name] = {'image': image_name, 'name': c_name} # get rid of all commented sections in various runtime # configurations manifest = Template(self.manifest) overall_dict = {} for setting in ['info', 'docker', 'gpu', 'settings', 'service']: option = manifest.option(section, setting) if option[0]: overall_dict[setting] = {} settings_dict = json.loads(option[1]) for opt in settings_dict: if not opt.startswith('#'): overall_dict[setting][opt] = \ settings_dict[opt] if 'docker' in overall_dict: options_dict = overall_dict['docker'] for option in options_dict: options = options_dict[option] # check for commands to evaluate if '`' in options: cmds = options.split('`') if len(cmds) > 2: i = 1 while i < len(cmds): try: cmds[i] = check_output( shlex.split(cmds[i]), stderr=STDOUT, close_fds=True).strip().decode('utf-8') except Exception as e: # pragma: no cover self.logger.error( 'unable to evaluate command specified in vent.template: ' + str(e)) i += 2 options = ''.join(cmds) # check for commands to evaluate # store options set for docker try: tool_d[c_name][option] = literal_eval(options) except Exception as e: # pragma: no cover self.logger.info('unable to literal_eval: ' + str(options)) tool_d[c_name][option] = options if 'labels' not in tool_d[c_name]: tool_d[c_name]['labels'] = {} # get the service uri info if 'service' in overall_dict: try: options_dict = overall_dict['service'] for option in options_dict: tool_d[c_name]['labels'][option] = options_dict[option] except Exception as e: # pragma: no cover self.logger.error('unable to store service options for ' 'docker: ' + str(e)) # check for gpu settings if 'gpu' in overall_dict: try: options_dict = json.loads(status[1]) for option in options_dict: tool_d[c_name]['labels']['gpu.' + option] = options_dict[option] except Exception as e: # pragma: no cover self.logger.error('unable to store gpu options for ' 'docker: ' + str(e)) # get temporary name for links, etc. plugin_c = Template(template=self.manifest) status, plugin_sections = plugin_c.sections() self.logger.info(status) for plugin_section in plugin_sections: status = plugin_c.option(plugin_section, 'link_name') self.logger.info(status) image_status = plugin_c.option(plugin_section, 'image_name') self.logger.info(image_status) if status[0] and image_status[0]: cont_name = image_status[1].replace(':', '-') cont_name = cont_name.replace('/', '-') if cont_name not in tool_d: tool_d[cont_name] = { 'image': image_status[1], 'name': cont_name, 'start': False } tool_d[cont_name]['tmp_name'] = status[1] # add extra labels tool_d[c_name]['labels']['vent'] = Version() tool_d[c_name]['labels']['vent.namespace'] = s[section][ 'namespace'] tool_d[c_name]['labels']['vent.branch'] = branch tool_d[c_name]['labels']['vent.version'] = version tool_d[c_name]['labels']['vent.name'] = s[section]['name'] tool_d[c_name]['labels']['vent.section'] = section tool_d[c_name]['labels']['vent.repo'] = s[section]['repo'] tool_d[c_name]['labels']['vent.type'] = s[section]['type'] # check for log_config settings in external-services externally_configured = False vent_config = Template(self.path_dirs.cfg_file) for ext_tool in vent_config.section('external-services')[1]: if ext_tool[0].lower() == 'syslog': try: log_dict = json.loads(ext_tool[1]) # configure if not locally active if ('locally_active' not in log_dict or log_dict['locally_active'] == 'no'): del log_dict['locally_active'] log_config = {} log_config['type'] = 'syslog' log_config['config'] = {} ip_address = '' port = '' for option in log_dict: if option == 'ip_address': ip_address = log_dict[option] elif option == 'port': port = log_dict['port'] syslog_address = 'tcp://' + ip_address + ':' + port syslog_config = { 'syslog-address': syslog_address, 'syslog-facility': 'daemon', 'tag': '{{.Name}}' } log_config['config'].update(syslog_config) externally_configured = True except Exception as e: # pragma: no cover self.logger.error('external settings for log_config' " couldn't be stored because: " + str(e)) externally_configured = False if not externally_configured: log_config = { 'type': 'syslog', 'config': { 'syslog-address': 'tcp://0.0.0.0:514', 'syslog-facility': 'daemon', 'tag': '{{.Name}}' } } if 'groups' in s[section]: # add labels for groups tool_d[c_name]['labels']['vent.groups'] = s[section]['groups'] # add restart=always to core containers if 'core' in s[section]['groups']: tool_d[c_name]['restart_policy'] = {'Name': 'always'} # map network names to environment variables if 'network' in s[section]['groups']: vent_config = Template(template=self.path_dirs.cfg_file) nic_mappings = vent_config.section('network-mapping') nics = '' if nic_mappings[0]: for nic in nic_mappings[1]: nics += nic[0] + ':' + nic[1] + ',' nics = nics[:-1] if nics: if 'environment' in tool_d[c_name]: tool_d[c_name]['environment'].append('VENT_NICS=' + nics) else: tool_d[c_name]['environment'] = [ 'VENT_NICS=' + nics ] # send logs to syslog if ('syslog' not in s[section]['groups'] and 'core' in s[section]['groups']): log_config['config']['tag'] = '{{.Name}}' tool_d[c_name]['log_config'] = log_config if 'syslog' not in s[section]['groups']: tool_d[c_name]['log_config'] = log_config # mount necessary directories if 'files' in s[section]['groups']: ulimits = [] ulimits.append( docker.types.Ulimit(name='nofile', soft=1048576, hard=1048576)) tool_d[c_name]['ulimits'] = ulimits # check if running in a docker container if 'VENT_CONTAINERIZED' in environ and environ[ 'VENT_CONTAINERIZED'] == 'true': if 'volumes_from' in tool_d[c_name]: tool_d[c_name]['volumes_from'].append( environ['HOSTNAME']) else: tool_d[c_name]['volumes_from'] = [ environ['HOSTNAME'] ] else: if 'volumes' in tool_d[c_name]: tool_d[c_name]['volumes'][ self.path_dirs.base_dir[:-1]] = { 'bind': '/vent', 'mode': 'ro' } else: tool_d[c_name]['volumes'] = { self.path_dirs.base_dir[:-1]: { 'bind': '/vent', 'mode': 'ro' } } if files[0]: if 'volumes' in tool_d[c_name]: tool_d[c_name]['volumes'][files[1]] = { 'bind': '/files', 'mode': 'rw' } else: tool_d[c_name]['volumes'] = { files[1]: { 'bind': '/files', 'mode': 'rw' } } else: tool_d[c_name]['log_config'] = log_config # add label for priority if 'settings' in overall_dict: try: options_dict = overall_dict['settings'] for option in options_dict: if option == 'priority': tool_d[c_name]['labels'][ 'vent.priority'] = options_dict[option] except Exception as e: # pragma: no cover self.logger.error('unable to store settings options ' 'for docker ' + str(e)) # only start tools that have been built if s[section]['built'] != 'yes': del tool_d[c_name] # store section information for adding info to manifest later else: tool_d[c_name]['section'] = section return status, tool_d
def _build_manifest(self, matches): """ Builds and writes the manifest for the tools being added """ # !! TODO check for pre-existing that conflict with request and disable and/or remove image for match in matches: template = Template(template=self.manifest) # !! TODO check for special settings here first for the specific match self.version = match[1] response = self.checkout() if response[0]: section = self.org + ":" + self.name + ":" + match[0] + ":" + self.branch + ":" + self.version match_path = self.path + match[0] image_name = self.org + "-" + self.name + "-" if match[0] != '': # if tool is in a subdir, add that to the name of the image image_name += '-'.join(match[0].split('/')[1:]) + "-" image_name += self.branch + ":" + self.version # check if the section already exists exists, options = template.section(section) previous_commit = None previous_commits = None head = False if exists: for option in options: # TODO check if tool name but a different version exists - then disable/remove if set if option[0] == 'version' and option[1] == 'HEAD': head = True if option[0] == 'built' and option[1] == 'yes': # !! TODO remove pre-existing image pass if option[0] == 'commit_id': previous_commit = option[1] if option[0] == 'previous_versions': previous_commits = option[1] # !! TODO # check if section should be removed from config - i.e. all tools, # but new commit removed one that was in a previous commit # set template section and options for tool at version and branch template.add_section(section) template.set_option(section, "name", match[0].split('/')[-1]) template.set_option(section, "namespace", self.org+'/'+self.name) template.set_option(section, "path", match_path) template.set_option(section, "repo", self.repo) template.set_option(section, "enabled", "yes") template.set_option(section, "branch", self.branch) template.set_option(section, "version", self.version) template.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") template.set_option(section, "image_name", image_name) vent_template = Template(template=os.path.join(match_path, 'vent.template')) vent_status, response = vent_template.option("info", "name") if vent_status: template.set_option(section, "link_name", response) else: template.set_option(section, "link_name", match[0].split('/')[-1]) commit_id = None if self.version == 'HEAD': os.chdir(match_path) commit_id = subprocess.check_output(shlex.split("git rev-parse --short HEAD"), stderr=subprocess.STDOUT, close_fds=True).strip() template.set_option(section, "commit_id", commit_id) if head: # no need to store previous commits if not HEAD, since # the version will always be the same commit ID if previous_commit and previous_commit != commit_id: if previous_commits and previous_commit not in previous_commits: previous_commits = previous_commit+','+previous_commits elif not previous_commits: previous_commits = previous_commit if previous_commits and previous_commits != commit_id: template.set_option(section, "previous_versions", previous_commits) if self.version_alias: template.set_option(section, "version_alias", self.version_alias) if self.groups: template.set_option(section, "groups", self.groups) else: vent_template = os.path.join(match_path, 'vent.template') if os.path.exists(vent_template): v_template = Template(template=vent_template) groups = v_template.option("info", "groups") if groups[0]: template.set_option(section, "groups", groups[1]) template = self._build_image(template, match_path, image_name, section) # write out configuration to the manifest file template.write_config() # reset to repo directory os.chdir(self.path) return
def Services(core, vent=True, external=False, **kargs): """ Get services that have exposed ports, expects param core to be True or False based on which type of services to return, by default limit to vent containers and processes not running externally, if not limited by vent containers, then core is ignored. """ services = [] path_dirs = PathDirs(**kargs) template = Template(template=path_dirs.cfg_file) services_uri = template.option("main", "services_uri") try: # look for internal services if not external: d_client = docker.from_env() if vent: c_filter = {'label': 'vent'} containers = d_client.containers.list(filters=c_filter) else: containers = d_client.containers.list() for c in containers: uri_prefix = '' uri_postfix = '' uri_user = '' uri_pw = '' name = None if vent and 'vent.name' in c.attrs['Config']['Labels']: if ((core and 'vent.groups' in c.attrs['Config']['Labels'] and 'core' in c.attrs['Config']['Labels']['vent.groups']) or (not core and 'vent.groups' in c.attrs['Config']['Labels'] and 'core' not in c.attrs['Config']['Labels']['vent.groups'])): name = c.attrs['Config']['Labels']['vent.name'] if 'uri_prefix' in c.attrs['Config']['Labels']: uri_prefix = c.attrs['Config']['Labels'][ 'uri_prefix'] if 'uri_postfix' in c.attrs['Config']['Labels']: uri_postfix = c.attrs['Config']['Labels'][ 'uri_postfix'] if 'uri_user' in c.attrs['Config']['Labels']: uri_user = "******" uri_user += c.attrs['Config']['Labels']['uri_user'] if 'uri_pw' in c.attrs['Config']['Labels']: uri_pw = " pw:" uri_pw += c.attrs['Config']['Labels']['uri_pw'] else: name = c.name ports = c.attrs['NetworkSettings']['Ports'] p = [] for port in ports: if ports[port]: uri_creds = '' if uri_user or uri_pw: uri_creds = " - (" + uri_user + uri_pw + " )" host = ports[port][0]['HostIp'] if services_uri[0] and host == '0.0.0.0': host = services_uri[1] p.append(uri_prefix + host + ":" + ports[port][0]['HostPort'] + uri_postfix + uri_creds) if p and name: services.append((name, p)) # look for external services else: ext_tools = template.section('external-services')[1] for ext_tool in ext_tools: try: name = ext_tool[0].lower() p = [] settings_dict = json.loads(ext_tool[1]) if ('locally_active' in settings_dict and settings_dict['locally_active'] == 'no'): # default protocol to display will be http protocol = 'http' ip_address = '' port = '' for setting in settings_dict: if setting == 'ip_address': ip_address = settings_dict[setting] if setting == 'port': port = settings_dict[setting] if setting == 'protocol': protocol = settings_dict[setting] p.append(protocol + '://' + ip_address + ':' + port) if p and name: services.append((name, p)) except Exception: # pragma: no cover p = None except Exception as e: # pragma: no cover pass return services
def create(self, group_view=False): """ Update with current tools """ self.add_handlers({"^T": self.quit, "^Q": self.quit}) self.add(npyscreen.TitleText, name='Select which tools to ' + self.action['action'] + ':', editable=False) togglable = ['remove', 'enable', 'disable', 'build'] if self.action['action_name'] in togglable: self.cur_view = self.add(npyscreen.TitleText, name='Group view:', value='all groups', editable=False, rely=3) self.add_handlers({"^V": self.toggle_view}) i = 5 else: i = 4 if self.action['action_name'] == 'start': response = self.action['api_action'].inventory(choices=[ 'repos', 'tools', 'built', 'enabled', 'running', 'core' ]) else: response = self.action['api_action'].inventory( choices=['core', 'repos', 'tools']) if response[0]: inventory = response[1] repos = inventory['repos'] # dict has repo as key and list of core/non-core tools as values has_core = {} has_non_core = {} # find all tools that are in this repo # and list them if they are core for repo in repos: core_list = [] ncore_list = [] # splice the repo names for processing if (repo.startswith("http")): repo_name = repo.rsplit("/", 2)[1:] else: repo_name = repo.split("/") # determine if enabled or disabled tools should be shown show_disabled = False if 'action_name' in self.action: if self.action['action_name'] == 'enable': show_disabled = True for tool in inventory['tools']: tool_repo_name = tool.split(":") # cross reference repo names if (repo_name[0] == tool_repo_name[0] and repo_name[1] == tool_repo_name[1]): # check to ensure tool not set to locally active = no # in vent.cfg externally_active = False vent_cfg_file = self.action['api_action'].vent_config vent_cfg = Template(vent_cfg_file) tool_pairs = vent_cfg.section('external-services')[1] for ext_tool in tool_pairs: if ext_tool[0].lower() == inventory['tools'][tool]: try: ext_tool_options = json.loads(ext_tool[1]) loc = 'locally_active' if (loc in ext_tool_options and ext_tool_options[loc] == 'no'): externally_active = True except Exception as e: self.logger.error("Couldn't check ext" " because: " + str(e)) externally_active = False # check to ensure not disabled disabled = False manifest = Template(self.api_action.plugin.manifest) if manifest.option(tool, 'enabled')[1] == 'no': disabled = True if (not externally_active and not disabled and not show_disabled): instance_num = re.search( r'\d+$', manifest.option(tool, 'name')[1]) if not instance_num: ncore_list.append(tool) # multiple instances share same image elif self.action[ 'action_name'] not in self.no_instance: ncore_list.append(tool) elif (not externally_active and disabled and show_disabled): instance_num = re.search( r'\d+$', manifest.option(tool, 'name')[1]) if not instance_num: ncore_list.append(tool) # multiple instances share same image elif self.action[ 'action_name'] not in self.no_instance: ncore_list.append(tool) for tool in inventory['core']: tool_repo_name = tool.split(":") # cross reference repo names if (repo_name[0] == tool_repo_name[0] and repo_name[1] == tool_repo_name[1]): # check to ensure tool not set to locally active = no # in vent.cfg externally_active = False vent_cfg_file = self.action['api_action'].vent_config vent_cfg = Template(vent_cfg_file) tool_pairs = vent_cfg.section('external-services')[1] for ext_tool in tool_pairs: if ext_tool[0].lower() == inventory['core'][tool]: try: ext_tool_options = json.loads(ext_tool[1]) loc = 'locally_active' if (loc in ext_tool_options and ext_tool_options[loc] == 'no'): externally_active = True except Exception as e: self.logger.error("Couldn't check ext" " because: " + str(e)) externally_active = False # check to ensure not disabled disabled = False manifest = Template(self.api_action.plugin.manifest) if manifest.option(tool, 'enabled')[1] == 'no': disabled = True if (not externally_active and not disabled and not show_disabled): instance_num = re.search( r'\d+$', manifest.option(tool, 'name')[1]) if not instance_num: core_list.append(tool) # multiple instances share same image elif self.action[ 'action_name'] not in self.no_instance: core_list.append(tool) elif (not externally_active and disabled and show_disabled): instance_num = re.search( r'\d+$', manifest.option(tool, 'name')[1]) if not instance_num: core_list.append(tool) # multiple instances share same image elif self.action[ 'action_name'] not in self.no_instance: core_list.append(tool) has_core[repo] = core_list has_non_core[repo] = ncore_list for repo in repos: self.tools_tc[repo] = {} if self.action['cores']: # make sure only repos with core tools are displayed if has_core.get(repo): self.repo_widgets[repo] = self.add(npyscreen.TitleText, name='Plugin: ' + repo, editable=False, rely=i, relx=5) for tool in has_core[repo]: tool_name = tool.split(":", 2)[2].split("/")[-1] if tool_name == "": tool_name = "/" self.tools_tc[repo][tool] = self.add( npyscreen.CheckBox, name=tool_name, value=True, relx=10) i += 1 i += 3 else: # make sure only repos with non-core tools are displayed if has_non_core.get(repo): self.repo_widgets[repo] = self.add(npyscreen.TitleText, name='Plugin: ' + repo, editable=False, rely=i, relx=5) for tool in has_non_core[repo]: tool_name = tool.split(":", 2)[2].split("/")[-1] if tool_name == "": tool_name = "/" self.tools_tc[repo][tool] = self.add( npyscreen.CheckBox, name=tool_name, value=True, relx=10) i += 1 i += 3 return
def _build_manifest(self, matches): """ Builds and writes the manifest for the tools being added """ # !! TODO check for pre-existing that conflict with request and disable and/or remove image for match in matches: template = Template(template=self.manifest) # !! TODO check for special settings here first for the specific match self.version = match[1] response = self.checkout() if response[0]: section = self.org + ":" + self.name + ":" + match[ 0] + ":" + self.branch + ":" + self.version match_path = self.path + match[0] image_name = self.org + "-" + self.name + "-" if match[0] != '': # if tool is in a subdir, add that to the name of the image image_name += '-'.join(match[0].split('/')[1:]) + "-" image_name += self.branch + ":" + self.version # check if the section already exists exists, options = template.section(section) previous_commit = None previous_commits = None head = False if exists: for option in options: # TODO check if tool name but a different version exists - then disable/remove if set if option[0] == 'version' and option[1] == 'HEAD': head = True if option[0] == 'built' and option[1] == 'yes': # !! TODO remove pre-existing image pass if option[0] == 'commit_id': previous_commit = option[1] if option[0] == 'previous_versions': previous_commits = option[1] # !! TODO # check if section should be removed from config - i.e. all tools, # but new commit removed one that was in a previous commit # set template section and options for tool at version and branch template.add_section(section) template.set_option(section, "name", match[0].split('/')[-1]) template.set_option(section, "namespace", self.org + '/' + self.name) template.set_option(section, "path", match_path) template.set_option(section, "repo", self.repo) template.set_option(section, "enabled", "yes") template.set_option(section, "branch", self.branch) template.set_option(section, "version", self.version) template.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") template.set_option(section, "image_name", image_name) vent_template = Template( template=os.path.join(match_path, 'vent.template')) vent_status, response = vent_template.option("info", "name") if vent_status: template.set_option(section, "link_name", response) else: template.set_option(section, "link_name", match[0].split('/')[-1]) commit_id = None if self.version == 'HEAD': os.chdir(match_path) commit_id = subprocess.check_output( shlex.split("git rev-parse --short HEAD"), stderr=subprocess.STDOUT, close_fds=True).strip() template.set_option(section, "commit_id", commit_id) if head: # no need to store previous commits if not HEAD, since # the version will always be the same commit ID if previous_commit and previous_commit != commit_id: if previous_commits and previous_commit not in previous_commits: previous_commits = previous_commit + ',' + previous_commits elif not previous_commits: previous_commits = previous_commit if previous_commits and previous_commits != commit_id: template.set_option(section, "previous_versions", previous_commits) if self.version_alias: template.set_option(section, "version_alias", self.version_alias) if self.groups: template.set_option(section, "groups", self.groups) else: vent_template = os.path.join(match_path, 'vent.template') if os.path.exists(vent_template): v_template = Template(template=vent_template) groups = v_template.option("info", "groups") if groups[0]: template.set_option(section, "groups", groups[1]) template = self._build_image(template, match_path, image_name, section) # write out configuration to the manifest file template.write_config() # reset to repo directory os.chdir(self.path) return
def prep_start(self, repo=None, name=None, groups=None, enabled="yes", branch="master", version="HEAD", run_build=False): """ Start a set of tools that match the parameters given, if no parameters are given, start all installed tools on the master branch at verison HEAD that are enabled """ args = locals() self.logger.info("Starting: prep_start") self.logger.info("Arguments: " + str(args)) status = (True, None) tool_dict = {} try: del args['run_build'] options = [ 'name', 'namespace', 'built', 'groups', 'path', 'image_name', 'branch', 'version' ] vent_config = Template(template=self.vent_config) files = vent_config.option('main', 'files') sections, template = self.plugin.constraint_options(args, options) for section in sections: # initialize needed vars template_path = os.path.join(sections[section]['path'], 'vent.template') container_name = sections[section]['image_name'].replace( ':', '-') container_name = container_name.replace('/', '-') image_name = sections[section]['image_name'] # checkout the right version and branch of the repo self.plugin.branch = branch self.plugin.version = version cwd = os.getcwd() self.logger.info("current directory is: " + str(cwd)) os.chdir(os.path.join(sections[section]['path'])) status = self.plugin.checkout() self.logger.info(status) os.chdir(cwd) if run_build: status = self.build(name=sections[section]['name'], groups=groups, enabled=enabled, branch=branch, version=version) self.logger.info(status) # set docker settings for container vent_template = Template(template_path) status = vent_template.section('docker') self.logger.info(status) tool_dict[container_name] = { 'image': image_name, 'name': container_name } if status[0]: for option in status[1]: options = option[1] # check for commands to evaluate if '`' in options: cmds = options.split('`') # TODO this probably needs better error checking to handle mismatched `` if len(cmds) > 2: i = 1 while i < len(cmds): try: cmds[i] = subprocess.check_output( shlex.split(cmds[i]), stderr=subprocess.STDOUT, close_fds=True).strip() except Exception as e: # pragma: no cover self.logger.error( "unable to evaluate command specified in vent.template: " + str(e)) i += 2 options = "".join(cmds) # store options set for docker try: tool_dict[container_name][ option[0]] = ast.literal_eval(options) except Exception as e: # pragma: no cover self.logger.error( "unable to store the options set for docker: " + str(e)) tool_dict[container_name][option[0]] = options # get temporary name for links, etc. status = vent_template.section('info') self.logger.info(status) plugin_config = Template(template=self.plugin.manifest) status, plugin_sections = plugin_config.sections() self.logger.info(status) for plugin_section in plugin_sections: status = plugin_config.option(plugin_section, "link_name") self.logger.info(status) image_status = plugin_config.option( plugin_section, "image_name") self.logger.info(image_status) if status[0] and image_status[0]: cont_name = image_status[1].replace(':', '-') cont_name = cont_name.replace('/', '-') if cont_name not in tool_dict: tool_dict[cont_name] = { 'image': image_status[1], 'name': cont_name, 'start': False } tool_dict[cont_name]['tmp_name'] = status[1] # add extra labels if 'labels' not in tool_dict[container_name]: tool_dict[container_name]['labels'] = {} tool_dict[container_name]['labels']['vent'] = Version() tool_dict[container_name]['labels'][ 'vent.namespace'] = sections[section]['namespace'] tool_dict[container_name]['labels']['vent.branch'] = branch tool_dict[container_name]['labels']['vent.version'] = version tool_dict[container_name]['labels']['vent.name'] = sections[ section]['name'] if 'groups' in sections[section]: # add labels for groups tool_dict[container_name]['labels'][ 'vent.groups'] = sections[section]['groups'] # send logs to syslog if 'syslog' not in sections[section][ 'groups'] and 'core' in sections[section]['groups']: tool_dict[container_name]['log_config'] = { 'type': 'syslog', 'config': { 'syslog-address': 'tcp://0.0.0.0:514', 'syslog-facility': 'daemon', 'tag': 'core' } } if 'syslog' not in sections[section]['groups']: tool_dict[container_name]['log_config'] = { 'type': 'syslog', 'config': { 'syslog-address': 'tcp://0.0.0.0:514', 'syslog-facility': 'daemon', 'tag': 'plugin' } } # mount necessary directories if 'files' in sections[section]['groups']: if 'volumes' in tool_dict[container_name]: tool_dict[container_name]['volumes'][ self.plugin.path_dirs.base_dir[:-1]] = { 'bind': '/vent', 'mode': 'ro' } else: tool_dict[container_name]['volumes'] = { self.plugin.path_dirs.base_dir[:-1]: { 'bind': '/vent', 'mode': 'ro' } } if files[0]: tool_dict[container_name]['volumes'][files[1]] = { 'bind': '/files', 'mode': 'ro' } else: tool_dict[container_name]['log_config'] = { 'type': 'syslog', 'config': { 'syslog-address': 'tcp://0.0.0.0:514', 'syslog-facility': 'daemon', 'tag': 'plugin' } } # add label for priority status = vent_template.section('settings') self.logger.info(status) if status[0]: for option in status[1]: if option[0] == 'priority': tool_dict[container_name]['labels'][ 'vent.priority'] = option[1] # only start tools that have been built if sections[section]['built'] != 'yes': del tool_dict[container_name] # check and update links, volumes_from, network_mode for container in tool_dict.keys(): if 'links' in tool_dict[container]: for link in tool_dict[container]['links']: for c in tool_dict.keys(): if 'tmp_name' in tool_dict[c] and tool_dict[c][ 'tmp_name'] == link: tool_dict[container]['links'][ tool_dict[c]['name']] = tool_dict[ container]['links'].pop(link) if 'volumes_from' in tool_dict[container]: tmp_volumes_from = tool_dict[container]['volumes_from'] tool_dict[container]['volumes_from'] = [] for volumes_from in list(tmp_volumes_from): for c in tool_dict.keys(): if 'tmp_name' in tool_dict[c] and tool_dict[c][ 'tmp_name'] == volumes_from: tool_dict[container]['volumes_from'].append( tool_dict[c]['name']) tmp_volumes_from.remove(volumes_from) tool_dict[container]['volumes_from'] += tmp_volumes_from if 'network_mode' in tool_dict[container]: if tool_dict[container]['network_mode'].startswith( 'container:'): network_c_name = tool_dict[container][ 'network_mode'].split('container:')[1] for c in tool_dict.keys(): if 'tmp_name' in tool_dict[c] and tool_dict[c][ 'tmp_name'] == network_c_name: tool_dict[container][ 'network_mode'] = 'container:' + tool_dict[ c]['name'] # remove tmp_names for c in tool_dict.keys(): if 'tmp_name' in tool_dict[c]: del tool_dict[c]['tmp_name'] # remove containers that shouldn't be started for c in tool_dict.keys(): if 'start' in tool_dict[c] and not tool_dict[c]['start']: del tool_dict[c] except Exception as e: self.logger.error("prep_start failed with error: " + str(e)) status = (False, e) status = (True, tool_dict) self.logger.info("Status of prep_start: " + str(status)) self.logger.info("Finished: prep_start") return status
def prep_start(self, repo=None, name=None, groups=None, enabled="yes", branch="master", version="HEAD", run_build=False): """ Start a set of tools that match the parameters given, if no parameters are given, start all installed tools on the master branch at verison HEAD that are enabled """ args = locals() self.logger.info("Starting: prep_start") self.logger.info("Arguments: "+str(args)) status = (True, None) tool_dict = {} try: del args['run_build'] options = ['name', 'namespace', 'built', 'groups', 'path', 'image_name', 'branch', 'version'] vent_config = Template(template=self.vent_config) files = vent_config.option('main', 'files') sections, template = self.plugin.constraint_options(args, options) for section in sections: # initialize needed vars template_path = os.path.join(sections[section]['path'], 'vent.template') container_name = sections[section]['image_name'].replace(':','-') container_name = container_name.replace('/','-') image_name = sections[section]['image_name'] # checkout the right version and branch of the repo self.plugin.branch = branch self.plugin.version = version cwd = os.getcwd() self.logger.info("current directory is: "+str(cwd)) os.chdir(os.path.join(sections[section]['path'])) status = self.plugin.checkout() self.logger.info(status) os.chdir(cwd) if run_build: status = self.build(name=sections[section]['name'], groups=groups, enabled=enabled, branch=branch, version=version) self.logger.info(status) # set docker settings for container vent_template = Template(template_path) status = vent_template.section('docker') self.logger.info(status) tool_dict[container_name] = {'image':image_name, 'name':container_name} if status[0]: for option in status[1]: options = option[1] # check for commands to evaluate if '`' in options: cmds = options.split('`') # TODO this probably needs better error checking to handle mismatched `` if len(cmds) > 2: i = 1 while i < len(cmds): try: cmds[i] = subprocess.check_output(shlex.split(cmds[i]), stderr=subprocess.STDOUT, close_fds=True).strip() except Exception as e: # pragma: no cover self.logger.error("unable to evaluate command specified in vent.template: "+str(e)) i += 2 options = "".join(cmds) # store options set for docker try: tool_dict[container_name][option[0]] = ast.literal_eval(options) except Exception as e: # pragma: no cover self.logger.error("unable to store the options set for docker: "+str(e)) tool_dict[container_name][option[0]] = options # get temporary name for links, etc. status = vent_template.section('info') self.logger.info(status) plugin_config = Template(template=self.plugin.manifest) status, plugin_sections = plugin_config.sections() self.logger.info(status) for plugin_section in plugin_sections: status = plugin_config.option(plugin_section, "link_name") self.logger.info(status) image_status = plugin_config.option(plugin_section, "image_name") self.logger.info(image_status) if status[0] and image_status[0]: cont_name = image_status[1].replace(':','-') cont_name = cont_name.replace('/','-') if cont_name not in tool_dict: tool_dict[cont_name] = {'image':image_status[1], 'name':cont_name, 'start':False} tool_dict[cont_name]['tmp_name'] = status[1] # add extra labels if 'labels' not in tool_dict[container_name]: tool_dict[container_name]['labels'] = {} tool_dict[container_name]['labels']['vent'] = Version() tool_dict[container_name]['labels']['vent.namespace'] = sections[section]['namespace'] tool_dict[container_name]['labels']['vent.branch'] = branch tool_dict[container_name]['labels']['vent.version'] = version tool_dict[container_name]['labels']['vent.name'] = sections[section]['name'] if 'groups' in sections[section]: # add labels for groups tool_dict[container_name]['labels']['vent.groups'] = sections[section]['groups'] # send logs to syslog if 'syslog' not in sections[section]['groups'] and 'core' in sections[section]['groups']: tool_dict[container_name]['log_config'] = {'type':'syslog', 'config': {'syslog-address':'tcp://0.0.0.0:514', 'syslog-facility':'daemon', 'tag':'core'}} if 'syslog' not in sections[section]['groups']: tool_dict[container_name]['log_config'] = {'type':'syslog', 'config': {'syslog-address':'tcp://0.0.0.0:514', 'syslog-facility':'daemon', 'tag':'plugin'}} # mount necessary directories if 'files' in sections[section]['groups']: if 'volumes' in tool_dict[container_name]: tool_dict[container_name]['volumes'][self.plugin.path_dirs.base_dir[:-1]] = {'bind': '/vent', 'mode': 'ro'} else: tool_dict[container_name]['volumes'] = {self.plugin.path_dirs.base_dir[:-1]: {'bind': '/vent', 'mode': 'ro'}} if files[0]: tool_dict[container_name]['volumes'][files[1]] = {'bind': '/files', 'mode': 'ro'} else: tool_dict[container_name]['log_config'] = {'type':'syslog', 'config': {'syslog-address':'tcp://0.0.0.0:514', 'syslog-facility':'daemon', 'tag':'plugin'}} # add label for priority status = vent_template.section('settings') self.logger.info(status) if status[0]: for option in status[1]: if option[0] == 'priority': tool_dict[container_name]['labels']['vent.priority'] = option[1] # only start tools that have been built if sections[section]['built'] != 'yes': del tool_dict[container_name] # check and update links, volumes_from, network_mode for container in tool_dict.keys(): if 'links' in tool_dict[container]: for link in tool_dict[container]['links']: for c in tool_dict.keys(): if 'tmp_name' in tool_dict[c] and tool_dict[c]['tmp_name'] == link: tool_dict[container]['links'][tool_dict[c]['name']] = tool_dict[container]['links'].pop(link) if 'volumes_from' in tool_dict[container]: tmp_volumes_from = tool_dict[container]['volumes_from'] tool_dict[container]['volumes_from'] = [] for volumes_from in list(tmp_volumes_from): for c in tool_dict.keys(): if 'tmp_name' in tool_dict[c] and tool_dict[c]['tmp_name'] == volumes_from: tool_dict[container]['volumes_from'].append(tool_dict[c]['name']) tmp_volumes_from.remove(volumes_from) tool_dict[container]['volumes_from'] += tmp_volumes_from if 'network_mode' in tool_dict[container]: if tool_dict[container]['network_mode'].startswith('container:'): network_c_name = tool_dict[container]['network_mode'].split('container:')[1] for c in tool_dict.keys(): if 'tmp_name' in tool_dict[c] and tool_dict[c]['tmp_name'] == network_c_name: tool_dict[container]['network_mode'] = 'container:'+tool_dict[c]['name'] # remove tmp_names for c in tool_dict.keys(): if 'tmp_name' in tool_dict[c]: del tool_dict[c]['tmp_name'] # remove containers that shouldn't be started for c in tool_dict.keys(): if 'start' in tool_dict[c] and not tool_dict[c]['start']: del tool_dict[c] except Exception as e: self.logger.error("prep_start failed with error: "+str(e)) status = (False, e) status = (True, tool_dict) self.logger.info("Status of prep_start: "+str(status)) self.logger.info("Finished: prep_start") return status
def Services(core, vent=True, external=False, **kargs): """ Get services that have exposed ports, expects param core to be True or False based on which type of services to return, by default limit to vent containers and processes not running externally, if not limited by vent containers, then core is ignored. """ services = [] path_dirs = PathDirs(**kargs) template = Template(template=path_dirs.cfg_file) services_uri = template.option('main', 'services_uri') try: # look for internal services if not external: d_client = docker.from_env() if vent: c_filter = {'label': 'vent'} containers = d_client.containers.list(filters=c_filter) else: containers = d_client.containers.list() for c in containers: uris = {} name = None if vent and 'vent.name' in c.attrs['Config']['Labels']: if ((core and 'vent.groups' in c.attrs['Config']['Labels'] and 'core' in c.attrs['Config']['Labels']['vent.groups']) or (not core and 'vent.groups' in c.attrs['Config']['Labels'] and 'core' not in c.attrs['Config']['Labels']['vent.groups'])): name = c.attrs['Config']['Labels']['vent.name'] if name == '': name = c.attrs['Config']['Labels'][ 'vent.namespace'].split('/')[1] for label in c.attrs['Config']['Labels']: if label.startswith('uri'): try: val = int(label[-1]) if val not in uris: uris[val] = {} uris[val][label[:-1]] = c.attrs['Config'][ 'Labels'][label] except Exception as e: # pragma: no cover logger.error('Malformed services section' ' in the template file ' + str(e)) else: name = c.name if name and 'vent.repo' in c.attrs['Config']['Labels']: name = c.attrs['Config']['Labels']['vent.repo'].split( '/')[-1] + ': ' + name ports = c.attrs['NetworkSettings']['Ports'] p = [] port_num = 1 for port in ports: if ports[port]: try: service_str = '' if 'uri_prefix' in uris[port_num]: service_str += uris[port_num]['uri_prefix'] host = ports[port][0]['HostIp'] if services_uri[0] and host == '0.0.0.0': host = services_uri[1] service_str += host + ':' service_str += ports[port][0]['HostPort'] if 'uri_postfix' in uris[port_num]: service_str += uris[port_num]['uri_postfix'] uri_creds = '' if 'uri_user' in uris[port_num]: uri_creds += ' user:'******'uri_user'] if 'uri_pw' in uris[port_num]: uri_creds += ' pw:' uri_creds += uris[port_num]['uri_pw'] if uri_creds: service_str += ' - (' + uri_creds + ' )' p.append(service_str) except Exception as e: # pragma: no cover logger.info('No services defined for ' + str(name) + ' with exposed port ' + str(port_num) + ' because: ' + str(e)) port_num += 1 if p and name: services.append((name, p)) logger.info(services) # look for external services else: ext_tools = template.section('external-services')[1] for ext_tool in ext_tools: try: name = ext_tool[0].lower() p = [] settings_dict = json.loads(ext_tool[1]) if ('locally_active' in settings_dict and settings_dict['locally_active'] == 'no'): # default protocol to display will be http protocol = 'http' ip_address = '' port = '' for setting in settings_dict: if setting == 'ip_address': ip_address = settings_dict[setting] if setting == 'port': port = settings_dict[setting] if setting == 'protocol': protocol = settings_dict[setting] p.append(protocol + '://' + ip_address + ':' + port) if p and name: services.append((name, p)) except Exception: # pragma: no cover p = None except Exception as e: # pragma: no cover logger.error('Could not get services ' + str(e)) return services