def Dependencies(tools): """ Takes in a list of tools that are being updated and returns any tools that depend on linking to them """ dependencies = [] if tools: path_dirs = PathDirs() man = Template(os.path.join(path_dirs.meta_dir, 'plugin_manifest.cfg')) for section in man.sections()[1]: # don't worry about dealing with tool if it's not running running = man.option(section, 'running') if not running[0] or running[1] != 'yes': continue t_name = man.option(section, 'name')[1] t_branch = man.option(section, 'branch')[1] t_version = man.option(section, 'version')[1] t_identifier = { 'name': t_name, 'branch': t_branch, 'version': t_version } options = man.options(section)[1] if 'docker' in options: d_settings = json.loads(man.option(section, 'docker')[1]) if 'links' in d_settings: for link in json.loads(d_settings['links']): if link in tools: dependencies.append(t_identifier) return dependencies
def Core(branch="master", **kargs): """ Get the normal core tools, and the currently installed/built/running ones, including custom core services """ # !! TODO this might need to store namespaces/branches/versions core = {'built':[], 'running':[], 'installed':[], 'normal':[]} # get normal core tools plugins = Plugin(plugins_dir=".internals/plugins") status, cwd = plugins.clone('https://github.com/cyberreboot/vent') if status: plugins.version = 'HEAD' plugins.branch = branch response = plugins.checkout() matches = plugins._available_tools(groups='core') for match in matches: core['normal'].append(match[0].split('/')[-1]) else: core['normal'] = 'failed' # get core tools that have been installed path_dirs = PathDirs(**kargs) manifest = os.path.join(path_dirs.meta_dir, "plugin_manifest.cfg") template = Template(template=manifest) tools = template.sections() if tools[0]: for tool in tools[1]: groups = template.option(tool, "groups") if groups[0] and "core" in groups[1]: name = template.option(tool, "name") if name[0]: core['installed'].append(name[1]) # get core tools that have been built and/or are running try: d_client = docker.from_env() images = d_client.images.list() for image in images: try: if "vent.groups" in image.attrs['Labels'] and 'core' in image.attrs['Labels']['vent.groups']: if 'vent.name' in image.attrs['Labels']: core['built'].append(image.attrs['Labels']['vent.name']) except Exception as err: # pragma: no cover pass containers = d_client.containers.list() for container in containers: try: if "vent.groups" in container.attrs['Config']['Labels'] and 'core' in container.attrs['Config']['Labels']['vent.groups']: if 'vent.name' in container.attrs['Config']['Labels']: core['running'].append(container.attrs['Config']['Labels']['vent.name']) except Exception as err: # pragma: no cover pass except Exception as e: # pragma: no cover pass return core
def DropLocation(): """ Get the directory that file drop is watching """ template = Template(template=PathDirs().cfg_file) drop_loc = template.option("main", "files")[1] drop_loc = os.path.expanduser(drop_loc) drop_loc = os.path.abspath(drop_loc) return (True, drop_loc)
def list_tools(self): """ Return list of tuples of all tools """ tools = [] template = Template(template=self.manifest) exists, sections = template.sections() if exists: for section in sections: options = { 'section': section, 'enabled': None, 'built': None, 'version': None, 'repo': None, 'branch': None, 'name': None, 'groups': None, 'image_name': None } for option in options.keys(): exists, value = template.option(section, option) if exists: options[option] = value tools.append(options) return tools
def start_priority_containers(self, groups, group_orders, tool_d): """ Select containers based on priorities to start """ vent_cfg = Template(self.path_dirs.cfg_file) cfg_groups = vent_cfg.option('groups', 'start_order') if cfg_groups[0]: cfg_groups = cfg_groups[1].split(',') else: cfg_groups = [] all_groups = sorted(set(groups)) s_conts = [] f_conts = [] # start tools in order of group defined in vent.cfg for group in cfg_groups: # remove from all_groups because already checked out if group in all_groups: all_groups.remove(group) if group in group_orders: for cont_t in sorted(group_orders[group]): if cont_t[1] not in s_conts: s_conts, f_conts = self.start_containers( cont_t[1], tool_d, s_conts, f_conts) # start tools that haven't been specified in the vent.cfg, if any for group in all_groups: if group in group_orders: for cont_t in sorted(group_orders[group]): if cont_t[1] not in s_conts: s_conts, f_conts = self.start_containers( cont_t[1], tool_d, s_conts, f_conts) return (s_conts, f_conts)
def toggle_view(self, *args, **kwargs): group = self.views.popleft() new_display = [] new_display.append('Tools for group ' + group + ' found:') manifest = Template(self.api_action.plugin.manifest) cur_repo = '' for i in range(1, len(self.all_tools) - 1): val = self.all_tools[i] # get repo val if val.startswith(" Plugin:"): new_display.append(val) cur_repo = val.split(':', 1)[1].strip() # determine if tool should be displayed in this group elif val.startswith(" ") and not val.startswith(" "): name = val.strip() constraints = {"repo": cur_repo, "name": name} t_section = self.api_action.p_helper \ .constraint_options(constraints, [])[0] t_section = t_section.keys()[0] if group in manifest.option(t_section, 'groups')[1].split(','): new_display += self.all_tools[i:i+5] elif val == '': new_display.append(val) # if all groups display all groups if group == 'all groups': self.display_val.values = self.all_tools else: self.display_val.values = new_display # redraw self.display() # add group back into cycle self.views.append(group)
def _available_tools(self, groups=None): """ Return list of possible tools in repo for the given version and branch """ matches = [] if not hasattr(self, 'path'): return matches if groups: groups = groups.split(",") for root, dirnames, filenames in os.walk(self.path): for filename in fnmatch.filter(filenames, 'Dockerfile'): # !! TODO deal with wild/etc.? if groups: try: template = Template( template=os.path.join(root, 'vent.template')) for group in groups: template_groups = template.option("info", "groups") if template_groups[0] and group in template_groups[ 1]: matches.append( (root.split(self.path)[1], self.version)) except Exception as e: # pragma: no cover pass else: matches.append((root.split(self.path)[1], self.version)) return matches
def host_config(self): """ Ensure the host configuration file exists """ default_file_dir = "/tmp/vent_files" config = Template(template=os.path.join(self.base_dir, "vent.cfg")) resp = config.section("main") if resp[0]: resp = config.option("main", "files") if not resp[0]: config.add_option("main", "files", default_file_dir) self.ensure_dir(default_file_dir) else: config.add_option("main", "files", default_file_dir) self.ensure_dir(default_file_dir) config.write_config() return
def available_tools(self, path, version="HEAD", groups=None): """ Return list of possible tools in repo for the given version and branch """ matches = [] if groups: groups = groups.split(",") for root, _, filenames in walk(path): files = fnmatch.filter(filenames, 'Dockerfile*') # append additional identifiers to tools if multiple in same # directory add_info = len(files) > 1 for f in files: # !! TODO deal with wild/etc.? addtl_info = '' if add_info: # @ will be delimiter symbol for multi-tools try: addtl_info = '@' + f.split('.')[1] except Exception as e: addtl_info = '@unspecified' if groups: if add_info and not addtl_info == '@unspecified': tool_template = addtl_info.split('@')[1] + '.template' else: tool_template = 'vent.template' try: template = Template(template=join(root, tool_template)) for group in groups: template_groups = template.option("info", "groups") if (template_groups[0] and group in template_groups[1]): matches.append((root.split(path)[1] + addtl_info, version)) except Exception as e: # pragma: no cover self.logger.info("error: " + str(e)) else: matches.append((root.split(path)[1] + addtl_info, version)) return matches
def __init__(self, action=None, logger=None, *args, **keywords): """ Initialize inventory form objects """ self.action = action self.logger = logger self.api_action = self.action['api_action'] # get list of all possible group views to display self.views = deque() possible_groups = set() manifest = Template(self.api_action.plugin.manifest) if self.action['cores']: tools = self.api_action.inventory(choices=['core'])[1]['core'] else: tools = self.api_action.inventory(choices=['tools'])[1]['tools'] for tool in tools: groups = manifest.option(tool, 'groups')[1].split(',') for group in groups: # don't do core because that's the purpose of all in views if group != '' and group != 'core': possible_groups.add(group) self.views += possible_groups self.views.append('all groups') super(InventoryForm, self).__init__(*args, **keywords)
def tools(self): """ Return list of tuples of all tools """ tools = [] template = Template(template=self.manifest) exists, sections = template.sections() if exists: for section in sections: options = {'section':section, 'enabled':None, 'built':None, 'version':None, 'repo':None, 'branch':None, 'name':None, 'groups':None, 'image_name':None} for option in options.keys(): exists, value = template.option(section, option) if exists: options[option] = value tools.append(options) return tools
def _available_tools(self, groups=None): """ Return list of possible tools in repo for the given version and branch """ matches = [] if not hasattr(self, 'path'): return matches if groups: groups = groups.split(",") for root, dirnames, filenames in os.walk(self.path): for filename in fnmatch.filter(filenames, 'Dockerfile'): # !! TODO deal with wild/etc.? if groups: try: template = Template(template=os.path.join(root, 'vent.template')) for group in groups: template_groups = template.option("info", "groups") if template_groups[0] and group in template_groups[1]: matches.append((root.split(self.path)[1], self.version)) except Exception as e: # pragma: no cover pass else: matches.append((root.split(self.path)[1], self.version)) return matches
def __init__(self, *args, **keywords): """ Initialize tool form objects """ self.logger = Logger(__name__) self.logger.info(str(keywords['names'])) self.api_action = Action() self.m_helper = MenuHelper() action = {'api_action': self.api_action} self.tools_tc = {} self.repo_widgets = {} if keywords['action_dict']: action.update(keywords['action_dict']) if keywords['names']: i = 1 for name in keywords['names']: action['action_object' + str(i)] = getattr( self.api_action, name) i += 1 self.action = action # get list of all possible group views to display self.views = deque() possible_groups = set() manifest = Template(self.api_action.plugin.manifest) if self.action['cores']: tools = self.api_action.inventory(choices=['core'])[1]['core'] else: tools = self.api_action.inventory(choices=['tools'])[1]['tools'] for tool in tools: groups = manifest.option(tool, 'groups')[1].split(',') for group in groups: # don't do core because that's the purpose of all in views if group != '' and group != 'core': possible_groups.add(group) self.manifest = manifest self.views += possible_groups self.views.append('all groups') self.no_instance = ['build', 'remove'] super(ToolForm, self).__init__(*args, **keywords)
def GpuUsage(**kargs): """ Get the current GPU usage of available GPUs """ usage = (False, None) gpu_status = {'vent_usage': {'dedicated': [], 'mem_mb': {}}} path_dirs = PathDirs(**kargs) path_dirs.host_config() template = Template(template=path_dirs.cfg_file) # get running jobs using gpus try: d_client = docker.from_env() c = d_client.containers.list(all=False, filters={'label': 'vent-plugin'}) for container in c: if ('vent.gpu' in container.attrs['Config']['Labels'] and container.attrs['Config']['Labels']['vent.gpu'] == 'yes'): device = container.attrs['Config']['Labels']['vent.gpu.device'] if ('vent.gpu.dedicated' in container.attrs['Config']['Labels'] and container.attrs['Config']['Labels'] ['vent.gpu.dedicated'] == 'yes'): gpu_status['vent_usage']['dedicated'].append(device) elif 'vent.gpu.mem_mb' in container.attrs['Config']['Labels']: if device not in gpu_status['vent_usage']['mem_mb']: gpu_status['vent_usage']['mem_mb'][device] = 0 gpu_status['vent_usage']['mem_mb'][device] += int( container.attrs['Config']['Labels']['vent.gpu.mem_mb']) except Exception as e: # pragma: no cover pass port = '3476' # default docker gateway host = '172.17.0.1' result = template.option('nvidia-docker-plugin', 'port') if result[0]: port = result[1] result = template.option('nvidia-docker-plugin', 'host') if result[0]: host = result[1] else: try: # now just requires ip, ifconfig route = check_output(('ip', 'route')).split('\n') default = '' # grab the default network device. for device in route: if 'default' in device: default = device.split()[4] break # grab the IP address for the default device ip_addr = check_output(('ifconfig', default)) ip_addr = ip_addr.split('\n')[1].split()[1] host = ip_addr except Exception as e: # pragma: no cover pass # have to get the info separately to determine how much memory is availabe nd_url = 'http://' + host + ':' + port + '/v1.0/gpu/info/json' try: r = requests.get(nd_url) if r.status_code == 200: status = r.json() for i, device in enumerate(status['Devices']): gm = int(round(math.log(int(device['Memory']['Global']), 2))) gpu_status[i] = { 'global_memory': 2**gm, 'cores': device['Cores'] } else: usage = (False, "Unable to get GPU usage request error code: " + str(r.status_code)) except Exception as e: # pragma: no cover usage = (False, "Error: " + str(e)) # get actual status of each gpu nd_url = 'http://' + host + ':' + port + '/v1.0/gpu/status/json' try: r = requests.get(nd_url) if r.status_code == 200: status = r.json() for i, device in enumerate(status['Devices']): if i not in gpu_status: gpu_status[i] = {} gpu_status[i]['utilization'] = device['Utilization'] gpu_status[i]['memory'] = device['Memory'] gpu_status[i]['processes'] = device['Processes'] usage = (True, gpu_status) else: usage = (False, "Unable to get GPU usage request error code: " + str(r.status_code)) except Exception as e: # pragma: no cover usage = (False, "Error: " + str(e)) return usage
def prep_start(self, repo=None, name=None, groups=None, enabled='yes', branch='master', version='HEAD'): """ Start a set of tools that match the parameters given, if no parameters are given, start all installed tools on the master branch at verison HEAD that are enabled """ args = locals() self.logger.info('Starting: prep_start') self.logger.info('Arguments: ' + str(args)) status = (False, None) try: options = [ 'name', 'namespace', 'built', 'groups', 'path', 'image_name', 'branch', 'repo', 'type', 'version' ] vent_config = Template(template=self.path_dirs.cfg_file) manifest = Template(self.manifest) files = vent_config.option('main', 'files') files = (files[0], expanduser(files[1])) s, _ = self.constraint_options(args, options) status, tool_d = self.start_sections(s, files, groups, enabled, branch, version) # look out for links to delete because they're defined externally links_to_delete = set() # get instances for each tool tool_instances = {} sections = manifest.sections()[1] for section in sections: settings = manifest.option(section, 'settings') if settings[0]: settings = json.loads(settings[1]) if 'instances' in settings: l_name = manifest.option(section, 'link_name') if l_name[0]: tool_instances[l_name[1]] = int( settings['instances']) # check and update links, volumes_from, network_mode for container in list(tool_d.keys()): if 'labels' not in tool_d[ container] or 'vent.groups' not in tool_d[container][ 'labels'] or 'core' not in tool_d[container][ 'labels']['vent.groups']: tool_d[container]['remove'] = True if 'links' in tool_d[container]: for link in list(tool_d[container]['links'].keys()): # add links to external services already running if # necessary, by default configure local services too configure_local = True ext = 'external-services' if link in vent_config.options(ext)[1]: try: lconf = json.loads( vent_config.option(ext, link)[1]) if ('locally_active' not in lconf or lconf['locally_active'] == 'no'): ip_adr = lconf['ip_address'] port = lconf['port'] tool_d[container]['extra_hosts'] = {} # containers use lowercase names for # connections tool_d[container]['extra_hosts'][ link.lower()] = ip_adr # create an environment variable for container # to access port later env_variable = link.upper() + \ '_CUSTOM_PORT=' + port if 'environment' not in tool_d[container]: tool_d[container]['environment'] = [] tool_d[container]['environment'].append( env_variable) # remove the entry from links because no # longer connecting to local container links_to_delete.add(link) configure_local = False except Exception as e: # pragma: no cover self.logger.error("couldn't load external" ' settings because: ' + str(e)) configure_local = True status = False if configure_local: for c in list(tool_d.keys()): if ('tmp_name' in tool_d[c] and tool_d[c]['tmp_name'] == link): tool_d[container]['links'][ tool_d[c]['name']] = tool_d[container][ 'links'].pop(link) if link in tool_instances and tool_instances[ link] > 1: for i in range( 2, tool_instances[link] + 1): tool_d[container]['links'][ tool_d[c]['name'] + str(i)] = tool_d[container][ 'links'][tool_d[c] ['name']] + str(i) if 'volumes_from' in tool_d[container]: tmp_volumes_from = tool_d[container]['volumes_from'] tool_d[container]['volumes_from'] = [] for volumes_from in list(tmp_volumes_from): for c in list(tool_d.keys()): if ('tmp_name' in tool_d[c] and tool_d[c]['tmp_name'] == volumes_from): tool_d[container]['volumes_from'].append( tool_d[c]['name']) tmp_volumes_from.remove(volumes_from) tool_d[container]['volumes_from'] += tmp_volumes_from if 'network_mode' in tool_d[container]: if tool_d[container]['network_mode'].startswith( 'container:'): network_c_name = tool_d[container][ 'network_mode'].split('container:')[1] for c in list(tool_d.keys()): if ('tmp_name' in tool_d[c] and tool_d[c]['tmp_name'] == network_c_name): tool_d[container]['network_mode'] = 'container:' + \ tool_d[c]['name'] # remove tmp_names for c in list(tool_d.keys()): if 'tmp_name' in tool_d[c]: del tool_d[c]['tmp_name'] # remove links section if all were externally configured for c in list(tool_d.keys()): if 'links' in tool_d[c]: for link in links_to_delete: if link in tool_d[c]['links']: del tool_d[c]['links'][link] # delete links if no more defined if not tool_d[c]['links']: del tool_d[c]['links'] # remove containers that shouldn't be started for c in list(tool_d.keys()): deleted = False if 'start' in tool_d[c] and not tool_d[c]['start']: del tool_d[c] deleted = True if not deleted: # look for tools services that are being done externally # tools are capitalized in vent.cfg, so make them lowercase # for comparison ext = 'external-services' external_tools = vent_config.section(ext)[1] name = tool_d[c]['labels']['vent.name'] for tool in external_tools: if name == tool[0].lower(): try: tool_config = json.loads(tool[1]) if ('locally_active' in tool_config and tool_config['locally_active'] == 'no'): del tool_d[c] except Exception as e: # pragma: no cover self.logger.warning( 'Locally running container ' + name + ' may be redundant') if status: status = (True, tool_d) else: status = (False, tool_d) except Exception as e: # pragma: no cover self.logger.error('prep_start failed with error: ' + str(e)) status = (False, e) self.logger.info('Status of prep_start: ' + str(status[0])) self.logger.info('Finished: prep_start') return status
def start_sections(self, s, files, groups, enabled, branch, version): """ Run through sections for prep_start """ tool_d = {} status = (True, None) for section in s: # initialize needed vars c_name = s[section]['image_name'].replace(':', '-') c_name = c_name.replace('/', '-') instance_num = re.search(r'\d+$', s[section]['name']) if instance_num: c_name += instance_num.group() image_name = s[section]['image_name'] # checkout the right version and branch of the repo cwd = getcwd() self.logger.info('current directory is: ' + str(cwd)) # images built from registry won't have path if s[section]['path'] != '': chdir(join(s[section]['path'])) # TODO commenting out for now, should use update_repo #status = self.checkout(branch=branch, version=version) status = (True, None) self.logger.info(status) chdir(cwd) tool_d[c_name] = {'image': image_name, 'name': c_name} # get rid of all commented sections in various runtime # configurations manifest = Template(self.manifest) overall_dict = {} for setting in ['info', 'docker', 'gpu', 'settings', 'service']: option = manifest.option(section, setting) if option[0]: overall_dict[setting] = {} settings_dict = json.loads(option[1]) for opt in settings_dict: if not opt.startswith('#'): overall_dict[setting][opt] = \ settings_dict[opt] if 'docker' in overall_dict: options_dict = overall_dict['docker'] for option in options_dict: options = options_dict[option] # check for commands to evaluate if '`' in options: cmds = options.split('`') if len(cmds) > 2: i = 1 while i < len(cmds): try: cmds[i] = check_output( shlex.split(cmds[i]), stderr=STDOUT, close_fds=True).strip().decode('utf-8') except Exception as e: # pragma: no cover self.logger.error( 'unable to evaluate command specified in vent.template: ' + str(e)) i += 2 options = ''.join(cmds) # check for commands to evaluate # store options set for docker try: tool_d[c_name][option] = literal_eval(options) except Exception as e: # pragma: no cover self.logger.info('unable to literal_eval: ' + str(options)) tool_d[c_name][option] = options if 'labels' not in tool_d[c_name]: tool_d[c_name]['labels'] = {} # get the service uri info if 'service' in overall_dict: try: options_dict = overall_dict['service'] for option in options_dict: tool_d[c_name]['labels'][option] = options_dict[option] except Exception as e: # pragma: no cover self.logger.error('unable to store service options for ' 'docker: ' + str(e)) # check for gpu settings if 'gpu' in overall_dict: try: options_dict = json.loads(status[1]) for option in options_dict: tool_d[c_name]['labels']['gpu.' + option] = options_dict[option] except Exception as e: # pragma: no cover self.logger.error('unable to store gpu options for ' 'docker: ' + str(e)) # get temporary name for links, etc. plugin_c = Template(template=self.manifest) status, plugin_sections = plugin_c.sections() self.logger.info(status) for plugin_section in plugin_sections: status = plugin_c.option(plugin_section, 'link_name') self.logger.info(status) image_status = plugin_c.option(plugin_section, 'image_name') self.logger.info(image_status) if status[0] and image_status[0]: cont_name = image_status[1].replace(':', '-') cont_name = cont_name.replace('/', '-') if cont_name not in tool_d: tool_d[cont_name] = { 'image': image_status[1], 'name': cont_name, 'start': False } tool_d[cont_name]['tmp_name'] = status[1] # add extra labels tool_d[c_name]['labels']['vent'] = Version() tool_d[c_name]['labels']['vent.namespace'] = s[section][ 'namespace'] tool_d[c_name]['labels']['vent.branch'] = branch tool_d[c_name]['labels']['vent.version'] = version tool_d[c_name]['labels']['vent.name'] = s[section]['name'] tool_d[c_name]['labels']['vent.section'] = section tool_d[c_name]['labels']['vent.repo'] = s[section]['repo'] tool_d[c_name]['labels']['vent.type'] = s[section]['type'] # check for log_config settings in external-services externally_configured = False vent_config = Template(self.path_dirs.cfg_file) for ext_tool in vent_config.section('external-services')[1]: if ext_tool[0].lower() == 'syslog': try: log_dict = json.loads(ext_tool[1]) # configure if not locally active if ('locally_active' not in log_dict or log_dict['locally_active'] == 'no'): del log_dict['locally_active'] log_config = {} log_config['type'] = 'syslog' log_config['config'] = {} ip_address = '' port = '' for option in log_dict: if option == 'ip_address': ip_address = log_dict[option] elif option == 'port': port = log_dict['port'] syslog_address = 'tcp://' + ip_address + ':' + port syslog_config = { 'syslog-address': syslog_address, 'syslog-facility': 'daemon', 'tag': '{{.Name}}' } log_config['config'].update(syslog_config) externally_configured = True except Exception as e: # pragma: no cover self.logger.error('external settings for log_config' " couldn't be stored because: " + str(e)) externally_configured = False if not externally_configured: log_config = { 'type': 'syslog', 'config': { 'syslog-address': 'tcp://0.0.0.0:514', 'syslog-facility': 'daemon', 'tag': '{{.Name}}' } } if 'groups' in s[section]: # add labels for groups tool_d[c_name]['labels']['vent.groups'] = s[section]['groups'] # add restart=always to core containers if 'core' in s[section]['groups']: tool_d[c_name]['restart_policy'] = {'Name': 'always'} # map network names to environment variables if 'network' in s[section]['groups']: vent_config = Template(template=self.path_dirs.cfg_file) nic_mappings = vent_config.section('network-mapping') nics = '' if nic_mappings[0]: for nic in nic_mappings[1]: nics += nic[0] + ':' + nic[1] + ',' nics = nics[:-1] if nics: if 'environment' in tool_d[c_name]: tool_d[c_name]['environment'].append('VENT_NICS=' + nics) else: tool_d[c_name]['environment'] = [ 'VENT_NICS=' + nics ] # send logs to syslog if ('syslog' not in s[section]['groups'] and 'core' in s[section]['groups']): log_config['config']['tag'] = '{{.Name}}' tool_d[c_name]['log_config'] = log_config if 'syslog' not in s[section]['groups']: tool_d[c_name]['log_config'] = log_config # mount necessary directories if 'files' in s[section]['groups']: ulimits = [] ulimits.append( docker.types.Ulimit(name='nofile', soft=1048576, hard=1048576)) tool_d[c_name]['ulimits'] = ulimits # check if running in a docker container if 'VENT_CONTAINERIZED' in environ and environ[ 'VENT_CONTAINERIZED'] == 'true': if 'volumes_from' in tool_d[c_name]: tool_d[c_name]['volumes_from'].append( environ['HOSTNAME']) else: tool_d[c_name]['volumes_from'] = [ environ['HOSTNAME'] ] else: if 'volumes' in tool_d[c_name]: tool_d[c_name]['volumes'][ self.path_dirs.base_dir[:-1]] = { 'bind': '/vent', 'mode': 'ro' } else: tool_d[c_name]['volumes'] = { self.path_dirs.base_dir[:-1]: { 'bind': '/vent', 'mode': 'ro' } } if files[0]: if 'volumes' in tool_d[c_name]: tool_d[c_name]['volumes'][files[1]] = { 'bind': '/files', 'mode': 'rw' } else: tool_d[c_name]['volumes'] = { files[1]: { 'bind': '/files', 'mode': 'rw' } } else: tool_d[c_name]['log_config'] = log_config # add label for priority if 'settings' in overall_dict: try: options_dict = overall_dict['settings'] for option in options_dict: if option == 'priority': tool_d[c_name]['labels'][ 'vent.priority'] = options_dict[option] except Exception as e: # pragma: no cover self.logger.error('unable to store settings options ' 'for docker ' + str(e)) # only start tools that have been built if s[section]['built'] != 'yes': del tool_d[c_name] # store section information for adding info to manifest later else: tool_d[c_name]['section'] = section return status, tool_d
def test_option(): """ Test the option function """ instance = Template() instance.option('foo', 'bar')
def prep_start(self, repo=None, name=None, groups=None, enabled="yes", branch="master", version="HEAD", run_build=False): """ Start a set of tools that match the parameters given, if no parameters are given, start all installed tools on the master branch at verison HEAD that are enabled """ args = locals() self.logger.info("Starting: prep_start") self.logger.info("Arguments: " + str(args)) status = (True, None) tool_dict = {} try: del args['run_build'] options = [ 'name', 'namespace', 'built', 'groups', 'path', 'image_name', 'branch', 'version' ] vent_config = Template(template=self.vent_config) files = vent_config.option('main', 'files') sections, template = self.plugin.constraint_options(args, options) for section in sections: # initialize needed vars template_path = os.path.join(sections[section]['path'], 'vent.template') container_name = sections[section]['image_name'].replace( ':', '-') container_name = container_name.replace('/', '-') image_name = sections[section]['image_name'] # checkout the right version and branch of the repo self.plugin.branch = branch self.plugin.version = version cwd = os.getcwd() self.logger.info("current directory is: " + str(cwd)) os.chdir(os.path.join(sections[section]['path'])) status = self.plugin.checkout() self.logger.info(status) os.chdir(cwd) if run_build: status = self.build(name=sections[section]['name'], groups=groups, enabled=enabled, branch=branch, version=version) self.logger.info(status) # set docker settings for container vent_template = Template(template_path) status = vent_template.section('docker') self.logger.info(status) tool_dict[container_name] = { 'image': image_name, 'name': container_name } if status[0]: for option in status[1]: options = option[1] # check for commands to evaluate if '`' in options: cmds = options.split('`') # TODO this probably needs better error checking to handle mismatched `` if len(cmds) > 2: i = 1 while i < len(cmds): try: cmds[i] = subprocess.check_output( shlex.split(cmds[i]), stderr=subprocess.STDOUT, close_fds=True).strip() except Exception as e: # pragma: no cover self.logger.error( "unable to evaluate command specified in vent.template: " + str(e)) i += 2 options = "".join(cmds) # store options set for docker try: tool_dict[container_name][ option[0]] = ast.literal_eval(options) except Exception as e: # pragma: no cover self.logger.error( "unable to store the options set for docker: " + str(e)) tool_dict[container_name][option[0]] = options # get temporary name for links, etc. status = vent_template.section('info') self.logger.info(status) plugin_config = Template(template=self.plugin.manifest) status, plugin_sections = plugin_config.sections() self.logger.info(status) for plugin_section in plugin_sections: status = plugin_config.option(plugin_section, "link_name") self.logger.info(status) image_status = plugin_config.option( plugin_section, "image_name") self.logger.info(image_status) if status[0] and image_status[0]: cont_name = image_status[1].replace(':', '-') cont_name = cont_name.replace('/', '-') if cont_name not in tool_dict: tool_dict[cont_name] = { 'image': image_status[1], 'name': cont_name, 'start': False } tool_dict[cont_name]['tmp_name'] = status[1] # add extra labels if 'labels' not in tool_dict[container_name]: tool_dict[container_name]['labels'] = {} tool_dict[container_name]['labels']['vent'] = Version() tool_dict[container_name]['labels'][ 'vent.namespace'] = sections[section]['namespace'] tool_dict[container_name]['labels']['vent.branch'] = branch tool_dict[container_name]['labels']['vent.version'] = version tool_dict[container_name]['labels']['vent.name'] = sections[ section]['name'] if 'groups' in sections[section]: # add labels for groups tool_dict[container_name]['labels'][ 'vent.groups'] = sections[section]['groups'] # send logs to syslog if 'syslog' not in sections[section][ 'groups'] and 'core' in sections[section]['groups']: tool_dict[container_name]['log_config'] = { 'type': 'syslog', 'config': { 'syslog-address': 'tcp://0.0.0.0:514', 'syslog-facility': 'daemon', 'tag': 'core' } } if 'syslog' not in sections[section]['groups']: tool_dict[container_name]['log_config'] = { 'type': 'syslog', 'config': { 'syslog-address': 'tcp://0.0.0.0:514', 'syslog-facility': 'daemon', 'tag': 'plugin' } } # mount necessary directories if 'files' in sections[section]['groups']: if 'volumes' in tool_dict[container_name]: tool_dict[container_name]['volumes'][ self.plugin.path_dirs.base_dir[:-1]] = { 'bind': '/vent', 'mode': 'ro' } else: tool_dict[container_name]['volumes'] = { self.plugin.path_dirs.base_dir[:-1]: { 'bind': '/vent', 'mode': 'ro' } } if files[0]: tool_dict[container_name]['volumes'][files[1]] = { 'bind': '/files', 'mode': 'ro' } else: tool_dict[container_name]['log_config'] = { 'type': 'syslog', 'config': { 'syslog-address': 'tcp://0.0.0.0:514', 'syslog-facility': 'daemon', 'tag': 'plugin' } } # add label for priority status = vent_template.section('settings') self.logger.info(status) if status[0]: for option in status[1]: if option[0] == 'priority': tool_dict[container_name]['labels'][ 'vent.priority'] = option[1] # only start tools that have been built if sections[section]['built'] != 'yes': del tool_dict[container_name] # check and update links, volumes_from, network_mode for container in tool_dict.keys(): if 'links' in tool_dict[container]: for link in tool_dict[container]['links']: for c in tool_dict.keys(): if 'tmp_name' in tool_dict[c] and tool_dict[c][ 'tmp_name'] == link: tool_dict[container]['links'][ tool_dict[c]['name']] = tool_dict[ container]['links'].pop(link) if 'volumes_from' in tool_dict[container]: tmp_volumes_from = tool_dict[container]['volumes_from'] tool_dict[container]['volumes_from'] = [] for volumes_from in list(tmp_volumes_from): for c in tool_dict.keys(): if 'tmp_name' in tool_dict[c] and tool_dict[c][ 'tmp_name'] == volumes_from: tool_dict[container]['volumes_from'].append( tool_dict[c]['name']) tmp_volumes_from.remove(volumes_from) tool_dict[container]['volumes_from'] += tmp_volumes_from if 'network_mode' in tool_dict[container]: if tool_dict[container]['network_mode'].startswith( 'container:'): network_c_name = tool_dict[container][ 'network_mode'].split('container:')[1] for c in tool_dict.keys(): if 'tmp_name' in tool_dict[c] and tool_dict[c][ 'tmp_name'] == network_c_name: tool_dict[container][ 'network_mode'] = 'container:' + tool_dict[ c]['name'] # remove tmp_names for c in tool_dict.keys(): if 'tmp_name' in tool_dict[c]: del tool_dict[c]['tmp_name'] # remove containers that shouldn't be started for c in tool_dict.keys(): if 'start' in tool_dict[c] and not tool_dict[c]['start']: del tool_dict[c] except Exception as e: self.logger.error("prep_start failed with error: " + str(e)) status = (False, e) status = (True, tool_dict) self.logger.info("Status of prep_start: " + str(status)) self.logger.info("Finished: prep_start") return status
def _build_manifest(self, matches): """ Builds and writes the manifest for the tools being added """ # !! TODO check for pre-existing that conflict with request and # disable and/or remove image for match in matches: # keep track of whether or not to write an additional manifest # entry for multiple instances, and how many additional entries # to write addtl_entries = 0 # remove the .git for adding repo info to manifest if self.repo.endswith('.git'): self.repo = self.repo[:-4] # remove @ in match for template setting purposes if match[0].find('@') >= 0: true_name = match[0].split('@')[1] else: true_name = match[0] template = Template(template=self.manifest) # TODO check for special settings here first for the specific match self.version = match[1] response = self.p_helper.checkout(branch=self.branch, version=self.version) if response[0]: section = self.org + ":" + self.name + ":" + true_name + ":" section += self.branch + ":" + self.version # need to get rid of temp identifiers for tools in same repo match_path = self.path + match[0].split('@')[0] if not self.core: image_name = self.org + "-" + self.name + "-" if match[0] != '': # if tool is in a subdir, add that to the name of the # image image_name += '-'.join(match[0].split('/')[1:]) + "-" image_name += self.branch + ":" + self.version else: image_name = ('cyberreboot/vent-' + match[0].split('/')[-1] + ':' + self.branch) image_name = image_name.replace('_', '-') # check if the section already exists exists, options = template.section(section) previous_commit = None previous_commits = None head = False if exists: for option in options: # TODO check if tool name but a different version # exists - then disable/remove if set if option[0] == 'version' and option[1] == 'HEAD': head = True if option[0] == 'built' and option[1] == 'yes': # !! TODO remove pre-existing image pass if option[0] == 'commit_id': previous_commit = option[1] if option[0] == 'previous_versions': previous_commits = option[1] # check if tool comes from multi directory multi_tool = "no" if match[0].find('@') >= 0: multi_tool = "yes" # !! TODO # check if section should be removed from config i.e. all tools # but new commit removed one that was in a previous commit image_name = image_name.lower() if image_name.endswith(":head"): image_name = image_name.split(":head")[0] + ":HEAD" # set template section & options for tool at version and branch template.add_section(section) template.set_option(section, "name", true_name.split('/')[-1]) template.set_option(section, "namespace", self.org + '/' + self.name) template.set_option(section, "path", match_path) template.set_option(section, "repo", self.repo) template.set_option(section, "enabled", "yes") template.set_option(section, "multi_tool", multi_tool) template.set_option(section, "branch", self.branch) template.set_option(section, "version", self.version) template.set_option(section, "last_updated", str(datetime.utcnow()) + " UTC") template.set_option(section, "image_name", image_name.replace('@', '-')) template.set_option(section, "type", "repository") # save settings in vent.template to plugin_manifest # watch for multiple tools in same directory # just wanted to store match path with @ for path for use in # other actions tool_template = 'vent.template' if match[0].find('@') >= 0: tool_template = match[0].split('@')[1] + '.template' vent_template_path = join(match_path, tool_template) if os.path.exists(vent_template_path): with open(vent_template_path) as f: vent_template_val = f.read() else: vent_template_val = '' settings_dict = ParsedSections(vent_template_val) for setting in settings_dict: template.set_option(section, setting, json.dumps(settings_dict[setting])) # TODO do we need this if we save as a dictionary? vent_template = Template(vent_template_path) vent_status, response = vent_template.option("info", "name") if vent_status: template.set_option(section, "link_name", response) else: template.set_option(section, "link_name", true_name.split('/')[-1]) commit_id = None if self.version == 'HEAD': # remove @ in multi-tools chdir(match_path) cmd = "git rev-parse --short HEAD" commit_id = check_output(shlex.split(cmd), stderr=STDOUT, close_fds=True).strip() template.set_option(section, "commit_id", commit_id) if head: # no need to store previous commits if not HEAD, since # the version will always be the same commit ID if previous_commit and previous_commit != commit_id: if (previous_commits and previous_commit not in previous_commits): previous_commits = (previous_commit + ',' + previous_commits) elif not previous_commits: previous_commits = previous_commit if previous_commits and previous_commits != commit_id: template.set_option(section, "previous_versions", previous_commits) if self.version_alias: template.set_option(section, "version_alias", self.version_alias) if self.groups: template.set_option(section, "groups", self.groups) else: groups = vent_template.option("info", "groups") if groups[0]: template.set_option(section, "groups", groups[1]) # set groups to empty string if no groups defined for tool else: template.set_option(section, "groups", '') template = self._build_image(template, match_path, image_name, section) # write additional entries for multiple instances if addtl_entries > 0: # add 2 for naming conventions for i in range(2, addtl_entries + 2): addtl_section = section.rsplit(':', 2) addtl_section[0] += str(i) addtl_section = ':'.join(addtl_section) template.add_section(addtl_section) orig_vals = template.section(section)[1] for val in orig_vals: template.set_option(addtl_section, val[0], val[1]) template.set_option(addtl_section, "name", true_name.split('/')[-1] + str(i)) # write out configuration to the manifest file template.write_config() # reset to repo directory chdir(self.path) return
def _build_manifest(self, matches): """ Builds and writes the manifest for the tools being added """ # !! TODO check for pre-existing that conflict with request and disable and/or remove image for match in matches: template = Template(template=self.manifest) # !! TODO check for special settings here first for the specific match self.version = match[1] response = self.checkout() if response[0]: section = self.org + ":" + self.name + ":" + match[ 0] + ":" + self.branch + ":" + self.version match_path = self.path + match[0] image_name = self.org + "-" + self.name + "-" if match[0] != '': # if tool is in a subdir, add that to the name of the image image_name += '-'.join(match[0].split('/')[1:]) + "-" image_name += self.branch + ":" + self.version # check if the section already exists exists, options = template.section(section) previous_commit = None previous_commits = None head = False if exists: for option in options: # TODO check if tool name but a different version exists - then disable/remove if set if option[0] == 'version' and option[1] == 'HEAD': head = True if option[0] == 'built' and option[1] == 'yes': # !! TODO remove pre-existing image pass if option[0] == 'commit_id': previous_commit = option[1] if option[0] == 'previous_versions': previous_commits = option[1] # !! TODO # check if section should be removed from config - i.e. all tools, # but new commit removed one that was in a previous commit # set template section and options for tool at version and branch template.add_section(section) template.set_option(section, "name", match[0].split('/')[-1]) template.set_option(section, "namespace", self.org + '/' + self.name) template.set_option(section, "path", match_path) template.set_option(section, "repo", self.repo) template.set_option(section, "enabled", "yes") template.set_option(section, "branch", self.branch) template.set_option(section, "version", self.version) template.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") template.set_option(section, "image_name", image_name) vent_template = Template( template=os.path.join(match_path, 'vent.template')) vent_status, response = vent_template.option("info", "name") if vent_status: template.set_option(section, "link_name", response) else: template.set_option(section, "link_name", match[0].split('/')[-1]) commit_id = None if self.version == 'HEAD': os.chdir(match_path) commit_id = subprocess.check_output( shlex.split("git rev-parse --short HEAD"), stderr=subprocess.STDOUT, close_fds=True).strip() template.set_option(section, "commit_id", commit_id) if head: # no need to store previous commits if not HEAD, since # the version will always be the same commit ID if previous_commit and previous_commit != commit_id: if previous_commits and previous_commit not in previous_commits: previous_commits = previous_commit + ',' + previous_commits elif not previous_commits: previous_commits = previous_commit if previous_commits and previous_commits != commit_id: template.set_option(section, "previous_versions", previous_commits) if self.version_alias: template.set_option(section, "version_alias", self.version_alias) if self.groups: template.set_option(section, "groups", self.groups) else: vent_template = os.path.join(match_path, 'vent.template') if os.path.exists(vent_template): v_template = Template(template=vent_template) groups = v_template.option("info", "groups") if groups[0]: template.set_option(section, "groups", groups[1]) template = self._build_image(template, match_path, image_name, section) # write out configuration to the manifest file template.write_config() # reset to repo directory os.chdir(self.path) return
def start_containers(self, container, tool_d, s_containers, f_containers): """ Start container that was passed in and return status """ # use section to add info to manifest section = tool_d[container]['section'] del tool_d[container]['section'] manifest = Template(self.manifest) try: c = self.d_client.containers.get(container) c.start() s_containers.append(container) manifest.set_option(section, 'running', 'yes') self.logger.info('started ' + str(container) + ' with ID: ' + str(c.short_id)) except Exception as err: try: gpu = 'gpu.enabled' failed = False if (gpu in tool_d[container]['labels'] and tool_d[container]['labels'][gpu] == 'yes'): vent_config = Template(template=self.path_dirs.cfg_file) port = '' host = '' result = vent_config.option('nvidia-docker-plugin', 'port') if result[0]: port = result[1] else: port = '3476' result = vent_config.option('nvidia-docker-plugin', 'host') if result[0]: host = result[1] else: # now just requires ip, ifconfig try: route = check_output( ('ip', 'route')).decode('utf-8').split('\n') default = '' # grab the default network device. for device in route: if 'default' in device: default = device.split()[4] break # grab the IP address for the default device ip_addr = check_output( ('ifconfig', default)).decode('utf-8') ip_addr = ip_addr.split('\n')[1].split()[1] host = ip_addr except Exception as e: # pragma no cover self.logger.error('failed to grab ip. Ensure that \ ip and ifconfig are installed') nd_url = 'http://' + host + ':' + port + '/v1.0/docker/cli' params = {'vol': 'nvidia_driver'} r = requests.get(nd_url, params=params) if r.status_code == 200: options = r.text.split() for option in options: if option.startswith('--volume-driver='): tool_d[container][ 'volume_driver'] = option.split('=', 1)[1] elif option.startswith('--volume='): vol = option.split('=', 1)[1].split(':') if 'volumes' in tool_d[container]: if isinstance(tool_d[container]['volumes'], list): if len(vol) == 2: c_vol = vol[0] + \ ':' + vol[1] + ':rw' else: c_vol = vol[0] + ':' + \ vol[1] + ':' + vol[2] tool_d[container]['volumes'].append( c_vol) else: # Dictionary tool_d[container]['volumes'][ vol[0]] = { 'bind': vol[1], 'mode': vol[2] } else: tool_d[container]['volumes'] = { vol[0]: { 'bind': vol[1], 'mode': vol[2] } } elif option.startswith('--device='): dev = option.split('=', 1)[1] if 'devices' in tool_d[container]: tool_d[container]['devices'].append(dev + ':' + dev + ':rwm') else: tool_d[container]['devices'] = [ dev + ':' + dev + ':rwm' ] else: self.logger.error('Unable to parse ' + 'nvidia-docker option: ' + str(option)) else: failed = True f_containers.append(container) manifest.set_option(section, 'running', 'failed') self.logger.error('failed to start ' + str(container) + ' because nvidia-docker-plugin ' + 'failed with: ' + str(r.status_code)) if not failed: try: self.d_client.containers.remove(container, force=True) self.logger.info('removed old existing container: ' + str(container)) except Exception as e: pass cont_id = self.d_client.containers.run(detach=True, **tool_d[container]) s_containers.append(container) manifest.set_option(section, 'running', 'yes') self.logger.info('started ' + str(container) + ' with ID: ' + str(cont_id)) except Exception as e: # pragma: no cover f_containers.append(container) manifest.set_option(section, 'running', 'failed') self.logger.error('failed to start ' + str(container) + ' because: ' + str(e)) # save changes made to manifest manifest.write_config() return s_containers, f_containers
def create(self, group_view=False): """ Update with current tools """ self.add_handlers({"^T": self.quit, "^Q": self.quit}) self.add(npyscreen.TitleText, name='Select which tools to ' + self.action['action'] + ':', editable=False) togglable = ['remove', 'enable', 'disable', 'build'] if self.action['action_name'] in togglable: self.cur_view = self.add(npyscreen.TitleText, name='Group view:', value='all groups', editable=False, rely=3) self.add_handlers({"^V": self.toggle_view}) i = 5 else: i = 4 if self.action['action_name'] == 'start': response = self.action['api_action'].inventory(choices=[ 'repos', 'tools', 'built', 'enabled', 'running', 'core' ]) else: response = self.action['api_action'].inventory( choices=['core', 'repos', 'tools']) if response[0]: inventory = response[1] repos = inventory['repos'] # dict has repo as key and list of core/non-core tools as values has_core = {} has_non_core = {} # find all tools that are in this repo # and list them if they are core for repo in repos: core_list = [] ncore_list = [] # splice the repo names for processing if (repo.startswith("http")): repo_name = repo.rsplit("/", 2)[1:] else: repo_name = repo.split("/") # determine if enabled or disabled tools should be shown show_disabled = False if 'action_name' in self.action: if self.action['action_name'] == 'enable': show_disabled = True for tool in inventory['tools']: tool_repo_name = tool.split(":") # cross reference repo names if (repo_name[0] == tool_repo_name[0] and repo_name[1] == tool_repo_name[1]): # check to ensure tool not set to locally active = no # in vent.cfg externally_active = False vent_cfg_file = self.action['api_action'].vent_config vent_cfg = Template(vent_cfg_file) tool_pairs = vent_cfg.section('external-services')[1] for ext_tool in tool_pairs: if ext_tool[0].lower() == inventory['tools'][tool]: try: ext_tool_options = json.loads(ext_tool[1]) loc = 'locally_active' if (loc in ext_tool_options and ext_tool_options[loc] == 'no'): externally_active = True except Exception as e: self.logger.error("Couldn't check ext" " because: " + str(e)) externally_active = False # check to ensure not disabled disabled = False manifest = Template(self.api_action.plugin.manifest) if manifest.option(tool, 'enabled')[1] == 'no': disabled = True if (not externally_active and not disabled and not show_disabled): instance_num = re.search( r'\d+$', manifest.option(tool, 'name')[1]) if not instance_num: ncore_list.append(tool) # multiple instances share same image elif self.action[ 'action_name'] not in self.no_instance: ncore_list.append(tool) elif (not externally_active and disabled and show_disabled): instance_num = re.search( r'\d+$', manifest.option(tool, 'name')[1]) if not instance_num: ncore_list.append(tool) # multiple instances share same image elif self.action[ 'action_name'] not in self.no_instance: ncore_list.append(tool) for tool in inventory['core']: tool_repo_name = tool.split(":") # cross reference repo names if (repo_name[0] == tool_repo_name[0] and repo_name[1] == tool_repo_name[1]): # check to ensure tool not set to locally active = no # in vent.cfg externally_active = False vent_cfg_file = self.action['api_action'].vent_config vent_cfg = Template(vent_cfg_file) tool_pairs = vent_cfg.section('external-services')[1] for ext_tool in tool_pairs: if ext_tool[0].lower() == inventory['core'][tool]: try: ext_tool_options = json.loads(ext_tool[1]) loc = 'locally_active' if (loc in ext_tool_options and ext_tool_options[loc] == 'no'): externally_active = True except Exception as e: self.logger.error("Couldn't check ext" " because: " + str(e)) externally_active = False # check to ensure not disabled disabled = False manifest = Template(self.api_action.plugin.manifest) if manifest.option(tool, 'enabled')[1] == 'no': disabled = True if (not externally_active and not disabled and not show_disabled): instance_num = re.search( r'\d+$', manifest.option(tool, 'name')[1]) if not instance_num: core_list.append(tool) # multiple instances share same image elif self.action[ 'action_name'] not in self.no_instance: core_list.append(tool) elif (not externally_active and disabled and show_disabled): instance_num = re.search( r'\d+$', manifest.option(tool, 'name')[1]) if not instance_num: core_list.append(tool) # multiple instances share same image elif self.action[ 'action_name'] not in self.no_instance: core_list.append(tool) has_core[repo] = core_list has_non_core[repo] = ncore_list for repo in repos: self.tools_tc[repo] = {} if self.action['cores']: # make sure only repos with core tools are displayed if has_core.get(repo): self.repo_widgets[repo] = self.add(npyscreen.TitleText, name='Plugin: ' + repo, editable=False, rely=i, relx=5) for tool in has_core[repo]: tool_name = tool.split(":", 2)[2].split("/")[-1] if tool_name == "": tool_name = "/" self.tools_tc[repo][tool] = self.add( npyscreen.CheckBox, name=tool_name, value=True, relx=10) i += 1 i += 3 else: # make sure only repos with non-core tools are displayed if has_non_core.get(repo): self.repo_widgets[repo] = self.add(npyscreen.TitleText, name='Plugin: ' + repo, editable=False, rely=i, relx=5) for tool in has_non_core[repo]: tool_name = tool.split(":", 2)[2].split("/")[-1] if tool_name == "": tool_name = "/" self.tools_tc[repo][tool] = self.add( npyscreen.CheckBox, name=tool_name, value=True, relx=10) i += 1 i += 3 return
def test_option(): """ Test the option function """ instance = Template() instance.option('foo', 'bar')
def cores(self, action, branch='master', version='HEAD'): """ Supply action (install, build, start, stop, clean) for core tools """ self.logger.info('Starting: cores') status = (False, None) try: self.logger.info('action provided: ' + str(action)) core = self.tools_status(True, branch=branch, version=version)[1] if action in ['install', 'build']: tools = [] core_repo = 'https://github.com/cyberreboot/vent' resp = self.p_helper.apply_path(core_repo) if resp[0]: cwd = resp[1] else: self.logger.info('apply_path failed. Exiting cores' ' with status ' + str(resp)) return resp path = os.path.join(self.plugin.path_dirs.plugins_dir, 'cyberreboot/vent') # TODO commenting out for now, should use update_repo # response = self.p_helper.checkout(branch=branch, # version=version) response = (True, None) self.logger.info('status of plugin checkout ' + str(response)) matches = self.p_helper.available_tools(path, version=version, groups='core') for match in matches: name = match[0].rsplit('/')[-1] constraints = {'name': name, 'repo': core_repo} prev_installed, _ = self.p_helper. \ constraint_options(constraints, []) if not prev_installed: tools.append((match[0], '')) # only add stuff not already installed or repo specification if ((tools) or (isinstance(matches, list) and len(matches) == 0)): status = self.plugin.add(core_repo, tools=tools, branch=branch, build=False, core=True) self.logger.info('status of plugin add: ' + str(status)) else: self.logger.info('no new tools to install') status = (True, 'previously installed') plugin_c = Template(template=self.plugin.manifest) sections = plugin_c.sections() for tool in core['normal']: for section in sections[1]: name = plugin_c.option(section, 'name') orig_branch = plugin_c.option(section, 'branch') namespace = plugin_c.option(section, 'namespace') version = plugin_c.option(section, 'version') if (name[1] == tool and orig_branch[1] == branch and namespace[1] == 'cyberreboot/vent' and version[1] == 'HEAD'): plugin_c.set_option(section, 'image_name', 'cyberreboot/vent-' + tool.replace('_', '-') + ':' + branch) plugin_c.write_config() chdir(cwd) if action == 'build': plugin_c = Template(template=self.plugin.manifest) sections = plugin_c.sections() try: for tool in core['normal']: for section in sections[1]: tool = tool.replace('_', '-') image_name = plugin_c.option(section, 'image_name') check_image = 'cyberreboot/vent-' check_image += tool + ':' + branch if image_name[1] == check_image: timestamp = str(datetime.utcnow()) + ' UTC' try: # currently can't use docker-py because it # returns a 404 on pull so no way to valid # if it worked or didn't image_id = None cmd = 'docker pull ' + check_image output = check_output(shlex.split(cmd), stderr=STDOUT).decode('utf-8') # image_name in format of (bool, image_name) name = image_name[1] d_client = docker.from_env() image_attrs = d_client.images.get(name) image_attrs = image_attrs.attrs image_id = image_attrs['Id'].split(':')[ 1][:12] if image_id: plugin_c.set_option(section, 'built', 'yes') plugin_c.set_option(section, 'image_id', image_id) plugin_c.set_option(section, 'last_updated', timestamp) status = (True, 'Pulled ' + tool) self.logger.info(str(status)) else: plugin_c.set_option(section, 'built', 'failed') plugin_c.set_option(section, 'last_updated', timestamp) status = (False, 'Failed to pull image ' + str(output.split('\n')[-1])) self.logger.error(str(status)) except Exception as e: # pragma: no cover plugin_c.set_option(section, 'built', 'failed') plugin_c.set_option(section, 'last_updated', timestamp) status = (False, 'Failed to pull image ' + str(e)) self.logger.error(str(status)) except Exception as e: # pragma: no cover status = (False, 'Failed to pull images ' + str(e)) self.logger.error(str(status)) plugin_c.write_config() elif action == 'start': status = self.api_action.prep_start(groups='core', branch=branch) if status[0]: tool_d = status[1] status = self.api_action.start(tool_d) elif action == 'stop': status = self.api_action.stop(groups='core', branch=branch) elif action == 'clean': status = self.api_action.clean(groups='core', branch=branch) except Exception as e: # pragma: no cover self.logger.info('core failed with error: ' + str(e)) status = (False, e) self.logger.info('Status of core: ' + str(status[0])) self.logger.info('Finished: core') return status
def Services(core, vent=True, external=False, **kargs): """ Get services that have exposed ports, expects param core to be True or False based on which type of services to return, by default limit to vent containers and processes not running externally, if not limited by vent containers, then core is ignored. """ services = [] path_dirs = PathDirs(**kargs) template = Template(template=path_dirs.cfg_file) services_uri = template.option("main", "services_uri") try: # look for internal services if not external: d_client = docker.from_env() if vent: c_filter = {'label': 'vent'} containers = d_client.containers.list(filters=c_filter) else: containers = d_client.containers.list() for c in containers: uri_prefix = '' uri_postfix = '' uri_user = '' uri_pw = '' name = None if vent and 'vent.name' in c.attrs['Config']['Labels']: if ((core and 'vent.groups' in c.attrs['Config']['Labels'] and 'core' in c.attrs['Config']['Labels']['vent.groups']) or (not core and 'vent.groups' in c.attrs['Config']['Labels'] and 'core' not in c.attrs['Config']['Labels']['vent.groups'])): name = c.attrs['Config']['Labels']['vent.name'] if 'uri_prefix' in c.attrs['Config']['Labels']: uri_prefix = c.attrs['Config']['Labels'][ 'uri_prefix'] if 'uri_postfix' in c.attrs['Config']['Labels']: uri_postfix = c.attrs['Config']['Labels'][ 'uri_postfix'] if 'uri_user' in c.attrs['Config']['Labels']: uri_user = "******" uri_user += c.attrs['Config']['Labels']['uri_user'] if 'uri_pw' in c.attrs['Config']['Labels']: uri_pw = " pw:" uri_pw += c.attrs['Config']['Labels']['uri_pw'] else: name = c.name ports = c.attrs['NetworkSettings']['Ports'] p = [] for port in ports: if ports[port]: uri_creds = '' if uri_user or uri_pw: uri_creds = " - (" + uri_user + uri_pw + " )" host = ports[port][0]['HostIp'] if services_uri[0] and host == '0.0.0.0': host = services_uri[1] p.append(uri_prefix + host + ":" + ports[port][0]['HostPort'] + uri_postfix + uri_creds) if p and name: services.append((name, p)) # look for external services else: ext_tools = template.section('external-services')[1] for ext_tool in ext_tools: try: name = ext_tool[0].lower() p = [] settings_dict = json.loads(ext_tool[1]) if ('locally_active' in settings_dict and settings_dict['locally_active'] == 'no'): # default protocol to display will be http protocol = 'http' ip_address = '' port = '' for setting in settings_dict: if setting == 'ip_address': ip_address = settings_dict[setting] if setting == 'port': port = settings_dict[setting] if setting == 'protocol': protocol = settings_dict[setting] p.append(protocol + '://' + ip_address + ':' + port) if p and name: services.append((name, p)) except Exception: # pragma: no cover p = None except Exception as e: # pragma: no cover pass return services
def auto_install(self): """ Automatically detects images and installs them in the manifest if they are not there already """ template = Template(template=self.manifest) sections = template.sections() images = self.d_client.images.list(filters={'label': 'vent'}) add_sections = [] status = (True, None) for image in images: if ('Labels' in image.attrs and 'vent.section' in image.attrs['Config']['Labels'] and not image.attrs['Config']['Labels']['vent.section'] in sections[1]): section = image.attrs['Config']['Labels']['vent.section'] section_str = image.attrs['Config']['Labels'][ 'vent.section'].split(":") template.add_section(section) if 'vent.name' in image.attrs['Config']['Labels']: template.set_option( section, 'name', image.attrs['Config']['Labels']['vent.name']) if 'vent.repo' in image.attrs['Config']['Labels']: template.set_option( section, 'repo', image.attrs['Config']['Labels']['vent.repo']) git_path = join(self.path_dirs.plugins_dir, "/".join(section_str[:2])) if not isdir(git_path): # clone it down status = self.p_helper.clone( image.attrs['Config']['Labels']['vent.repo']) template.set_option(section, 'path', join(git_path, section_str[-3][1:])) # get template settings # TODO account for template files not named vent.template v_template = Template(template=join( git_path, section_str[-3][1:], 'vent.template')) tool_sections = v_template.sections() if tool_sections[0]: for s in tool_sections[1]: section_dict = {} options = v_template.options(s) if options[0]: for option in options[1]: option_name = option if option == 'name': # get link name template.set_option( section, "link_name", v_template.option(s, option)[1]) option_name = 'link_name' opt_val = v_template.option(s, option)[1] section_dict[option_name] = opt_val if section_dict: template.set_option(section, s, json.dumps(section_dict)) if ('vent.type' in image.attrs['Config']['Labels'] and image.attrs['Config']['Labels']['vent.type'] == 'repository'): template.set_option(section, 'namespace', "/".join(section_str[:2])) template.set_option(section, 'enabled', 'yes') template.set_option(section, 'branch', section_str[-2]) template.set_option(section, 'version', section_str[-1]) template.set_option(section, 'last_updated', str(datetime.utcnow()) + " UTC") template.set_option(section, 'image_name', image.attrs['RepoTags'][0]) template.set_option(section, 'type', 'repository') if 'vent.groups' in image.attrs['Config']['Labels']: template.set_option( section, 'groups', image.attrs['Config']['Labels']['vent.groups']) template.set_option(section, 'built', 'yes') template.set_option(section, 'image_id', image.attrs['Id'].split(":")[1][:12]) template.set_option(section, 'running', 'no') # check if image is running as a container containers = self.d_client.containers.list( filters={'label': 'vent'}) for container in containers: if container.attrs['Image'] == image.attrs['Id']: template.set_option(section, 'running', 'yes') add_sections.append(section) template.write_config() if status[0]: status = (True, add_sections) return status
def _build_image(self, template, match_path, image_name, section, build_local=False): """ Build docker images and store results in template """ self.logger.info('Starting: _build_image') status = True def set_instances(template, section, built, image_id=None): """ Set build information for multiple instances """ self.logger.info('entering set_instances') i = 2 while True: addtl_section = section.rsplit(':', 2) addtl_section[0] += str(i) addtl_section = ':'.join(addtl_section) self.logger.info(addtl_section) if template.section(addtl_section)[0]: template.set_option(addtl_section, 'built', built) if image_id: template.set_option(addtl_section, 'image_id', image_id) template.set_option(addtl_section, 'last_updated', Timestamp()) else: break i += 1 # determine whether a tool should be considered a multi instance try: settings_dict = json.loads(template.option(section, 'settings')[1]) if int(settings_dict['instances']) > 1: multi_instance = True else: multi_instance = False except Exception: multi_instance = False status = False # !! TODO return status of whether it built successfully or not if self.build: cwd = getcwd() chdir(match_path) try: name = template.option(section, 'name') groups = template.option(section, 'groups') repo = template.option(section, 'repo') t_type = template.option(section, 'type') path = template.option(section, 'path') must_build = self.fill_config(path[1]) if groups[1] == '' or not groups[0]: groups = (True, 'none') if not name[0]: name = (True, image_name) pull = False image_exists = False output = '' cfg_template = Template(template=self.path_dirs.cfg_file) use_existing_image = False result = cfg_template.option('build-options', 'use_existing_images') if result[0]: use_existing_image = result[1] if use_existing_image == 'yes' and not must_build: try: self.d_client.images.get(image_name) i_attrs = self.d_client.images.get(image_name).attrs image_id = i_attrs['Id'].split(':')[1][:12] template.set_option(section, 'built', 'yes') template.set_option(section, 'image_id', image_id) template.set_option(section, 'last_updated', str(datetime.utcnow()) + ' UTC') # set other instances too if multi_instance: set_instances(template, section, 'yes', image_id) status = (True, 'Found ' + image_name) self.logger.info(str(status)) image_exists = True except docker.errors.ImageNotFound: image_exists = False except Exception as e: # pragma: no cover self.logger.warning('Failed to query Docker for images' ' because: ' + str(e)) if not image_exists: # pull if '/' in image_name, fallback to build if '/' in image_name and not build_local and not must_build: try: # currently can't use docker-py because it doesn't support # support labels on images yet self.logger.info('Trying to pull ' + image_name) output = check_output( shlex.split('docker pull ' + image_name), stderr=STDOUT, close_fds=True).decode('utf-8') self.logger.info('Pulling ' + name[1] + '\n' + output) i_attrs = self.d_client.images.get( image_name).attrs image_id = i_attrs['Id'].split(':')[1][:12] if image_id: template.set_option(section, 'built', 'yes') template.set_option(section, 'image_id', image_id) template.set_option( section, 'last_updated', str(datetime.utcnow()) + ' UTC') # set other instances too if multi_instance: set_instances(template, section, 'yes', image_id) status = (True, 'Pulled ' + image_name) self.logger.info(str(status)) else: template.set_option(section, 'built', 'failed') template.set_option( section, 'last_updated', str(datetime.utcnow()) + ' UTC') # set other instances too if multi_instace: set_instances(template, section, 'failed') status = (False, 'Failed to pull image ' + str(output.split('\n')[-1])) self.logger.warning(str(status)) pull = True except Exception as e: # pragma: no cover self.logger.warning('Failed to pull image, going' ' to build instead: ' + str(e)) status = False if not pull and not image_exists: # get username to label built image with username = getpass.getuser() # see if additional tags needed for images tagged at HEAD commit_tag = '' if image_name.endswith('HEAD'): commit_id = template.option(section, 'commit_id') if commit_id[0]: commit_tag = (' -t ' + image_name[:-4] + str(commit_id[1])) # see if additional file arg needed for building multiple # images from same directory file_tag = ' .' multi_tool = template.option(section, 'multi_tool') if multi_tool[0] and multi_tool[1] == 'yes': specific_file = template.option(section, 'name')[1] if specific_file == 'unspecified': file_tag = ' -f Dockerfile .' else: file_tag = ' -f Dockerfile.' + specific_file + ' .' # update image name with new version for update image_name = image_name.rsplit(':', 1)[0] + ':' + self.branch output = check_output( shlex.split('docker build --label' ' vent --label' ' vent.section=' + section + ' --label' ' vent.repo=' + repo[1] + ' --label' ' vent.type=' + t_type[1] + ' --label' ' vent.name=' + name[1] + ' --label' ' vent.groups=' + groups[1] + ' --label' + ' built-by=' + username + ' -t ' + image_name + commit_tag + file_tag), stderr=STDOUT, close_fds=True).decode('utf-8') self.logger.info('Building ' + name[1] + '\n' + output) image_id = '' for line in output.split('\n'): suc_str = 'Successfully built ' if line.startswith(suc_str): image_id = line.split(suc_str)[1].strip() template.set_option(section, 'built', 'yes') template.set_option(section, 'image_id', image_id) template.set_option(section, 'last_updated', str(datetime.utcnow()) + ' UTC') # set other instances too if multi_instance: set_instances(template, section, 'yes', image_id) except Exception as e: # pragma: no cover self.logger.info('current working directory: ' + str(os.getcwd())) self.logger.error('unable to build image: ' + str(image_name) + ' because: ' + str(e) + ' and ' + str(output)) template.set_option(section, 'built', 'failed') template.set_option(section, 'last_updated', str(datetime.utcnow()) + ' UTC') if multi_instance: set_instances(template, section, 'failed') status = False chdir(cwd) else: template.set_option(section, 'built', 'no') template.set_option(section, 'last_updated', str(datetime.utcnow()) + ' UTC') if multi_instance: set_instances(template, section, 'no') template.set_option(section, 'running', 'no') self.logger.info('Status of _build_image: ' + str(status)) self.logger.info('Finished: _build_image:') return template
def tools_status(self, core, branch="master", version="HEAD", **kargs): """ Get tools that are currently installed/built/running and also the number of repos that those tools come from; can toggle whether looking for core tools or plugin tools """ # !! TODO this might need to store namespaces/branches/versions all_tools = {'built': [], 'running': [], 'installed': [], 'normal': []} core_repo = 'https://github.com/cyberreboot/vent' repos = set() tools = Tools(**kargs) # get manifest file manifest = os.path.join(self.api_action.plugin.path_dirs.meta_dir, "plugin_manifest.cfg") template = Template(template=manifest) tools = template.sections() # get repos if core: p_helper = PluginHelper(plugins_dir='.internals/plugins/') repos.add(core_repo) else: p_helper = PluginHelper(plugins_dir='plugins/') for tool in tools[1]: repo = template.option(tool, 'repo') if repo[0] and repo[1] != core_repo: repos.add(repo[1]) # get normal tools for repo in repos: status, _ = p_helper.clone(repo) if status: p_helper.apply_path(repo) p_helper.checkout(branch=branch, version=version) path, _, _ = p_helper.get_path(repo, core=core) matches = None if core: matches = p_helper.available_tools(path, version=version, groups='core') else: matches = p_helper.available_tools(path, version=version) for match in matches: if core: all_tools['normal'].append(match[0].split('/')[-1].replace('_', '-')) else: all_tools['normal'].append(match[0].split('/')[-1]) # get tools that have been installed for tool in tools[1]: repo = template.option(tool, "repo") if repo[0] and repo[1] in repos: name = template.option(tool, "name") if name[0]: all_tools['installed'].append(name[1].replace('_', '-')) # get tools that have been built and/or are running try: d_client = docker.from_env() images = d_client.images.list(filters={'label': 'vent'}) for image in images: try: core_check = ("vent.groups" in image.attrs['Config']['Labels'] and 'core' in image.attrs['Config']['Labels']['vent.groups']) image_check = None if core: image_check = core_check else: image_check = not core_check if image_check: if ('vent.name' in image.attrs['Config']['Labels'] and 'hidden' not in image.attrs['Config']['Labels']['vent.groups']): if core: all_tools['built'].append(image.attrs['Config']['Labels']['vent.name'].replace('_', '-')) else: all_tools['built'].append(image.attrs['Config']['Labels']['vent.name']) except Exception as err: # pragma: no cover self.logger.error("image_check went wrong " + str(err)) containers = d_client.containers.list(filters={'label': 'vent'}) for container in containers: try: core_check = ("vent.groups" in container.attrs['Config']['Labels'] and 'core' in container.attrs['Config']['Labels']['vent.groups']) container_check = None if core: container_check = core_check else: container_check = not core_check if container_check: if ('vent.name' in container.attrs['Config']['Labels'] and 'hidden' not in image.attrs['Config']['Labels']['vent.groups']): if core: all_tools['running'].append(container.attrs['Config']['Labels']['vent.name'].replace('_', '-')) else: all_tools['running'].append(container.attrs['Config']['Labels']['vent.name']) except Exception as err: # pragma: no cover self.logger.error("core_check went wrong " + str(err)) except Exception as e: # pragma: no cover self.logger.error("Something with docker went wrong " + str(e)) return (len(repos), all_tools)
def cores(self, action, branch="master"): """ Supply action (install, build, start, stop, clean) for core tools """ self.logger.info("Starting: cores") status = (True, None) try: self.logger.info("action provided: "+str(action)) core = Core(branch=branch) if action in ["install", "build"]: tools = [] plugins = Plugin(plugins_dir=".internals/plugins") plugins.version = 'HEAD' plugins.branch = branch plugins.apply_path('https://github.com/cyberreboot/vent') response = plugins.checkout() self.logger.info("status of plugin checkout "+str(response)) matches = plugins._available_tools(groups='core') for match in matches: tools.append((match[0], '')) status = plugins.add('https://github.com/cyberreboot/vent', tools=tools, branch=branch, build=False) self.logger.info("status of plugin add: "+str(status)) plugin_config = Template(template=self.plugin.manifest) sections = plugin_config.sections() for tool in core['normal']: for section in sections[1]: name = plugin_config.option(section, "name") orig_branch = plugin_config.option(section, "branch") namespace = plugin_config.option(section, "namespace") version = plugin_config.option(section, "version") if name[1] == tool and orig_branch[1] == branch and namespace[1] == "cyberreboot/vent" and version[1] == "HEAD": plugin_config.set_option(section, "image_name", "cyberreboot/vent-"+tool+":"+branch) plugin_config.write_config() if action == "build": plugin_config = Template(template=self.plugin.manifest) sections = plugin_config.sections() try: for tool in core['normal']: for section in sections[1]: image_name = plugin_config.option(section, "image_name") if image_name[1] == "cyberreboot/vent-"+tool+":"+branch: try: # currently can't use docker-py because it # returns a 404 on pull so no way to valid if it # worked or didn't #image_id = self.d_client.images.pull('cyberreboot/vent-'+tool, tag=branch) image_id = None output = subprocess.check_output(shlex.split("docker pull cyberreboot/vent-"+tool+":"+branch), stderr=subprocess.STDOUT) for line in output.split('\n'): if line.startswith("Digest: sha256:"): image_id = line.split("Digest: sha256:")[1][:12] if image_id: plugin_config.set_option(section, "built", "yes") plugin_config.set_option(section, "image_id", image_id) plugin_config.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") status = (True, "Pulled "+tool) self.logger.info(str(status)) else: plugin_config.set_option(section, "built", "failed") plugin_config.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") status = (False, "Failed to pull image "+str(output.split('\n')[-1])) self.logger.error(str(status)) except Exception as e: # pragma: no cover plugin_config.set_option(section, "built", "failed") plugin_config.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") status = (False, "Failed to pull image "+str(e)) self.logger.error(str(status)) except Exception as e: # pragma: no cover status = (False, "Failed to pull images "+str(e)) self.logger.error(str(status)) plugin_config.write_config() elif action == "start": status = self.prep_start(groups="core", branch=branch) if status[0]: tool_dict = status[1] status = self.start(tool_dict) elif action == "stop": status = self.stop(groups="core", branch=branch) elif action == "clean": status = self.clean(groups="core", branch=branch) except Exception as e: self.logger.info("core failed with error: "+str(e)) status = (False, e) self.logger.info("Status of core: "+str(status)) self.logger.info("Finished: core") return status
def cores(self, action, branch="master"): """ Supply action (install, build, start, stop, clean) for core tools """ self.logger.info("Starting: cores") status = (True, None) try: self.logger.info("action provided: " + str(action)) core = Core(branch=branch) if action in ["install", "build"]: tools = [] plugins = Plugin(plugins_dir=".internals/plugins") plugins.version = 'HEAD' plugins.branch = branch plugins.apply_path('https://github.com/cyberreboot/vent') response = plugins.checkout() self.logger.info("status of plugin checkout " + str(response)) matches = plugins._available_tools(groups='core') for match in matches: tools.append((match[0], '')) status = plugins.add('https://github.com/cyberreboot/vent', tools=tools, branch=branch, build=False) self.logger.info("status of plugin add: " + str(status)) plugin_config = Template(template=self.plugin.manifest) sections = plugin_config.sections() for tool in core['normal']: for section in sections[1]: name = plugin_config.option(section, "name") orig_branch = plugin_config.option(section, "branch") namespace = plugin_config.option(section, "namespace") version = plugin_config.option(section, "version") if name[1] == tool and orig_branch[ 1] == branch and namespace[ 1] == "cyberreboot/vent" and version[ 1] == "HEAD": plugin_config.set_option( section, "image_name", "cyberreboot/vent-" + tool + ":" + branch) plugin_config.write_config() if action == "build": plugin_config = Template(template=self.plugin.manifest) sections = plugin_config.sections() try: for tool in core['normal']: for section in sections[1]: image_name = plugin_config.option( section, "image_name") if image_name[ 1] == "cyberreboot/vent-" + tool + ":" + branch: try: # currently can't use docker-py because it # returns a 404 on pull so no way to valid if it # worked or didn't #image_id = self.d_client.images.pull('cyberreboot/vent-'+tool, tag=branch) image_id = None output = subprocess.check_output( shlex.split( "docker pull cyberreboot/vent-" + tool + ":" + branch), stderr=subprocess.STDOUT) for line in output.split('\n'): if line.startswith("Digest: sha256:"): image_id = line.split( "Digest: sha256:")[1][:12] if image_id: plugin_config.set_option( section, "built", "yes") plugin_config.set_option( section, "image_id", image_id) plugin_config.set_option( section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") status = (True, "Pulled " + tool) self.logger.info(str(status)) else: plugin_config.set_option( section, "built", "failed") plugin_config.set_option( section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") status = (False, "Failed to pull image " + str(output.split('\n')[-1])) self.logger.error(str(status)) except Exception as e: # pragma: no cover plugin_config.set_option( section, "built", "failed") plugin_config.set_option( section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") status = (False, "Failed to pull image " + str(e)) self.logger.error(str(status)) except Exception as e: # pragma: no cover status = (False, "Failed to pull images " + str(e)) self.logger.error(str(status)) plugin_config.write_config() elif action == "start": status = self.prep_start(groups="core", branch=branch) if status[0]: tool_dict = status[1] status = self.start(tool_dict) elif action == "stop": status = self.stop(groups="core", branch=branch) elif action == "clean": status = self.clean(groups="core", branch=branch) except Exception as e: self.logger.info("core failed with error: " + str(e)) status = (False, e) self.logger.info("Status of core: " + str(status)) self.logger.info("Finished: core") return status
def cores(self, action, branch="master", version='HEAD'): """ Supply action (install, build, start, stop, clean) for core tools """ self.logger.info("Starting: cores") status = (False, None) try: self.logger.info("action provided: " + str(action)) core = self.tools_status(True, branch=branch, version=version)[1] if action in ["install", "build"]: tools = [] core_repo = 'https://github.com/cyberreboot/vent' resp = self.p_helper.apply_path(core_repo) if resp[0]: cwd = resp[1] else: self.logger.info("apply_path failed. Exiting cores" " with status " + str(resp)) return resp path = os.path.join(self.plugin.path_dirs.plugins_dir, 'cyberreboot/vent') response = self.p_helper.checkout(branch=branch, version=version) self.logger.info("status of plugin checkout " + str(response)) matches = self.p_helper.available_tools(path, version=version, groups='core') for match in matches: name = match[0].rsplit('/')[-1] constraints = {'name': name, 'repo': core_repo} prev_installed, _ = self.p_helper. \ constraint_options(constraints, []) if not prev_installed: tools.append((match[0], '')) # only add stuff not already installed or repo specification if ((tools) or (isinstance(matches, list) and len(matches) == 0)): status = self.plugin.add(core_repo, tools=tools, branch=branch, build=False, core=True) self.logger.info("status of plugin add: " + str(status)) else: self.logger.info("no new tools to install") status = (True, "previously installed") plugin_c = Template(template=self.plugin.manifest) sections = plugin_c.sections() for tool in core['normal']: for section in sections[1]: name = plugin_c.option(section, "name") orig_branch = plugin_c.option(section, "branch") namespace = plugin_c.option(section, "namespace") version = plugin_c.option(section, "version") if (name[1] == tool and orig_branch[1] == branch and namespace[1] == "cyberreboot/vent" and version[1] == "HEAD"): plugin_c.set_option(section, "image_name", "cyberreboot/vent-" + tool.replace('_', '-') + ":" + branch) plugin_c.write_config() chdir(cwd) if action == "build": plugin_c = Template(template=self.plugin.manifest) sections = plugin_c.sections() try: for tool in core['normal']: for section in sections[1]: tool = tool.replace('_', '-') image_name = plugin_c.option(section, "image_name") check_image = "cyberreboot/vent-" check_image += tool + ":" + branch if image_name[1] == check_image: timestamp = str(datetime.utcnow()) + " UTC" try: # currently can't use docker-py because it # returns a 404 on pull so no way to valid # if it worked or didn't image_id = None cmd = "docker pull " + check_image output = check_output(shlex.split(cmd), stderr=STDOUT) # image_name in format of (bool, image_name) name = image_name[1] d_client = docker.from_env() image_attrs = d_client.images.get(name) image_attrs = image_attrs.attrs image_id = image_attrs['Id'].split(':')[1][:12] if image_id: plugin_c.set_option(section, "built", "yes") plugin_c.set_option(section, "image_id", image_id) plugin_c.set_option(section, "last_updated", timestamp) status = (True, "Pulled " + tool) self.logger.info(str(status)) else: plugin_c.set_option(section, "built", "failed") plugin_c.set_option(section, "last_updated", timestamp) status = (False, "Failed to pull image " + str(output.split('\n')[-1])) self.logger.error(str(status)) except Exception as e: # pragma: no cover plugin_c.set_option(section, "built", "failed") plugin_c.set_option(section, "last_updated", timestamp) status = (False, "Failed to pull image " + str(e)) self.logger.error(str(status)) except Exception as e: # pragma: no cover status = (False, "Failed to pull images " + str(e)) self.logger.error(str(status)) plugin_c.write_config() elif action == "start": status = self.api_action.prep_start(groups="core", branch=branch) if status[0]: tool_d = status[1] status = self.api_action.start(tool_d) elif action == "stop": status = self.api_action.stop(groups="core", branch=branch) elif action == "clean": status = self.api_action.clean(groups="core", branch=branch) except Exception as e: # pragma: no cover self.logger.info("core failed with error: " + str(e)) status = (False, e) self.logger.info("Status of core: " + str(status[0])) self.logger.info("Finished: core") return status
def _build_manifest(self, matches): """ Builds and writes the manifest for the tools being added """ # !! TODO check for pre-existing that conflict with request and disable and/or remove image for match in matches: template = Template(template=self.manifest) # !! TODO check for special settings here first for the specific match self.version = match[1] response = self.checkout() if response[0]: section = self.org + ":" + self.name + ":" + match[0] + ":" + self.branch + ":" + self.version match_path = self.path + match[0] image_name = self.org + "-" + self.name + "-" if match[0] != '': # if tool is in a subdir, add that to the name of the image image_name += '-'.join(match[0].split('/')[1:]) + "-" image_name += self.branch + ":" + self.version # check if the section already exists exists, options = template.section(section) previous_commit = None previous_commits = None head = False if exists: for option in options: # TODO check if tool name but a different version exists - then disable/remove if set if option[0] == 'version' and option[1] == 'HEAD': head = True if option[0] == 'built' and option[1] == 'yes': # !! TODO remove pre-existing image pass if option[0] == 'commit_id': previous_commit = option[1] if option[0] == 'previous_versions': previous_commits = option[1] # !! TODO # check if section should be removed from config - i.e. all tools, # but new commit removed one that was in a previous commit # set template section and options for tool at version and branch template.add_section(section) template.set_option(section, "name", match[0].split('/')[-1]) template.set_option(section, "namespace", self.org+'/'+self.name) template.set_option(section, "path", match_path) template.set_option(section, "repo", self.repo) template.set_option(section, "enabled", "yes") template.set_option(section, "branch", self.branch) template.set_option(section, "version", self.version) template.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") template.set_option(section, "image_name", image_name) vent_template = Template(template=os.path.join(match_path, 'vent.template')) vent_status, response = vent_template.option("info", "name") if vent_status: template.set_option(section, "link_name", response) else: template.set_option(section, "link_name", match[0].split('/')[-1]) commit_id = None if self.version == 'HEAD': os.chdir(match_path) commit_id = subprocess.check_output(shlex.split("git rev-parse --short HEAD"), stderr=subprocess.STDOUT, close_fds=True).strip() template.set_option(section, "commit_id", commit_id) if head: # no need to store previous commits if not HEAD, since # the version will always be the same commit ID if previous_commit and previous_commit != commit_id: if previous_commits and previous_commit not in previous_commits: previous_commits = previous_commit+','+previous_commits elif not previous_commits: previous_commits = previous_commit if previous_commits and previous_commits != commit_id: template.set_option(section, "previous_versions", previous_commits) if self.version_alias: template.set_option(section, "version_alias", self.version_alias) if self.groups: template.set_option(section, "groups", self.groups) else: vent_template = os.path.join(match_path, 'vent.template') if os.path.exists(vent_template): v_template = Template(template=vent_template) groups = v_template.option("info", "groups") if groups[0]: template.set_option(section, "groups", groups[1]) template = self._build_image(template, match_path, image_name, section) # write out configuration to the manifest file template.write_config() # reset to repo directory os.chdir(self.path) return
def Services(core, vent=True, external=False, **kargs): """ Get services that have exposed ports, expects param core to be True or False based on which type of services to return, by default limit to vent containers and processes not running externally, if not limited by vent containers, then core is ignored. """ services = [] path_dirs = PathDirs(**kargs) template = Template(template=path_dirs.cfg_file) services_uri = template.option('main', 'services_uri') try: # look for internal services if not external: d_client = docker.from_env() if vent: c_filter = {'label': 'vent'} containers = d_client.containers.list(filters=c_filter) else: containers = d_client.containers.list() for c in containers: uris = {} name = None if vent and 'vent.name' in c.attrs['Config']['Labels']: if ((core and 'vent.groups' in c.attrs['Config']['Labels'] and 'core' in c.attrs['Config']['Labels']['vent.groups']) or (not core and 'vent.groups' in c.attrs['Config']['Labels'] and 'core' not in c.attrs['Config']['Labels']['vent.groups'])): name = c.attrs['Config']['Labels']['vent.name'] if name == '': name = c.attrs['Config']['Labels'][ 'vent.namespace'].split('/')[1] for label in c.attrs['Config']['Labels']: if label.startswith('uri'): try: val = int(label[-1]) if val not in uris: uris[val] = {} uris[val][label[:-1]] = c.attrs['Config'][ 'Labels'][label] except Exception as e: # pragma: no cover logger.error('Malformed services section' ' in the template file ' + str(e)) else: name = c.name if name and 'vent.repo' in c.attrs['Config']['Labels']: name = c.attrs['Config']['Labels']['vent.repo'].split( '/')[-1] + ': ' + name ports = c.attrs['NetworkSettings']['Ports'] p = [] port_num = 1 for port in ports: if ports[port]: try: service_str = '' if 'uri_prefix' in uris[port_num]: service_str += uris[port_num]['uri_prefix'] host = ports[port][0]['HostIp'] if services_uri[0] and host == '0.0.0.0': host = services_uri[1] service_str += host + ':' service_str += ports[port][0]['HostPort'] if 'uri_postfix' in uris[port_num]: service_str += uris[port_num]['uri_postfix'] uri_creds = '' if 'uri_user' in uris[port_num]: uri_creds += ' user:'******'uri_user'] if 'uri_pw' in uris[port_num]: uri_creds += ' pw:' uri_creds += uris[port_num]['uri_pw'] if uri_creds: service_str += ' - (' + uri_creds + ' )' p.append(service_str) except Exception as e: # pragma: no cover logger.info('No services defined for ' + str(name) + ' with exposed port ' + str(port_num) + ' because: ' + str(e)) port_num += 1 if p and name: services.append((name, p)) logger.info(services) # look for external services else: ext_tools = template.section('external-services')[1] for ext_tool in ext_tools: try: name = ext_tool[0].lower() p = [] settings_dict = json.loads(ext_tool[1]) if ('locally_active' in settings_dict and settings_dict['locally_active'] == 'no'): # default protocol to display will be http protocol = 'http' ip_address = '' port = '' for setting in settings_dict: if setting == 'ip_address': ip_address = settings_dict[setting] if setting == 'port': port = settings_dict[setting] if setting == 'protocol': protocol = settings_dict[setting] p.append(protocol + '://' + ip_address + ':' + port) if p and name: services.append((name, p)) except Exception: # pragma: no cover p = None except Exception as e: # pragma: no cover logger.error('Could not get services ' + str(e)) return services
def prep_start(self, repo=None, name=None, groups=None, enabled="yes", branch="master", version="HEAD", run_build=False): """ Start a set of tools that match the parameters given, if no parameters are given, start all installed tools on the master branch at verison HEAD that are enabled """ args = locals() self.logger.info("Starting: prep_start") self.logger.info("Arguments: "+str(args)) status = (True, None) tool_dict = {} try: del args['run_build'] options = ['name', 'namespace', 'built', 'groups', 'path', 'image_name', 'branch', 'version'] vent_config = Template(template=self.vent_config) files = vent_config.option('main', 'files') sections, template = self.plugin.constraint_options(args, options) for section in sections: # initialize needed vars template_path = os.path.join(sections[section]['path'], 'vent.template') container_name = sections[section]['image_name'].replace(':','-') container_name = container_name.replace('/','-') image_name = sections[section]['image_name'] # checkout the right version and branch of the repo self.plugin.branch = branch self.plugin.version = version cwd = os.getcwd() self.logger.info("current directory is: "+str(cwd)) os.chdir(os.path.join(sections[section]['path'])) status = self.plugin.checkout() self.logger.info(status) os.chdir(cwd) if run_build: status = self.build(name=sections[section]['name'], groups=groups, enabled=enabled, branch=branch, version=version) self.logger.info(status) # set docker settings for container vent_template = Template(template_path) status = vent_template.section('docker') self.logger.info(status) tool_dict[container_name] = {'image':image_name, 'name':container_name} if status[0]: for option in status[1]: options = option[1] # check for commands to evaluate if '`' in options: cmds = options.split('`') # TODO this probably needs better error checking to handle mismatched `` if len(cmds) > 2: i = 1 while i < len(cmds): try: cmds[i] = subprocess.check_output(shlex.split(cmds[i]), stderr=subprocess.STDOUT, close_fds=True).strip() except Exception as e: # pragma: no cover self.logger.error("unable to evaluate command specified in vent.template: "+str(e)) i += 2 options = "".join(cmds) # store options set for docker try: tool_dict[container_name][option[0]] = ast.literal_eval(options) except Exception as e: # pragma: no cover self.logger.error("unable to store the options set for docker: "+str(e)) tool_dict[container_name][option[0]] = options # get temporary name for links, etc. status = vent_template.section('info') self.logger.info(status) plugin_config = Template(template=self.plugin.manifest) status, plugin_sections = plugin_config.sections() self.logger.info(status) for plugin_section in plugin_sections: status = plugin_config.option(plugin_section, "link_name") self.logger.info(status) image_status = plugin_config.option(plugin_section, "image_name") self.logger.info(image_status) if status[0] and image_status[0]: cont_name = image_status[1].replace(':','-') cont_name = cont_name.replace('/','-') if cont_name not in tool_dict: tool_dict[cont_name] = {'image':image_status[1], 'name':cont_name, 'start':False} tool_dict[cont_name]['tmp_name'] = status[1] # add extra labels if 'labels' not in tool_dict[container_name]: tool_dict[container_name]['labels'] = {} tool_dict[container_name]['labels']['vent'] = Version() tool_dict[container_name]['labels']['vent.namespace'] = sections[section]['namespace'] tool_dict[container_name]['labels']['vent.branch'] = branch tool_dict[container_name]['labels']['vent.version'] = version tool_dict[container_name]['labels']['vent.name'] = sections[section]['name'] if 'groups' in sections[section]: # add labels for groups tool_dict[container_name]['labels']['vent.groups'] = sections[section]['groups'] # send logs to syslog if 'syslog' not in sections[section]['groups'] and 'core' in sections[section]['groups']: tool_dict[container_name]['log_config'] = {'type':'syslog', 'config': {'syslog-address':'tcp://0.0.0.0:514', 'syslog-facility':'daemon', 'tag':'core'}} if 'syslog' not in sections[section]['groups']: tool_dict[container_name]['log_config'] = {'type':'syslog', 'config': {'syslog-address':'tcp://0.0.0.0:514', 'syslog-facility':'daemon', 'tag':'plugin'}} # mount necessary directories if 'files' in sections[section]['groups']: if 'volumes' in tool_dict[container_name]: tool_dict[container_name]['volumes'][self.plugin.path_dirs.base_dir[:-1]] = {'bind': '/vent', 'mode': 'ro'} else: tool_dict[container_name]['volumes'] = {self.plugin.path_dirs.base_dir[:-1]: {'bind': '/vent', 'mode': 'ro'}} if files[0]: tool_dict[container_name]['volumes'][files[1]] = {'bind': '/files', 'mode': 'ro'} else: tool_dict[container_name]['log_config'] = {'type':'syslog', 'config': {'syslog-address':'tcp://0.0.0.0:514', 'syslog-facility':'daemon', 'tag':'plugin'}} # add label for priority status = vent_template.section('settings') self.logger.info(status) if status[0]: for option in status[1]: if option[0] == 'priority': tool_dict[container_name]['labels']['vent.priority'] = option[1] # only start tools that have been built if sections[section]['built'] != 'yes': del tool_dict[container_name] # check and update links, volumes_from, network_mode for container in tool_dict.keys(): if 'links' in tool_dict[container]: for link in tool_dict[container]['links']: for c in tool_dict.keys(): if 'tmp_name' in tool_dict[c] and tool_dict[c]['tmp_name'] == link: tool_dict[container]['links'][tool_dict[c]['name']] = tool_dict[container]['links'].pop(link) if 'volumes_from' in tool_dict[container]: tmp_volumes_from = tool_dict[container]['volumes_from'] tool_dict[container]['volumes_from'] = [] for volumes_from in list(tmp_volumes_from): for c in tool_dict.keys(): if 'tmp_name' in tool_dict[c] and tool_dict[c]['tmp_name'] == volumes_from: tool_dict[container]['volumes_from'].append(tool_dict[c]['name']) tmp_volumes_from.remove(volumes_from) tool_dict[container]['volumes_from'] += tmp_volumes_from if 'network_mode' in tool_dict[container]: if tool_dict[container]['network_mode'].startswith('container:'): network_c_name = tool_dict[container]['network_mode'].split('container:')[1] for c in tool_dict.keys(): if 'tmp_name' in tool_dict[c] and tool_dict[c]['tmp_name'] == network_c_name: tool_dict[container]['network_mode'] = 'container:'+tool_dict[c]['name'] # remove tmp_names for c in tool_dict.keys(): if 'tmp_name' in tool_dict[c]: del tool_dict[c]['tmp_name'] # remove containers that shouldn't be started for c in tool_dict.keys(): if 'start' in tool_dict[c] and not tool_dict[c]['start']: del tool_dict[c] except Exception as e: self.logger.error("prep_start failed with error: "+str(e)) status = (False, e) status = (True, tool_dict) self.logger.info("Status of prep_start: "+str(status)) self.logger.info("Finished: prep_start") return status