def Checkout(path, branch='master', version='HEAD', **kargs): status = (True, None) path_dirs = PathDirs(**kargs) status = path_dirs.apply_path(path) git_stdout = '' if status[0]: try: git_stdout += check_output(shlex.split('git checkout ' + branch), stderr=STDOUT, close_fds=True).decode('utf-8') git_stdout += check_output(shlex.split('git pull origin ' + version), stderr=STDOUT, close_fds=True).decode('utf-8') if version: git_stdout += check_output(shlex.split('git reset --hard ' + version), stderr=STDOUT, close_fds=True).decode('utf-8') chdir(status[1]) except Exception as e: # pragma: no cover if str(e).endswith('exit status 128.'): status = (True, 'Repo has already been checked out.') else: logger.error('Checkout failed with error: {0} {1}'.format( str(e), git_stdout)) status = (False, str(e)) return status
def __init__(self, *args, **kwargs): self.d_client = docker.from_env() self.path_dirs = PathDirs(**kwargs) self.manifest = join(self.path_dirs.meta_dir, 'plugin_manifest.cfg') self.vent_config = self.path_dirs.cfg_file self.startup_file = self.path_dirs.startup_file self.logger = Logger(__name__) self._auto_install()
def run_menu(test_input): """ Actually run the menu and process any input """ # initialize tutorial paths = PathDirs() first_time = paths.ensure_file(paths.init_file) assert first_time[0] == True npyscreen.TEST_SETTINGS['TEST_INPUT'] = test_input A = VentApp() try: A.run(fork=False) except npyscreen.ExhaustedTestInput as e: pass
def Dependencies(tools): """ Takes in a list of tools that are being updated and returns any tools that depend on linking to them """ dependencies = [] if tools: path_dirs = PathDirs() man = Template(join(path_dirs.meta_dir, 'plugin_manifest.cfg')) for section in man.sections()[1]: # don't worry about dealing with tool if it's not running running = man.option(section, 'running') if not running[0] or running[1] != 'yes': continue t_name = man.option(section, 'name')[1] t_branch = man.option(section, 'branch')[1] t_version = man.option(section, 'version')[1] t_identifier = {'name': t_name, 'branch': t_branch, 'version': t_version} options = man.options(section)[1] if 'docker' in options: d_settings = json.loads(man.option(section, 'docker')[1]) if 'links' in d_settings: for link in json.loads(d_settings['links']): if link in tools: dependencies.append(t_identifier) return dependencies
def __init__(self, **kargs): self.path_dirs = PathDirs(**kargs) self.manifest = join(self.path_dirs.meta_dir, 'plugin_manifest.cfg') self.p_helper = PluginHelper(**kargs) self.d_client = docker.from_env() self.logger = Logger(__name__) self.plugin_config_file = self.path_dirs.plugin_config_file
def Tools(**kargs): """ Get tools that exist in the manifest """ path_dirs = PathDirs(**kargs) manifest = os.path.join(path_dirs.meta_dir, "plugin_manifest.cfg") template = Template(template=manifest) tools = template.sections() return tools[1]
def __init__(self, version='HEAD', branch='master', user=None, pw=None, *args, **kwargs): self.version = version self.branch = branch self.user = user self.pw = pw self.d_client = docker.from_env() self.path_dirs = PathDirs(**kwargs) self.path_dirs.host_config() self.manifest = join(self.path_dirs.meta_dir, 'plugin_manifest.cfg') self.logger = Logger(__name__)
def DropLocation(): """ Get the directory that file drop is watching """ template = Template(template=PathDirs().cfg_file) drop_loc = template.option('main', 'files')[1] drop_loc = expanduser(drop_loc) drop_loc = abspath(drop_loc) return (True, drop_loc)
def test_file_queue(): """ Tests simulation of new file """ path_dirs = PathDirs() images = file_watch.file_queue('/tmp/foo') assert images[0] == False images = file_watch.file_queue('host_/tmp/foo', template_path=path_dirs.base_dir) assert images[0] == True assert type(images[1]) == list
def Core(branch="master", **kargs): """ Get the normal core tools, and the currently installed/built/running ones, including custom core services """ # !! TODO this might need to store namespaces/branches/versions core = {'built':[], 'running':[], 'installed':[], 'normal':[]} # get normal core tools plugins = Plugin(plugins_dir=".internals/plugins") status, cwd = plugins.clone('https://github.com/cyberreboot/vent') if status: plugins.version = 'HEAD' plugins.branch = branch response = plugins.checkout() matches = plugins._available_tools(groups='core') for match in matches: core['normal'].append(match[0].split('/')[-1]) else: core['normal'] = 'failed' # get core tools that have been installed path_dirs = PathDirs(**kargs) manifest = os.path.join(path_dirs.meta_dir, "plugin_manifest.cfg") template = Template(template=manifest) tools = template.sections() if tools[0]: for tool in tools[1]: groups = template.option(tool, "groups") if groups[0] and "core" in groups[1]: name = template.option(tool, "name") if name[0]: core['installed'].append(name[1]) # get core tools that have been built and/or are running try: d_client = docker.from_env() images = d_client.images.list() for image in images: try: if "vent.groups" in image.attrs['Labels'] and 'core' in image.attrs['Labels']['vent.groups']: if 'vent.name' in image.attrs['Labels']: core['built'].append(image.attrs['Labels']['vent.name']) except Exception as err: # pragma: no cover pass containers = d_client.containers.list() for container in containers: try: if "vent.groups" in container.attrs['Config']['Labels'] and 'core' in container.attrs['Config']['Labels']['vent.groups']: if 'vent.name' in container.attrs['Config']['Labels']: core['running'].append(container.attrs['Config']['Labels']['vent.name']) except Exception as err: # pragma: no cover pass except Exception as e: # pragma: no cover pass return core
def Checkout(path, branch='master', version='HEAD', **kargs): status = (True, None) path_dirs = PathDirs(**kargs) status = path_dirs.apply_path(path) if status[0]: try: check_output(shlex.split('git checkout ' + branch), stderr=STDOUT, close_fds=True).decode('utf-8') check_output(shlex.split('git pull origin ' + version), stderr=STDOUT, close_fds=True).decode('utf-8') if version: check_output(shlex.split('git reset --hard ' + version), stderr=STDOUT, close_fds=True).decode('utf-8') chdir(status[1]) except Exception as e: # pragma: no cover logger.error( 'Checkout failed with error: {0}'.format(str(e))) status = (False, str(e)) return status
def test_jobs(): """ Test the jobs function """ jobs = Jobs() assert isinstance(jobs, tuple) path = PathDirs() status = path.host_config() assert isinstance(status, tuple) assert status[0] # run test job with open('/opt/vent_files/foo.matrix', 'w') as f: f.write('24,23\n10,22') pcap = 'https://s3.amazonaws.com/tcpreplay-pcap-files/test.pcap' r = requests.get(pcap, stream=True) if r.status_code == 200: with open('/opt/vent_files/foo.pcap', 'wb') as f: r.raw.decode_content = True shutil.copyfileobj(r.raw, f) services = Services(True) assert isinstance(services, list) jobs = Jobs() assert isinstance(jobs, tuple)
def test_jobs(): """ Test the jobs function """ jobs = Jobs() assert isinstance(jobs, tuple) path = PathDirs() status = path.host_config() assert isinstance(status, tuple) assert status[0] m_helper = MenuHelper() status = m_helper.cores('install') assert isinstance(status, tuple) assert status[0] status = m_helper.cores('build') assert isinstance(status, tuple) assert status[0] status = m_helper.cores('start') assert isinstance(status, tuple) assert status[0] status = m_helper.api_action.add( 'https://github.com/cyberreboot/vent-plugins', tools=[('tcpdump_hex_parser', ''), ('gpu_example', '')]) assert isinstance(status, tuple) assert status[0] # run test job with open('/opt/vent_files/foo.matrix', 'w') as f: f.write('24,23\n10,22') pcap = 'https://s3.amazonaws.com/tcpreplay-pcap-files/test.pcap' r = requests.get(pcap, stream=True) if r.status_code == 200: with open('/opt/vent_files/foo.pcap', 'wb') as f: r.raw.decode_content = True shutil.copyfileobj(r.raw, f) services = Services(True) assert isinstance(services, list) jobs = Jobs() assert isinstance(jobs, tuple)
def Logger(name, **kargs): """ Create and return logger """ path_dirs = PathDirs(**kargs) logging.captureWarnings(True) logger = logging.getLogger(name) logger.setLevel(logging.INFO) handler = logging.handlers.WatchedFileHandler( os.path.join(path_dirs.meta_dir, "vent.log")) handler.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - ' '%(message)s') handler.setFormatter(formatter) if not len(logger.handlers): logger.addHandler(handler) return logger
def test_file_queue(): """ Tests simulation of new file """ path_dirs = PathDirs() images = file_queue('/tmp/foo') assert not images[0] images = file_queue('host_/tmp/foo', template_path=path_dirs.base_dir, r_host='localhost') assert isinstance(images, tuple) assert images[0] assert isinstance(images[1], list) images = file_queue('host_/tmp/foo.matrix', template_path=path_dirs.base_dir, r_host='localhost') assert isinstance(images, tuple) assert images[0] assert isinstance(images[1], list)
def test_file_queue(): """ Tests simulation of new file """ path_dirs = PathDirs() images = watch.file_queue('/tmp/foo') assert not images[0] images = watch.file_queue('host_/tmp/foo', template_path=path_dirs.base_dir, r_host="localhost") assert isinstance(images, tuple) assert images[0] assert isinstance(images[1], list) instance = Action() status = instance.add('https://github.com/cyberreboot/vent-plugins', branch='master', tools=[('gpu_example', '')], build=True) assert isinstance(status, tuple) assert status[0] images = watch.file_queue('host_/tmp/foo.matrix', template_path=path_dirs.base_dir, r_host="localhost") assert isinstance(images, tuple) assert images[0] assert isinstance(images[1], list)
def Services(core, vent=True, external=False, **kargs): """ Get services that have exposed ports, expects param core to be True or False based on which type of services to return, by default limit to vent containers and processes not running externally, if not limited by vent containers, then core is ignored. """ services = [] path_dirs = PathDirs(**kargs) template = Template(template=path_dirs.cfg_file) services_uri = template.option('main', 'services_uri') try: # look for internal services if not external: d_client = docker.from_env() if vent: c_filter = {'label': 'vent'} containers = d_client.containers.list(filters=c_filter) else: containers = d_client.containers.list() for c in containers: uris = {} name = None if vent and 'vent.name' in c.attrs['Config']['Labels']: if ((core and 'vent.groups' in c.attrs['Config']['Labels'] and 'core' in c.attrs['Config']['Labels']['vent.groups']) or (not core and 'vent.groups' in c.attrs['Config']['Labels'] and 'core' not in c.attrs['Config']['Labels']['vent.groups'])): name = c.attrs['Config']['Labels']['vent.name'] if name == '': name = c.attrs['Config']['Labels']['vent.namespace'].split( '/')[1] for label in c.attrs['Config']['Labels']: if label.startswith('uri'): try: val = int(label[-1]) if val not in uris: uris[val] = {} uris[val][label[:-1] ] = c.attrs['Config']['Labels'][label] except Exception as e: # pragma: no cover logger.error('Malformed services section' ' in the template file ' + str(e)) else: name = c.name if name and 'vent.repo' in c.attrs['Config']['Labels']: name = c.attrs['Config']['Labels']['vent.repo'].split( '/')[-1] + ': ' + name ports = c.attrs['NetworkSettings']['Ports'] p = [] port_num = 1 for port in ports: if ports[port]: try: service_str = '' if 'uri_prefix' in uris[port_num]: service_str += uris[port_num]['uri_prefix'] host = ports[port][0]['HostIp'] if services_uri[0] and host == '0.0.0.0': host = services_uri[1] service_str += host + ':' service_str += ports[port][0]['HostPort'] if 'uri_postfix' in uris[port_num]: service_str += uris[port_num]['uri_postfix'] uri_creds = '' if 'uri_user' in uris[port_num]: uri_creds += ' user:'******'uri_user'] if 'uri_pw' in uris[port_num]: uri_creds += ' pw:' uri_creds += uris[port_num]['uri_pw'] if uri_creds: service_str += ' - (' + uri_creds + ' )' p.append(service_str) except Exception as e: # pragma: no cover logger.info('No services defined for ' + str(name) + ' with exposed port ' + str(port_num) + ' because: ' + str(e)) port_num += 1 if p and name: services.append((name, p)) logger.info(services) # look for external services else: ext_tools = template.section('external-services')[1] for ext_tool in ext_tools: try: name = ext_tool[0].lower() p = [] settings_dict = json.loads(ext_tool[1]) if ('locally_active' in settings_dict and settings_dict['locally_active'] == 'no'): # default protocol to display will be http protocol = 'http' ip_address = '' port = '' for setting in settings_dict: if setting == 'ip_address': ip_address = settings_dict[setting] if setting == 'port': port = settings_dict[setting] if setting == 'protocol': protocol = settings_dict[setting] p.append(protocol + '://' + ip_address + ':' + port) if p and name: services.append((name, p)) except Exception: # pragma: no cover p = None except Exception as e: # pragma: no cover logger.error('Could not get services ' + str(e)) return services
def __init__(self, manifest, *args, **kwargs): self.path_dirs = PathDirs(**kwargs) self.manifest = manifest self.d_client = docker.from_env() self.logger = Logger(__name__)
def Jobs(): """ Get the number of jobs that are running and finished, and the number of total tools running and finished for those jobs """ jobs = [0, 0, 0, 0] # get running jobs try: d_client = docker.from_env() c = d_client.containers.list(all=False, filters={'label': 'vent-plugin'}) files = [] for container in c: jobs[1] += 1 if 'file' in container.attrs['Config']['Labels']: if container.attrs['Config']['Labels']['file'] not in files: files.append(container.attrs['Config']['Labels']['file']) jobs[0] = len(files) except Exception as e: # pragma: no cover logger.error('Could not get running jobs ' + str(e)) # get finished jobs try: d_client = docker.from_env() c = d_client.containers.list(all=True, filters={'label': 'vent-plugin', 'status': 'exited'}) file_names = [] tool_names = [] finished_jobs = [] path_dirs = PathDirs() manifest = join(path_dirs.meta_dir, 'status.json') if exists(manifest): file_status = 'a' else: file_status = 'w' # get a list of past jobs' file names if status.json exists if file_status == 'a': with open(manifest, 'r') as infile: for line in infile: finished_jobs.append(json.loads(line)) # get a list of file names so we can check against each container file_names = [d['FileName'] for d in finished_jobs] # multiple tools can run on 1 file. Use a tuple to status check tool_names = [(d['FileName'], d['VentPlugin']) for d in finished_jobs] for container in c: jobs[3] += 1 if 'file' in container.attrs['Config']['Labels']: # make sure the file name and the tool tup exists because # multiple tools can run on 1 file. if (container.attrs['Config']['Labels']['file'], container.attrs['Config']['Labels']['vent.name']) not in \ tool_names: # TODO figure out a nicer way of getting desired values # from containers.attrs. new_file = {} new_file['FileName'] = \ container.attrs['Config']['Labels']['file'] new_file['VentPlugin'] = \ container.attrs['Config']['Labels']['vent.name'] new_file['StartedAt'] = \ container.attrs['State']['StartedAt'] new_file['FinishedAt'] = \ container.attrs['State']['FinishedAt'] new_file['ID'] = \ container.attrs['Id'][:12] # create/append a json file with all wanted information with open(manifest, file_status) as outfile: json.dump(new_file, outfile) outfile.write('\n') # delete any containers with 'vent-plugin' in the groups if 'vent-plugin' in container.attrs['Config']['Labels']: container.remove() # add extra one to account for file that just finished if the file was # just created since file_names is processed near the beginning if file_status == 'w' and len(file_names) == 1: jobs[2] = len(set(file_names)) + 1 else: jobs[2] = len(set(file_names)) jobs[3] = jobs[3] - jobs[1] except Exception as e: # pragma: no cover logger.error('Could not get finished jobs ' + str(e)) return tuple(jobs)
def GpuUsage(**kargs): """ Get the current GPU usage of available GPUs """ usage = (False, None) gpu_status = {'vent_usage': {'dedicated': [], 'mem_mb': {}}} path_dirs = PathDirs(**kargs) path_dirs.host_config() template = Template(template=path_dirs.cfg_file) # get running jobs using gpus try: d_client = docker.from_env() c = d_client.containers.list(all=False, filters={'label': 'vent-plugin'}) for container in c: if ('vent.gpu' in container.attrs['Config']['Labels'] and container.attrs['Config']['Labels']['vent.gpu'] == 'yes'): device = container.attrs['Config']['Labels']['vent.gpu.device'] if ('vent.gpu.dedicated' in container.attrs['Config']['Labels'] and container.attrs['Config']['Labels']['vent.gpu.dedicated'] == 'yes'): gpu_status['vent_usage']['dedicated'].append(device) elif 'vent.gpu.mem_mb' in container.attrs['Config']['Labels']: if device not in gpu_status['vent_usage']['mem_mb']: gpu_status['vent_usage']['mem_mb'][device] = 0 gpu_status['vent_usage']['mem_mb'][device] += int( container.attrs['Config']['Labels']['vent.gpu.mem_mb']) except Exception as e: # pragma: no cover logger.error('Could not get running jobs ' + str(e)) port = '3476' # default docker gateway host = '172.17.0.1' result = template.option('nvidia-docker-plugin', 'port') if result[0]: port = result[1] result = template.option('nvidia-docker-plugin', 'host') if result[0]: host = result[1] else: try: # now just requires ip, ifconfig route = check_output(('ip', 'route')).decode('utf-8').split('\n') default = '' # grab the default network device. for device in route: if 'default' in device: default = device.split()[4] break # grab the IP address for the default device ip_addr = check_output(('ifconfig', default)).decode('utf-8') ip_addr = ip_addr.split('\n')[1].split()[1] host = ip_addr except Exception as e: # pragma: no cover logger.error('Something with the ip addresses' 'went wrong ' + str(e)) # have to get the info separately to determine how much memory is availabe nd_url = 'http://' + host + ':' + port + '/v1.0/gpu/info/json' try: r = requests.get(nd_url) if r.status_code == 200: status = r.json() for i, device in enumerate(status['Devices']): gm = int(round(math.log(int(device['Memory']['Global']), 2))) gpu_status[i] = {'global_memory': 2**gm, 'cores': device['Cores']} else: usage = (False, 'Unable to get GPU usage request error code: ' + str(r.status_code)) except Exception as e: # pragma: no cover usage = (False, 'Error: ' + str(e)) # get actual status of each gpu nd_url = 'http://' + host + ':' + port + '/v1.0/gpu/status/json' try: r = requests.get(nd_url) if r.status_code == 200: status = r.json() for i, device in enumerate(status['Devices']): if i not in gpu_status: gpu_status[i] = {} gpu_status[i]['utilization'] = device['Utilization'] gpu_status[i]['memory'] = device['Memory'] gpu_status[i]['processes'] = device['Processes'] usage = (True, gpu_status) else: usage = (False, 'Unable to get GPU usage request error code: ' + str(r.status_code)) except Exception as e: # pragma: no cover usage = (False, 'Error: ' + str(e)) return usage
class Plugin: """ Handle Plugins """ def __init__(self, **kargs): self.path_dirs = PathDirs(**kargs) self.manifest = os.path.join(self.path_dirs.meta_dir, "plugin_manifest.cfg") self.d_client = docker.from_env() self.logger = Logger(__name__) def apply_path(self, repo): """ Set path to where the repo is and return original path """ self.logger.info("Starting: apply_path") self.logger.info("repo given: " + str(repo)) status = (True, None) try: # rewrite repo for consistency if repo.endswith(".git"): repo = repo.split(".git")[0] # get org and repo name and path repo will be cloned to org, name = repo.split("/")[-2:] self.path = os.path.join(self.path_dirs.plugins_dir, org, name) self.logger.info("cloning to path: " + str(self.path)) # save current path cwd = os.getcwd() # set to new repo path os.chdir(self.path) status = (True, cwd) except Exception as e: self.logger.error("apply_path failed with error: " + str(e)) status = (False, e) self.logger.info("Status of apply_path: " + str(status)) self.logger.info("Finished: apply_path") return status def repo_branches(self, repo): """ Get the branches of a repository """ self.logger.info("Starting: repo_branches") self.logger.info("repo given: " + str(repo)) status = (True, None) branches = [] try: # switch to directory where repo will be cloned to status = self.apply_path(repo) if status[0]: cwd = status[1] else: self.logger.info( "apply_path failed. Exiting repo_branches with status " + str(status)) return status junk = subprocess.check_output(shlex.split("git pull --all"), stderr=subprocess.STDOUT, close_fds=True) branch_output = subprocess.check_output( shlex.split("git branch -a"), stderr=subprocess.STDOUT, close_fds=True) branch_output = branch_output.split("\n") for branch in branch_output: b = branch.strip() if b.startswith('*'): b = b[2:] if "/" in b: branches.append(b.rsplit('/', 1)[1]) elif b: branches.append(b) branches = list(set(branches)) self.logger.info("branches found: " + str(branches)) for branch in branches: try: junk = subprocess.check_output( shlex.split("git checkout " + branch), stderr=subprocess.STDOUT, close_fds=True) except Exception as e: # pragma: no cover self.logger.error("repo_branches failed with error: " + str(e) + " on branch: " + str(branch)) status = (False, e) self.logger.info("Exiting repo_branches with status: " + str(status)) return status try: os.chdir(cwd) except Exception as e: # pragma: no cover self.logger.error("unable to change directory to: " + str(cwd) + "because: " + str(e)) status = (True, branches) except Exception as e: self.logger.error("repo_branches failed with error: " + str(e)) status = (False, e) self.logger.info("Status of repo_branches: " + str(status)) self.logger.info("Finished: repo_branches") return status def repo_commits(self, repo): """ Get the commit IDs for all of the branches of a repository """ self.logger.info("Starting: repo_commits") self.logger.info("repo given: " + str(repo)) status = (True, None) commits = [] try: status = self.apply_path(repo) # switch to directory where repo will be cloned to if status[0]: cwd = status[1] else: self.logger.info( "apply_path failed. Exiting repo_commits with status: " + str(status)) return status status = self.repo_branches(repo) if status[0]: branches = status[1] for branch in branches: try: branch_output = subprocess.check_output( shlex.split("git rev-list " + branch), stderr=subprocess.STDOUT, close_fds=True) branch_output = ['HEAD' ] + branch_output.split("\n")[:-1] commits.append((branch, branch_output)) except Exception as e: # pragma: no cover self.logger.error("repo_commits failed with error: " + str(e) + " on branch: " + str(branch)) status = (False, e) self.logger.info("Exiting repo_commits with status: " + str(status)) return status else: self.logger.info( "repo_branches failed. Exiting repo_commits with status: " + str(status)) return status try: os.chdir(cwd) except Exception as e: # pragma: no cover self.logger.error("unable to change directory to: " + str(cwd) + " because: " + str(e)) status = (True, commits) except Exception as e: self.logger.error("repo_commits failed with error: " + str(e)) status = (False, e) self.logger.info("Status of repo_commits: " + str(status)) self.logger.info("Finished: repo_commits") return status def repo_tools(self, repo, branch, version): """ Get available tools for a repository branch at a version """ self.logger.info("Starting: repo_tools") self.logger.info("repo given: " + str(repo)) self.logger.info("branch given: " + str(branch)) self.logger.info("version given: " + str(version)) status = (True, None) try: tools = [] status = self.apply_path(repo) # switch to directory where repo will be cloned to if status[0]: cwd = status[1] else: self.logger.info( "apply_path failed. Exiting repo_tools with status: " + str(status)) return status self.branch = branch self.version = version status = self.checkout() if status[0]: tools = self._available_tools() else: self.logger.info( "checkout failed. Exiting repo_tools with status: " + str(status)) return status try: os.chdir(cwd) except Exception as e: # pragma: no cover self.logger.error("unable to change directory to: " + str(cwd) + " because: " + str(e)) status = (True, tools) except Exception as e: self.logger.error("repo_tools failed with error: " + str(e)) status = (False, e) self.logger.info("Status of repo_tools: " + str(status)) self.logger.info("Finished: repo_tools") return status def clone(self, repo, user=None, pw=None): """ Clone the repository """ self.logger.info("Starting: clone") self.logger.info("repo given: " + str(repo)) self.logger.info("user given: " + str(user)) status = (True, None) try: self.org = None self.name = None self.repo = repo # save current path cwd = os.getcwd() self.logger.info("current working directory: " + str(cwd)) # rewrite repo for consistency if self.repo.endswith(".git"): self.repo = self.repo.split(".git")[0] # get org and repo name and path repo will be cloned to self.org, self.name = self.repo.split("/")[-2:] self.logger.info("org name found: " + str(self.org)) self.logger.info("repo name found: " + str(self.name)) self.path = os.path.join(self.path_dirs.plugins_dir, self.org, self.name) self.logger.info("path to clone to: " + str(self.path)) # check if the directory exists, if so return now status = self.path_dirs.ensure_dir(self.path) if not status[0]: self.logger.info( "ensure_dir failed. Exiting clone with status: " + str(status)) return status # set to new repo path os.chdir(self.path) # if path already exists, try git checkout to update if status[0] and status[1] == 'exists': try: response = subprocess.check_output( shlex.split("git -C " + self.path + " rev-parse"), stderr=subprocess.STDOUT, close_fds=True) self.logger.info("path already exists: " + str(self.path)) status = (True, cwd) self.logger.info("Status of clone: " + str(status)) self.logger.info("Finished: clone") return status except Exception as e: # pragma: no cover self.logger.error("unable to checkout: " + str(path) + " because: " + str(e)) status = (False, e) self.logger.info("Exiting clone with status: " + str(status)) return status # ensure cloning still works even if ssl is broken...probably should be improved response = subprocess.check_output( shlex.split("git config --global http.sslVerify false"), stderr=subprocess.STDOUT, close_fds=True) # check if user and pw were supplied, typically for private repos if user and pw: # only https is supported when using user/pw repo = 'https://' + user + ':' + pw + '@' + self.repo.split( "https://")[-1] # clone repo and build tools response = subprocess.check_output( shlex.split("git clone --recursive " + repo + " ."), stderr=subprocess.STDOUT, close_fds=True) status = (True, cwd) except Exception as e: self.logger.error("clone failed with error: " + str(e)) status = (False, e) self.logger.info("Status of clone: " + str(status)) self.logger.info("Finished: clone") return status def add(self, repo, tools=None, overrides=None, version="HEAD", branch="master", build=True, user=None, pw=None, groups=None, version_alias=None, wild=None, remove_old=True, disable_old=True, limit_groups=None): """ Adds a plugin of tool(s) tools is a list of tuples, where the pair is a tool name (path to Dockerfile) and version tools are for explicitly limiting which tools and versions (if version in tuple is '', then defaults to version) overrides is a list of tuples, where the pair is a tool name (path to Dockerfile) and a version overrides are for explicitly removing tools and overriding versions of tools (if version in tuple is '', then tool is removed, otherwise that tool is checked out at the specific version in the tuple) if tools and overrides are left as empty lists, then all tools in the repo are pulled down at the version and branch specified or defaulted to version is globally set for all tools, unless overridden in tools or overrides branch is globally set for all tools build is a boolean of whether or not to build the tools now user is the username for a private repo if needed pw is the password to go along with the username for a private repo groups is globally set for all tools version_alias is globally set for all tools and is a mapping from a friendly version tag to the real version commit ID wild lets you specify individual overrides for additional values in the tuple of tools or overrides. wild is a list containing one or more of the following: branch, build, groups, version_alias the order of the items in the wild list will expect values to be tacked on in the same order to the tuple for tools and overrides in additional to the tool name and version remove_old lets you specify whether or not to remove previously found tools that match to ones being added currently (note does not stop currently running instances of the older version) disable_old lets you specify whether or not to disable previously found tools that match to ones being added currently (note does not stop currently running instances of the older version) limit_groups is a list of groups to build tools for that match group names in vent.template of each tool if exists Examples: repo=fe (get all tools from repo 'fe' at version 'HEAD' on branch 'master') repo=foo, version="3d1f", branch="foo" (get all tools from repo 'foo' at verion '3d1f' on branch 'foo') repo=foo, tools=[('bar', ''), ('baz', '1d32')] (get only 'bar' from repo 'foo' at version 'HEAD' on branch 'master' and 'baz' from repo 'foo' at version '1d32' on branch 'master', ignore all other tools in repo 'foo') repo=foo overrides=[('baz/bar', ''), ('.', '1c4e')], version='4fad' (get all tools from repo 'foo' at verion '4fad' on branch 'master' except 'baz/bar' and for tool '.' get version '1c4e') repo=foo tools=[('bar', '1a2d')], overrides=[('baz', 'f2a1')] (not a particularly useful example, but get 'bar' from 'foo' at version '1a2d' and get 'baz' from 'foo' at version 'f2a1' on branch 'master', ignore all other tools) """ # initialize and store class objects self.tools = tools self.overrides = overrides self.version = version self.branch = branch self.build = build self.groups = groups # TODO these need to be implemented self.version_alias = version_alias self.wild = wild self.remove_old = remove_old self.disable_old = disable_old self.limit_groups = limit_groups status = (True, None) status_code, cwd = self.clone(repo, user=user, pw=pw) status = self._build_tools(status_code) # set back to original path try: os.chdir(cwd) except Exception as e: # pragma: no cover pass return status @ErrorHandler def builder(self, template, match_path, image_name, section, build=None, branch=None, version=None): """ Build tools """ self.logger.info("Starting: builder") self.logger.info("install path: " + str(match_path)) self.logger.info("image name: " + str(image_name)) self.logger.info("build: " + str(build)) self.logger.info("branch: " + str(branch)) self.logger.info("version: " + str(version)) if build: self.build = build elif not hasattr(self, 'build'): self.build = True if branch: self.branch = branch elif not hasattr(self, 'branch'): self.branch = 'master' if version: self.version = version elif not hasattr(self, 'version'): self.version = 'HEAD' cwd = os.getcwd() self.logger.info("current working directory: " + str(cwd)) try: os.chdir(match_path) except Exception as e: self.logger.error("unable to change to directory: " + str(match_path) + " because: " + str(e)) return None template = self._build_image(template, match_path, image_name, section) try: os.chdir(cwd) except Exception as e: # pragma: no cover self.logger.error("unable to change to directory: " + str(cwd) + " because: " + str(e)) self.logger.info("template of builder: " + str(template)) self.logger.info("Finished: builder") return template def _build_tools(self, status): """ Create list of tools, paths, and versions to be built and sends them to build_manifest """ response = (True, None) # !! TODO implement features: wild, remove_old, disable_old, limit_groups # check result of clone, ensure successful or that it already exists if status: response = self.checkout() if response[0]: matches = [] if self.tools is None and self.overrides is None: # get all tools matches = self._available_tools() elif self.tools is None: # there's only something in overrides # grab all the tools then apply overrides matches = self._available_tools() # !! TODO apply overrides to matches elif self.overrides is None: # there's only something in tools # only grab the tools specified matches = self.get_tool_matches() else: # both tools and overrides were specified # grab only the tools specified, with the overrides applied orig_matches = self.get_tool_matches() matches = orig_matches for override in self.overrides: override_t = None if override[0] == '.': override_t = ('', override[1]) else: override_t = override for match in orig_matches: if override_t[0] == match[0]: matches.remove(match) matches.append(override_t) if len(matches) > 0: self._build_manifest(matches) else: response = (False, status) return response def get_tool_matches(self): """ Get the tools paths and versions that were specified by self.tools and self.version """ matches = [] if not hasattr(self, 'tools'): self.tools = [] if not hasattr(self, 'version'): self.version = 'HEAD' for tool in self.tools: match_version = self.version if tool[1] != '': match_version = tool[1] match = '' if tool[0].endswith('/'): match = tool[0][:-1] elif tool[0] != '.': match = tool[0] if not match.startswith('/') and match != '': match = '/' + match matches.append((match, match_version)) return matches def _build_manifest(self, matches): """ Builds and writes the manifest for the tools being added """ # !! TODO check for pre-existing that conflict with request and disable and/or remove image for match in matches: template = Template(template=self.manifest) # !! TODO check for special settings here first for the specific match self.version = match[1] response = self.checkout() if response[0]: section = self.org + ":" + self.name + ":" + match[ 0] + ":" + self.branch + ":" + self.version match_path = self.path + match[0] image_name = self.org + "-" + self.name + "-" if match[0] != '': # if tool is in a subdir, add that to the name of the image image_name += '-'.join(match[0].split('/')[1:]) + "-" image_name += self.branch + ":" + self.version # check if the section already exists exists, options = template.section(section) previous_commit = None previous_commits = None head = False if exists: for option in options: # TODO check if tool name but a different version exists - then disable/remove if set if option[0] == 'version' and option[1] == 'HEAD': head = True if option[0] == 'built' and option[1] == 'yes': # !! TODO remove pre-existing image pass if option[0] == 'commit_id': previous_commit = option[1] if option[0] == 'previous_versions': previous_commits = option[1] # !! TODO # check if section should be removed from config - i.e. all tools, # but new commit removed one that was in a previous commit # set template section and options for tool at version and branch template.add_section(section) template.set_option(section, "name", match[0].split('/')[-1]) template.set_option(section, "namespace", self.org + '/' + self.name) template.set_option(section, "path", match_path) template.set_option(section, "repo", self.repo) template.set_option(section, "enabled", "yes") template.set_option(section, "branch", self.branch) template.set_option(section, "version", self.version) template.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") template.set_option(section, "image_name", image_name) vent_template = Template( template=os.path.join(match_path, 'vent.template')) vent_status, response = vent_template.option("info", "name") if vent_status: template.set_option(section, "link_name", response) else: template.set_option(section, "link_name", match[0].split('/')[-1]) commit_id = None if self.version == 'HEAD': os.chdir(match_path) commit_id = subprocess.check_output( shlex.split("git rev-parse --short HEAD"), stderr=subprocess.STDOUT, close_fds=True).strip() template.set_option(section, "commit_id", commit_id) if head: # no need to store previous commits if not HEAD, since # the version will always be the same commit ID if previous_commit and previous_commit != commit_id: if previous_commits and previous_commit not in previous_commits: previous_commits = previous_commit + ',' + previous_commits elif not previous_commits: previous_commits = previous_commit if previous_commits and previous_commits != commit_id: template.set_option(section, "previous_versions", previous_commits) if self.version_alias: template.set_option(section, "version_alias", self.version_alias) if self.groups: template.set_option(section, "groups", self.groups) else: vent_template = os.path.join(match_path, 'vent.template') if os.path.exists(vent_template): v_template = Template(template=vent_template) groups = v_template.option("info", "groups") if groups[0]: template.set_option(section, "groups", groups[1]) template = self._build_image(template, match_path, image_name, section) # write out configuration to the manifest file template.write_config() # reset to repo directory os.chdir(self.path) return def _build_image(self, template, match_path, image_name, section): """ Build docker images and store results in template """ # !! TODO return status of whether it built successfully or not if self.build: try: os.chdir(match_path) # currently can't use docker-py because it doesn't support labels on images yet name = template.option(section, "name") groups = template.option(section, "groups") if groups[1] == "" or not groups[0]: groups = (True, "none") if not name[0]: name = (True, image_name) # pull if '/' in image_name, fallback to build pull = False if '/' in image_name: try: self.logger.info("Trying to pull " + image_name) output = subprocess.check_output( shlex.split("docker pull " + image_name), stderr=subprocess.STDOUT, close_fds=True) self.logger.info("Pulling " + name[1] + "\n" + str(output)) for line in output.split('\n'): if line.startswith("Digest: sha256:"): image_id = line.split( "Digest: sha256:")[1][:12] if image_id: template.set_option(section, "built", "yes") template.set_option(section, "image_id", image_id) template.set_option( section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") status = (True, "Pulled " + image_name) self.logger.info(str(status)) else: template.set_option(section, "built", "failed") template.set_option( section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") status = (False, "Failed to pull image " + str(output.split('\n')[-1])) self.logger.warning(str(status)) pull = True except Exception as e: # pragma: no cover self.logger.warning( "Failed to pull image, going to build instead: " + str(e)) if not pull: output = subprocess.check_output(shlex.split( "docker build --label vent --label vent.name=" + name[1] + " --label vent.groups=" + groups[1] + " -t " + image_name + " ."), stderr=subprocess.STDOUT, close_fds=True) self.logger.info("Building " + name[1] + "\n" + str(output)) image_id = "" for line in output.split("\n"): if line.startswith("Successfully built "): image_id = line.split( "Successfully built ")[1].strip() template.set_option(section, "built", "yes") template.set_option(section, "image_id", image_id) template.set_option( section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") except Exception as e: # pragma: no cover self.logger.error("unable to build image: " + str(image_name) + " because: " + str(e)) template.set_option(section, "built", "failed") template.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") else: template.set_option(section, "built", "no") template.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") return template def _available_tools(self, groups=None): """ Return list of possible tools in repo for the given version and branch """ matches = [] if not hasattr(self, 'path'): return matches if groups: groups = groups.split(",") for root, dirnames, filenames in os.walk(self.path): for filename in fnmatch.filter(filenames, 'Dockerfile'): # !! TODO deal with wild/etc.? if groups: try: template = Template( template=os.path.join(root, 'vent.template')) for group in groups: template_groups = template.option("info", "groups") if template_groups[0] and group in template_groups[ 1]: matches.append( (root.split(self.path)[1], self.version)) except Exception as e: # pragma: no cover pass else: matches.append((root.split(self.path)[1], self.version)) return matches def checkout(self): """ Checkout a specific version and branch of a repo """ if not hasattr(self, 'branch'): self.branch = 'master' if not hasattr(self, 'version'): self.version = 'HEAD' response = (True, None) try: status = subprocess.check_output(shlex.split("git checkout " + self.branch), stderr=subprocess.STDOUT, close_fds=True) status = subprocess.check_output(shlex.split("git pull"), stderr=subprocess.STDOUT, close_fds=True) status = subprocess.check_output(shlex.split("git reset --hard " + self.version), stderr=subprocess.STDOUT, close_fds=True) response = (True, status) except Exception as e: # pragma: no cover response = (False, os.getcwd() + str(e)) return response @staticmethod def add_image(image, tag="latest"): """ Add an image from a registry/hub rather than building from a repository """ # !! TODO return def constraint_options(self, constraint_dict, options): """ Return result of constraints and options against a template """ constraints = {} template = Template(template=self.manifest) for constraint in constraint_dict: if constraint != 'self': if constraint_dict[constraint] or constraint_dict[ constraint] == '': constraints[constraint] = constraint_dict[constraint] results = template.constrained_sections(constraints=constraints, options=options) return results, template def tools(self): """ Return list of tuples of all tools """ tools = [] template = Template(template=self.manifest) exists, sections = template.sections() if exists: for section in sections: options = { 'section': section, 'enabled': None, 'built': None, 'version': None, 'repo': None, 'branch': None, 'name': None, 'groups': None, 'image_name': None } for option in options.keys(): exists, value = template.option(section, option) if exists: options[option] = value tools.append(options) return tools def remove(self, name=None, repo=None, namespace=None, branch="master", groups=None, enabled="yes", version="HEAD", built="yes"): """ Remove tool (name) or repository, repository is the url. If no arguments are specified, all tools will be removed for the defaults. """ # initialize args = locals() status = (True, None) # get resulting dictionary of sections with options that match constraints results, template = self.constraint_options(args, []) for result in results: response, image_name = template.option(result, 'image_name') # check for container and remove container_name = image_name.replace(':', '-').replace('/', '-') try: container = self.d_client.containers.get(container_name) response = container.remove(v=True, force=True) self.logger.info(response) self.logger.info("Removing plugin container: " + container_name) except Exception as e: # pragma: no cover self.logger.warn("Unable to remove the plugin container: " + container_name + " because: " + str(e)) # check for image and remove try: response = self.d_client.images.remove(image_name) self.logger.info(response) self.logger.info("Removing plugin image: " + image_name) except Exception as e: # pragma: no cover self.logger.warn("Unable to remove the plugin image: " + image_name + " because: " + str(e)) # remove tool from the manifest status = template.del_section(result) self.logger.info("Removing plugin tool: " + result) # TODO if all tools from a repo have been removed, remove the repo template.write_config() return status def update(self, name=None, repo=None, namespace=None, branch=None, groups=None): """ Update tool (name) or repository, repository is the url. If no arguments are specified, all tools will be updated """ # initialize args = locals() status = (False, None) options = ['branch', 'groups', 'image_name'] # get resulting dictionary of sections with options that match constraints results, template = self.constraint_options(args, options) for result in results: # check for container and remove try: container_name = results['image_name'].replace(':', '-') \ .replace('/', '-') container = self.d_client.containers.get(container_name) response = container.remove(v=True, force=True) except Exception as e: # pragma: no cover pass # TODO git pull # TODO build # TODO docker pull # TODO update tool in the manifest self.logger.info("Updating plugin tool: " + result) template.write_config() return status # !! TODO name or group ? def versions(self, name, namespace=None, branch="master"): """ Return available versions of a tool """ # initialize args = locals() versions = [] options = ['version', 'previous_versions'] # get resulting dictionary of sections with options that match constraints results, _ = self.constraint_options(args, options) for result in results: version_list = [results[result]['version']] if 'previous_versions' in results[result]: version_list = version_list + ( results[result]['previous_versions']).split(',') versions.append((result, version_list)) return versions # !! TODO name or group ? def current_version(self, name, namespace=None, branch="master"): """ Return current version for a given tool """ # initialize args = locals() versions = [] options = ['version'] # get resulting dictionary of sections with options that match constraints results, _ = self.constraint_options(args, options) for result in results: versions.append((result, results[result]['version'])) return versions # !! TODO name or group ? def state(self, name, namespace=None, branch="master"): """ Return state of a tool, disabled/enabled for each version """ # initialize args = locals() states = [] options = ['enabled'] # get resulting dictionary of sections with options that match constraints results, _ = self.constraint_options(args, options) for result in results: if results[result]['enabled'] == 'yes': states.append((result, 'enabled')) else: states.append((result, 'disabled')) return states # !! TODO name or group ? def enable(self, name, namespace=None, branch="master", version="HEAD"): """ Enable tool at a specific version, default to head """ # initialize args = locals() status = (False, None) # get resulting dictionary of sections with options that match constraints results, template = self.constraint_options(args, []) for result in results: status = template.set_option(result, 'enabled', 'yes') template.write_config() return status # !! TODO name or group ? def disable(self, name, namespace=None, branch="master", version="HEAD"): """ Disable tool at a specific version, default to head """ # initialize args = locals() status = (False, None) # get resulting dictionary of sections with options that match constraints results, template = self.constraint_options(args, []) for result in results: status = template.set_option(result, 'enabled', 'no') template.write_config() return status
class Tools: def __init__(self, version='HEAD', branch='master', user=None, pw=None, *args, **kwargs): self.version = version self.branch = branch self.user = user self.pw = pw self.d_client = docker.from_env() self.path_dirs = PathDirs(**kwargs) self.path_dirs.host_config() self.manifest = join(self.path_dirs.meta_dir, 'plugin_manifest.cfg') self.logger = Logger(__name__) def new(self, tool_type, uri, tools=None, link_name=None, image_name=None, overrides=None, tag=None, registry=None, groups=None): try: # remove tools that are already installed from being added if isinstance(tools, list): i = len(tools) - 1 while i >= 0: tool = tools[i] if tool[0].find('@') >= 0: tool_name = tool[0].split('@')[-1] else: tool_name = tool[0].rsplit('/', 1)[-1] constraints = {'name': tool_name, 'repo': uri.split('.git')[0]} prev_installed, _ = Template(template=self.manifest).constrain_opts(constraints, []) # don't reinstall if prev_installed: tools.remove(tool) i -= 1 if len(tools) == 0: tools = None except Exception as e: # pragma: no cover self.logger.error('Add failed with error: {0}'.format(str(e))) return (False, str(e)) if tool_type == 'image': status = Image(self.manifest).add(uri, link_name, tag=tag, registry=registry, groups=groups) else: if tool_type == 'core': uri = 'https://github.com/cyberreboot/vent' core = True else: core = False status = Repository(self.manifest).add(uri, tools, overrides=overrides, version=self.version, image_name=image_name, branch=self.branch, user=self.user, pw=self.pw, core=core) return status def configure(self, tool): # TODO return def inventory(self, choices=None): """ Return a dictionary of the inventory items and status """ status = (True, None) if not choices: return (False, 'No choices made') try: # choices: repos, tools, images, built, running, enabled items = {'repos': [], 'tools': {}, 'images': {}, 'built': {}, 'running': {}, 'enabled': {}} tools = Template(self.manifest).list_tools() for choice in choices: for tool in tools: try: if choice == 'repos': if 'repo' in tool: if (tool['repo'] and tool['repo'] not in items[choice]): items[choice].append(tool['repo']) elif choice == 'tools': items[choice][tool['section']] = tool['name'] elif choice == 'images': # TODO also check against docker items[choice][tool['section']] = tool['image_name'] elif choice == 'built': items[choice][tool['section']] = tool['built'] elif choice == 'running': containers = Containers() status = 'not running' for container in containers: image_name = tool['image_name'] \ .rsplit(':' + tool['version'], 1)[0] image_name = image_name.replace(':', '-') image_name = image_name.replace('/', '-') self.logger.info('image_name: ' + image_name) if container[0] == image_name: status = container[1] elif container[0] == image_name + \ '-' + tool['version']: status = container[1] items[choice][tool['section']] = status elif choice == 'enabled': items[choice][tool['section']] = tool['enabled'] else: # unknown choice pass except Exception as e: # pragma: no cover self.logger.error('Unable to grab info about tool: ' + str(tool) + ' because: ' + str(e)) status = (True, items) except Exception as e: # pragma: no cover self.logger.error( 'Inventory failed with error: {0}'.format(str(e))) status = (False, str(e)) return status def remove(self, repo, name): args = locals() status = (True, None) # get resulting dict of sections with options that match constraints template = Template(template=self.manifest) results, _ = template.constrain_opts(args, []) for result in results: response, image_name = template.option(result, 'image_name') name = template.option(result, 'name')[1] try: settings_dict = json.loads(template.option(result, 'settings')[1]) instances = int(settings_dict['instances']) except Exception: instances = 1 try: # check for container and remove c_name = image_name.replace(':', '-').replace('/', '-') for i in range(1, instances + 1): container_name = c_name + str(i) if i != 1 else c_name container = self.d_client.containers.get(container_name) response = container.remove(v=True, force=True) self.logger.info( 'Removing container: {0}'.format(container_name)) except Exception as e: # pragma: no cover self.logger.warning('Unable to remove the container: ' + container_name + ' because: ' + str(e)) # check for image and remove try: response = None image_id = template.option(result, 'image_id')[1] response = self.d_client.images.remove(image_id, force=True) self.logger.info('Removing image: ' + image_name) except Exception as e: # pragma: no cover self.logger.warning('Unable to remove the image: ' + image_name + ' because: ' + str(e)) # remove tool from the manifest for i in range(1, instances + 1): res = result.rsplit(':', 2) res[0] += str(i) if i != 1 else '' res = ':'.join(res) if template.section(res)[0]: status = template.del_section(res) self.logger.info('Removing tool: ' + res) # TODO if all tools from a repo have been removed, remove the repo template.write_config() return status def start(self, repo, name, is_tool_d=False): if is_tool_d: tool_d = repo else: args = locals() del args['self'] del args['is_tool_d'] tool_d = {} tool_d.update(self._prep_start(**args)[1]) status = (True, None) try: # check start priorities (priority of groups alphabetical for now) group_orders = {} groups = [] containers_remaining = [] username = getpass.getuser() # remove tools that have the hidden label tool_d_copy = copy.deepcopy(tool_d) for container in tool_d_copy: if 'labels' in tool_d_copy[container] and 'vent.groups' in tool_d_copy[container]['labels']: groups_copy = tool_d_copy[container]['labels']['vent.groups'].split( ',') if 'hidden' in groups_copy: del tool_d[container] for container in tool_d: containers_remaining.append(container) self.logger.info( "User: '******' starting container: {1}".format(username, container)) if 'labels' in tool_d[container]: if 'vent.groups' in tool_d[container]['labels']: groups += tool_d[container]['labels']['vent.groups'].split( ',') if 'vent.priority' in tool_d[container]['labels']: priorities = tool_d[container]['labels']['vent.priority'].split( ',') container_groups = tool_d[container]['labels']['vent.groups'].split( ',') for i, priority in enumerate(priorities): if container_groups[i] not in group_orders: group_orders[container_groups[i]] = [] group_orders[container_groups[i]].append( (int(priority), container)) containers_remaining.remove(container) tool_d[container]['labels'].update( {'started-by': username} ) else: tool_d[container].update( {'labels': {'started-by': username}} ) # start containers based on priorities p_results = self._start_priority_containers(groups, group_orders, tool_d) # start the rest of the containers that didn't have any priorities r_results = self._start_remaining_containers( containers_remaining, tool_d) results = (p_results[0] + r_results[0], p_results[1] + r_results[1]) if len(results[1]) > 0: status = (False, results) else: status = (True, results) except Exception as e: # pragma: no cover self.logger.error('Start failed with error: {0}'.format(str(e))) status = (False, str(e)) return status def _prep_start(self, repo, name): args = locals() status = (True, None) try: options = ['name', 'namespace', 'built', 'groups', 'path', 'image_name', 'branch', 'repo', 'type', 'version'] vent_config = Template(template=self.path_dirs.cfg_file) manifest = Template(self.manifest) files = vent_config.option('main', 'files') files = (files[0], expanduser(files[1])) s, _ = manifest.constrain_opts(args, options) status, tool_d = self._start_sections(s, files) # look out for links to delete because they're defined externally links_to_delete = set() # get instances for each tool tool_instances = {} sections = manifest.sections()[1] for section in sections: settings = manifest.option(section, 'settings') if settings[0]: settings = json.loads(settings[1]) if 'instances' in settings: l_name = manifest.option(section, 'link_name') if l_name[0]: tool_instances[l_name[1]] = int( settings['instances']) # check and update links, volumes_from, network_mode for container in list(tool_d.keys()): if 'labels' not in tool_d[container] or 'vent.groups' not in tool_d[container]['labels'] or 'core' not in tool_d[container]['labels']['vent.groups']: tool_d[container]['remove'] = True if 'links' in tool_d[container]: for link in list(tool_d[container]['links'].keys()): # add links to external services already running if # necessary, by default configure local services too configure_local = True ext = 'external-services' if link in vent_config.options(ext)[1]: try: lconf = json.loads(vent_config.option(ext, link)[1]) if ('locally_active' not in lconf or lconf['locally_active'] == 'no'): ip_adr = lconf['ip_address'] port = lconf['port'] tool_d[container]['extra_hosts'] = {} # containers use lowercase names for # connections tool_d[container]['extra_hosts'][link.lower() ] = ip_adr # create an environment variable for container # to access port later env_variable = link.upper() + \ '_CUSTOM_PORT=' + port if 'environment' not in tool_d[container]: tool_d[container]['environment'] = [] tool_d[container]['environment'].append( env_variable) # remove the entry from links because no # longer connecting to local container links_to_delete.add(link) configure_local = False except Exception as e: # pragma: no cover self.logger.error( 'Could not load external settings because: {0}'.format(str(e))) configure_local = True status = False if configure_local: for c in list(tool_d.keys()): if ('tmp_name' in tool_d[c] and tool_d[c]['tmp_name'] == link): tool_d[container]['links'][tool_d[c]['name'] ] = tool_d[container]['links'].pop(link) if link in tool_instances and tool_instances[link] > 1: for i in range(2, tool_instances[link] + 1): tool_d[container]['links'][tool_d[c]['name'] + str( i)] = tool_d[container]['links'][tool_d[c]['name']] + str(i) if 'volumes_from' in tool_d[container]: tmp_volumes_from = tool_d[container]['volumes_from'] tool_d[container]['volumes_from'] = [] for volumes_from in list(tmp_volumes_from): for c in list(tool_d.keys()): if ('tmp_name' in tool_d[c] and tool_d[c]['tmp_name'] == volumes_from): tool_d[container]['volumes_from'].append( tool_d[c]['name']) tmp_volumes_from.remove(volumes_from) tool_d[container]['volumes_from'] += tmp_volumes_from if 'network_mode' in tool_d[container]: if tool_d[container]['network_mode'].startswith('container:'): network_c_name = tool_d[container]['network_mode'].split('container:')[ 1] for c in list(tool_d.keys()): if ('tmp_name' in tool_d[c] and tool_d[c]['tmp_name'] == network_c_name): tool_d[container]['network_mode'] = 'container:' + \ tool_d[c]['name'] # remove tmp_names for c in list(tool_d.keys()): if 'tmp_name' in tool_d[c]: del tool_d[c]['tmp_name'] # remove links section if all were externally configured for c in list(tool_d.keys()): if 'links' in tool_d[c]: for link in links_to_delete: if link in tool_d[c]['links']: del tool_d[c]['links'][link] # delete links if no more defined if not tool_d[c]['links']: del tool_d[c]['links'] # remove containers that shouldn't be started for c in list(tool_d.keys()): deleted = False if 'start' in tool_d[c] and not tool_d[c]['start']: del tool_d[c] deleted = True if not deleted: # look for tools services that are being done externally # tools are capitalized in vent.cfg, so make them lowercase # for comparison ext = 'external-services' external_tools = vent_config.section(ext)[1] name = tool_d[c]['labels']['vent.name'] for tool in external_tools: if name == tool[0].lower(): try: tool_config = json.loads(tool[1]) if ('locally_active' in tool_config and tool_config['locally_active'] == 'no'): del tool_d[c] except Exception as e: # pragma: no cover self.logger.warning('Locally running container ' + name + ' may be redundant') if status: status = (True, tool_d) else: status = (False, tool_d) except Exception as e: # pragma: no cover self.logger.error('_prep_start failed with error: '+str(e)) status = (False, e) return status def _start_sections(self, s, files): tool_d = {} status = (True, None) for section in s: # initialize needed vars c_name = s[section]['image_name'].replace(':', '-') c_name = c_name.replace('/', '-') instance_num = re.search(r'\d+$', s[section]['name']) if instance_num: c_name += instance_num.group() image_name = s[section]['image_name'] # checkout the right version and branch of the repo tool_d[c_name] = {'image': image_name, 'name': c_name} # get rid of all commented sections in various runtime # configurations manifest = Template(self.manifest) overall_dict = {} for setting in ['info', 'docker', 'gpu', 'settings', 'service']: option = manifest.option(section, setting) if option[0]: overall_dict[setting] = {} settings_dict = json.loads(option[1]) for opt in settings_dict: if not opt.startswith('#'): overall_dict[setting][opt] = settings_dict[opt] if 'docker' in overall_dict: options_dict = overall_dict['docker'] for option in options_dict: options = options_dict[option] # check for commands to evaluate if '`' in options: cmds = options.split('`') if len(cmds) > 2: i = 1 while i < len(cmds): try: cmds[i] = check_output(shlex.split(cmds[i]), stderr=STDOUT, close_fds=True).strip().decode('utf-8') except Exception as e: # pragma: no cover self.logger.error( 'unable to evaluate command specified in vent.template: ' + str(e)) i += 2 options = ''.join(cmds) # check for commands to evaluate # store options set for docker try: tool_d[c_name][option] = ast.literal_eval(options) except Exception as e: # pragma: no cover self.logger.debug( 'Unable to literal_eval: {0}'.format(str(options))) tool_d[c_name][option] = options if 'labels' not in tool_d[c_name]: tool_d[c_name]['labels'] = {} # get the service uri info if 'service' in overall_dict: try: options_dict = overall_dict['service'] for option in options_dict: tool_d[c_name]['labels'][option] = options_dict[option] except Exception as e: # pragma: no cover self.logger.error('unable to store service options for ' 'docker: ' + str(e)) # check for gpu settings if 'gpu' in overall_dict: try: options_dict = json.loads(status[1]) for option in options_dict: tool_d[c_name]['labels']['gpu.' + option] = options_dict[option] except Exception as e: # pragma: no cover self.logger.error('unable to store gpu options for ' 'docker: ' + str(e)) # get temporary name for links, etc. plugin_c = Template(template=self.manifest) status, plugin_sections = plugin_c.sections() for plugin_section in plugin_sections: status = plugin_c.option(plugin_section, 'link_name') image_status = plugin_c.option(plugin_section, 'image_name') if status[0] and image_status[0]: cont_name = image_status[1].replace(':', '-') cont_name = cont_name.replace('/', '-') if cont_name not in tool_d: tool_d[cont_name] = {'image': image_status[1], 'name': cont_name, 'start': False} tool_d[cont_name]['tmp_name'] = status[1] # add extra labels tool_d[c_name]['labels']['vent'] = Version() tool_d[c_name]['labels']['vent.namespace'] = s[section]['namespace'] tool_d[c_name]['labels']['vent.branch'] = s[section]['branch'] tool_d[c_name]['labels']['vent.version'] = s[section]['version'] tool_d[c_name]['labels']['vent.name'] = s[section]['name'] tool_d[c_name]['labels']['vent.section'] = section tool_d[c_name]['labels']['vent.repo'] = s[section]['repo'] tool_d[c_name]['labels']['vent.type'] = s[section]['type'] # check for log_config settings in external-services externally_configured = False vent_config = Template(self.path_dirs.cfg_file) for ext_tool in vent_config.section('external-services')[1]: if ext_tool[0].lower() == 'syslog': try: log_dict = json.loads(ext_tool[1]) # configure if not locally active if ('locally_active' not in log_dict or log_dict['locally_active'] == 'no'): del log_dict['locally_active'] log_config = {} log_config['type'] = 'syslog' log_config['config'] = {} ip_address = '' port = '' for option in log_dict: if option == 'ip_address': ip_address = log_dict[option] elif option == 'port': port = log_dict['port'] syslog_address = 'tcp://' + ip_address + ':' + port syslog_config = {'syslog-address': syslog_address, 'syslog-facility': 'daemon', 'tag': '{{.Name}}'} log_config['config'].update(syslog_config) externally_configured = True except Exception as e: # pragma: no cover self.logger.error('external settings for log_config' " couldn't be stored because: " + str(e)) externally_configured = False if not externally_configured: log_config = {'type': 'syslog', 'config': {'syslog-address': 'tcp://0.0.0.0:514', 'syslog-facility': 'daemon', 'tag': '{{.Name}}'}} if 'groups' in s[section]: # add labels for groups tool_d[c_name]['labels']['vent.groups'] = s[section]['groups'] # add restart=always to core containers if 'core' in s[section]['groups']: tool_d[c_name]['restart_policy'] = {'Name': 'always'} # map network names to environment variables if 'network' in s[section]['groups']: vent_config = Template(template=self.path_dirs.cfg_file) nic_mappings = vent_config.section('network-mapping') nics = '' if nic_mappings[0]: for nic in nic_mappings[1]: nics += nic[0] + ':' + nic[1] + ',' nics = nics[:-1] if nics: if 'environment' in tool_d[c_name]: tool_d[c_name]['environment'].append( 'VENT_NICS='+nics) else: tool_d[c_name]['environment'] = ['VENT_NICS='+nics] # send logs to syslog if ('syslog' not in s[section]['groups'] and 'core' in s[section]['groups']): log_config['config']['tag'] = '{{.Name}}' tool_d[c_name]['log_config'] = log_config if 'syslog' not in s[section]['groups']: tool_d[c_name]['log_config'] = log_config # mount necessary directories if 'files' in s[section]['groups']: ulimits = [] ulimits.append(docker.types.Ulimit( name='nofile', soft=1048576, hard=1048576)) tool_d[c_name]['ulimits'] = ulimits # check if running in a docker container if 'VENT_CONTAINERIZED' in environ and environ['VENT_CONTAINERIZED'] == 'true': if 'volumes_from' in tool_d[c_name]: tool_d[c_name]['volumes_from'].append( environ['HOSTNAME']) else: tool_d[c_name]['volumes_from'] = [ environ['HOSTNAME']] else: if 'volumes' in tool_d[c_name]: tool_d[c_name]['volumes'][self.path_dirs.base_dir[:-1] ] = {'bind': '/vent', 'mode': 'ro'} else: tool_d[c_name]['volumes'] = { self.path_dirs.base_dir[:-1]: {'bind': '/vent', 'mode': 'ro'}} if files[0]: if 'volumes' in tool_d[c_name]: tool_d[c_name]['volumes'][files[1]] = { 'bind': '/files', 'mode': 'rw'} else: tool_d[c_name]['volumes'] = { files[1]: {'bind': '/files', 'mode': 'rw'}} else: tool_d[c_name]['log_config'] = log_config # add label for priority if 'settings' in overall_dict: try: options_dict = overall_dict['settings'] for option in options_dict: if option == 'priority': tool_d[c_name]['labels']['vent.priority'] = options_dict[option] except Exception as e: # pragma: no cover self.logger.error('unable to store settings options ' 'for docker ' + str(e)) # only start tools that have been built if s[section]['built'] != 'yes': del tool_d[c_name] # store section information for adding info to manifest later else: tool_d[c_name]['section'] = section return status, tool_d def _start_priority_containers(self, groups, group_orders, tool_d): """ Select containers based on priorities to start """ vent_cfg = Template(self.path_dirs.cfg_file) cfg_groups = vent_cfg.option('groups', 'start_order') if cfg_groups[0]: cfg_groups = cfg_groups[1].split(',') else: cfg_groups = [] all_groups = sorted(set(groups)) s_conts = [] f_conts = [] # start tools in order of group defined in vent.cfg for group in cfg_groups: # remove from all_groups because already checked out if group in all_groups: all_groups.remove(group) if group in group_orders: for cont_t in sorted(group_orders[group]): if cont_t[1] not in s_conts: s_conts, f_conts = self._start_container(cont_t[1], tool_d, s_conts, f_conts) # start tools that haven't been specified in the vent.cfg, if any for group in all_groups: if group in group_orders: for cont_t in sorted(group_orders[group]): if cont_t[1] not in s_conts: s_conts, f_conts = self._start_container(cont_t[1], tool_d, s_conts, f_conts) return (s_conts, f_conts) def _start_remaining_containers(self, containers_remaining, tool_d): """ Select remaining containers that didn't have priorities to start """ s_containers = [] f_containers = [] for container in containers_remaining: s_containers, f_containers = self._start_container(container, tool_d, s_containers, f_containers) return (s_containers, f_containers) def _start_container(self, container, tool_d, s_containers, f_containers): """ Start container that was passed in and return status """ # use section to add info to manifest section = tool_d[container]['section'] del tool_d[container]['section'] manifest = Template(self.manifest) try: # try to start an existing container first c = self.d_client.containers.get(container) c.start() s_containers.append(container) manifest.set_option(section, 'running', 'yes') self.logger.info('started ' + str(container) + ' with ID: ' + str(c.short_id)) except Exception as err: s_containers, f_containers = self._run_container( container, tool_d, section, s_containers, f_containers) # save changes made to manifest manifest.write_config() return s_containers, f_containers def _run_container(self, container, tool_d, section, s_containers, f_containers): manifest = Template(self.manifest) try: gpu = 'gpu.enabled' failed = False if (gpu in tool_d[container]['labels'] and tool_d[container]['labels'][gpu] == 'yes'): vent_config = Template(template=self.path_dirs.cfg_file) port = '' host = '' result = vent_config.option('nvidia-docker-plugin', 'port') if result[0]: port = result[1] else: port = '3476' result = vent_config.option('nvidia-docker-plugin', 'host') if result[0]: host = result[1] else: # now just requires ip, ifconfig try: route = check_output(('ip', 'route')).decode( 'utf-8').split('\n') default = '' # grab the default network device. for device in route: if 'default' in device: default = device.split()[4] break # grab the IP address for the default device ip_addr = check_output( ('ifconfig', default)).decode('utf-8') ip_addr = ip_addr.split('\n')[1].split()[1] host = ip_addr except Exception as e: # pragma no cover self.logger.error('failed to grab ip. Ensure that \ ip and ifconfig are installed') nd_url = 'http://' + host + ':' + port + '/v1.0/docker/cli' params = {'vol': 'nvidia_driver'} r = requests.get(nd_url, params=params) if r.status_code == 200: options = r.text.split() for option in options: if option.startswith('--volume-driver='): tool_d[container]['volume_driver'] = option.split('=', 1)[ 1] elif option.startswith('--volume='): vol = option.split('=', 1)[1].split(':') if 'volumes' in tool_d[container]: if isinstance(tool_d[container]['volumes'], list): if len(vol) == 2: c_vol = vol[0] + \ ':' + vol[1] + ':rw' else: c_vol = vol[0] + ':' + \ vol[1] + ':' + vol[2] tool_d[container]['volumes'].append( c_vol) else: # Dictionary tool_d[container]['volumes'][vol[0]] = {'bind': vol[1], 'mode': vol[2]} else: tool_d[container]['volumes'] = {vol[0]: {'bind': vol[1], 'mode': vol[2]}} elif option.startswith('--device='): dev = option.split('=', 1)[1] if 'devices' in tool_d[container]: tool_d[container]['devices'].append(dev + ':' + dev + ':rwm') else: tool_d[container]['devices'] = [ dev + ':' + dev + ':rwm'] else: self.logger.error('Unable to parse ' + 'nvidia-docker option: ' + str(option)) else: failed = True f_containers.append(container) manifest.set_option(section, 'running', 'failed') self.logger.error('failed to start ' + str(container) + ' because nvidia-docker-plugin ' + 'failed with: ' + str(r.status_code)) if not failed: try: self.d_client.containers.remove(container, force=True) self.logger.info( 'removed old existing container: ' + str(container)) except Exception as e: pass cont_id = self.d_client.containers.run(detach=True, **tool_d[container]) s_containers.append(container) manifest.set_option(section, 'running', 'yes') self.logger.info('started ' + str(container) + ' with ID: ' + str(cont_id)) except Exception as e: # pragma: no cover f_containers.append(container) manifest.set_option(section, 'running', 'failed') self.logger.error('failed to start ' + str(container) + ' because: ' + str(e)) return s_containers, f_containers def stop(self, repo, name): args = locals() status = (True, None) try: # !! TODO need to account for plugin containers that have random # names, use labels perhaps options = ['name', 'namespace', 'built', 'groups', 'path', 'image_name', 'branch', 'version'] s, _ = Template(template=self.manifest).constrain_opts(args, options) for section in s: container_name = s[section]['image_name'].replace(':', '-') container_name = container_name.replace('/', '-') try: container = self.d_client.containers.get(container_name) container.stop() self.logger.info('Stopped {0}'.format(str(container_name))) except Exception as e: # pragma: no cover self.logger.error('Failed to stop ' + str(container_name) + ' because: ' + str(e)) except Exception as e: # pragma: no cover self.logger.error('Stop failed with error: ' + str(e)) status = (False, e) return status def repo_commits(self, repo): """ Get the commit IDs for all of the branches of a repository """ commits = [] try: status = self.path_dirs.apply_path(repo) # switch to directory where repo will be cloned to if status[0]: cwd = status[1] else: self.logger.error('apply_path failed. Exiting repo_commits with' ' status: ' + str(status)) return status status = self.repo_branches(repo) if status[0]: branches = status[1] for branch in branches: try: branch_output = check_output(shlex .split('git rev-list origin/' + branch), stderr=STDOUT, close_fds=True).decode('utf-8') branch_output = branch_output.split('\n')[:-1] branch_output += ['HEAD'] commits.append((branch, branch_output)) except Exception as e: # pragma: no cover self.logger.error('repo_commits failed with error: ' + str(e) + ' on branch: ' + str(branch)) status = (False, e) return status else: self.logger.error('repo_branches failed. Exiting repo_commits' ' with status: ' + str(status)) return status chdir(cwd) status = (True, commits) except Exception as e: # pragma: no cover self.logger.error('repo_commits failed with error: ' + str(e)) status = (False, e) return status def repo_branches(self, repo): """ Get the branches of a repository """ branches = [] try: # switch to directory where repo will be cloned to status = self.path_dirs.apply_path(repo) if status[0]: cwd = status[1] else: self.logger.error('apply_path failed. Exiting repo_branches' ' with status ' + str(status)) return status branch_output = check_output(shlex.split('git branch -a'), stderr=STDOUT, close_fds=True) branch_output = branch_output.split(b'\n') for branch in branch_output: br = branch.strip() if br.startswith(b'*'): br = br[2:] if b'/' in br: branches.append(br.rsplit(b'/', 1)[1].decode('utf-8')) elif br: branches.append(br.decode('utf-8')) branches = list(set(branches)) for branch in branches: try: check_output(shlex.split('git checkout ' + branch), stderr=STDOUT, close_fds=True) except Exception as e: # pragma: no cover self.logger.error('repo_branches failed with error: ' + str(e) + ' on branch: ' + str(branch)) status = (False, e) return status chdir(cwd) status = (True, branches) except Exception as e: # pragma: no cover self.logger.error('repo_branches failed with error: ' + str(e)) status = (False, e) return status def repo_tools(self, repo, branch, version): """ Get available tools for a repository branch at a version """ try: tools = [] status = self.path_dirs.apply_path(repo) # switch to directory where repo will be cloned to if status[0]: cwd = status[1] else: self.logger.error('apply_path failed. Exiting repo_tools with' ' status: ' + str(status)) return status # TODO commenting out for now, should use update_repo #status = self.p_helper.checkout(branch=branch, version=version) status = (True, None) if status[0]: path, _, _ = self.path_dirs.get_path(repo) tools = AvailableTools(path, version=version) else: self.logger.error('checkout failed. Exiting repo_tools with' ' status: ' + str(status)) return status chdir(cwd) status = (True, tools) except Exception as e: # pragma: no cover self.logger.error('repo_tools failed with error: ' + str(e)) status = (False, e) return status
def test_ensure_file(): """ Test the ensure_file function """ paths = PathDirs() status = paths.ensure_file(paths.init_file) assert isinstance(status, tuple) assert status[0] == True
class VentApp(npyscreen.NPSAppManaged): """ Main menu app for vent CLI """ keypress_timeout_default = 10 repo_value = {} paths = PathDirs() first_time = paths.ensure_file(paths.init_file) if first_time[0] and first_time[1] != 'exists': npyscreen.NPSAppManaged.STARTING_FORM = 'TUTORIALINTRO' else: npyscreen.NPSAppManaged.STARTING_FORM = 'MAIN' def onStart(self): """ Override onStart method for npyscreen """ curses.mousemask(0) self.paths.host_config() version = Version() # setup initial runtime stuff if self.first_time[0] and self.first_time[1] != 'exists': plugins = Plugin() actions = Action() thr = Thread(target=MainForm.t_status, args=(), kwargs={'core': True}) thr.start() while thr.is_alive(): npyscreen.notify_wait( 'Please wait while Vent initializes...1/4', title='Setting up things...') time.sleep(1) thr.join() thr = Thread(target=MainForm.t_status, args=(), kwargs={'core': False}) thr.start() while thr.is_alive(): npyscreen.notify_wait( 'Please wait while Vent initializes...2/4', title='Setting up things...') time.sleep(1) thr.join() thr = Thread(target=plugins.auto_install, args=(), kwargs={}) thr.start() while thr.is_alive(): npyscreen.notify_wait( 'Please wait while Vent initializes...3/4', title='Setting up things...') time.sleep(1) thr.join() thr = Thread(target=actions.startup, args=(), kwargs={}) thr.start() while thr.is_alive(): npyscreen.notify_wait( 'Please wait while Vent initializes...4/4', title='Setting up things...') time.sleep(1) thr.join() quit_s = '\t' * 4 + '^Q to quit' tab_esc = '\t' * 4 + 'TAB to close menu popup' self.addForm('MAIN', MainForm, name='Vent ' + version + '\t\t\t\t\t^T for help' + quit_s + tab_esc, color='IMPORTANT') self.addForm('HELP', HelpForm, name='Help\t\t\t\t\t\t\t\t^T to toggle previous' + quit_s, color='DANGER') self.addForm('TUTORIALINTRO', TutorialIntroForm, name='Vent Tutorial' + quit_s, color='DANGER') self.addForm('TUTORIALBACKGROUND', TutorialBackgroundForm, name='About Vent' + quit_s, color='DANGER') self.addForm('TUTORIALTERMINOLOGY', TutorialTerminologyForm, name='About Vent' + quit_s, color='DANGER') self.addForm('TUTORIALGETTINGSETUP', TutorialGettingSetupForm, name='About Vent' + quit_s, color='DANGER') self.addForm('TUTORIALBUILDINGCORES', TutorialBuildingCoresForm, name='Working with Cores' + quit_s, color='DANGER') self.addForm('TUTORIALSTARTINGCORES', TutorialStartingCoresForm, name='Working with Cores' + quit_s, color='DANGER') self.addForm('TUTORIALADDINGPLUGINS', TutorialAddingPluginsForm, name='Working with Plugins' + quit_s, color='DANGER') self.addForm('TUTORIALADDINGFILES', TutorialAddingFilesForm, name='Files' + quit_s, color='DANGER') self.addForm('TUTORIALTROUBLESHOOTING', TutorialTroubleshootingForm, name='Troubleshooting' + quit_s, color='DANGER') def change_form(self, name): """ Changes the form (window) that is displayed """ self.switchForm(name)
class PluginHelper: """ Handle helper functions for the Plugin class """ def __init__(self, **kargs): self.d_client = docker.from_env() self.path_dirs = PathDirs(**kargs) self.manifest = join(self.path_dirs.meta_dir, 'plugin_manifest.cfg') self.logger = Logger(__name__) def constraint_options(self, constraint_dict, options): """ Return result of constraints and options against a template """ constraints = {} template = Template(template=self.manifest) for constraint in constraint_dict: if constraint != 'self': if (constraint_dict[constraint] or constraint_dict[constraint] == ''): constraints[constraint] = constraint_dict[constraint] results = template.constrained_sections(constraints=constraints, options=options) return results, template def get_path(self, repo, core=False): """ Return the path for the repo """ if repo.endswith('.git'): repo = repo.split('.git')[0] org, name = repo.split('/')[-2:] path = self.path_dirs.plugins_dir path = join(path, org, name) return path, org, name def apply_path(self, repo): """ Set path to where the repo is and return original path """ self.logger.info('Starting: apply_path') self.logger.info('repo given: ' + str(repo)) try: # rewrite repo for consistency if repo.endswith('.git'): repo = repo.split('.git')[0] # get org and repo name and path repo will be cloned to org, name = repo.split('/')[-2:] path = join(self.path_dirs.plugins_dir, org, name) self.logger.info('cloning to path: ' + str(path)) # save current path cwd = getcwd() # set to new repo path self.path_dirs.ensure_dir(path) chdir(path) status = (True, cwd, path) except Exception as e: # pragma: no cover self.logger.error('apply_path failed with error: ' + str(e)) status = (False, str(e)) self.logger.info('Status of apply_path: ' + str(status)) self.logger.info('Finished: apply_path') return status def checkout(self, branch='master', version='HEAD'): """ Checkout a specific version and branch of a repo """ self.logger.info('Starting: checkout') self.logger.info('branch given: ' + str(branch)) self.logger.info('version given: ' + str(version)) try: status = check_output(shlex.split('git checkout ' + branch), stderr=STDOUT, close_fds=True).decode('utf-8') status = check_output(shlex.split('git pull'), stderr=STDOUT, close_fds=True).decode('utf-8') status = check_output(shlex.split('git reset --hard ' + version), stderr=STDOUT, close_fds=True).decode('utf-8') response = (True, status) except Exception as e: # pragma: no cover self.logger.error('checkout failed with error: ' + str(e)) response = (False, str(e)) self.logger.info('Status of checkout: ' + str(response)) self.logger.info('Finished: checkout') return response def clone(self, repo, user=None, pw=None): """ Clone the repository """ self.logger.info('Starting: clone') self.logger.info('repo given: ' + str(repo)) self.logger.info('user given: ' + str(user)) status = (True, None) try: status = self.apply_path(repo) # if path already exists, try git checkout to update if status[0]: self.logger.info('path to clone to: ' + str(status[2])) try: check_output(shlex.split('git -C ' + status[2] + ' rev-parse'), stderr=STDOUT, close_fds=True).decode('utf-8') self.logger.info('path already exists: ' + str(status[2])) self.logger.info('Status of clone: ' + str(status[0])) self.logger.info('Finished: clone') chdir(status[1]) return (True, status[1]) except Exception as e: # pragma: no cover self.logger.info("repo doesn't exist, attempting to " + 'clone: ' + str(e)) else: self.logger.error('unable to clone') return status # ensure cloning still works even if ssl is broken cmd = 'git config --global http.sslVerify false' check_output(shlex.split(cmd), stderr=STDOUT, close_fds=True).decode('utf-8') # check if user and pw were supplied, typically for private repos if user and pw: # only https is supported when using user/pw auth_repo = 'https://' + user + ':' + pw + '@' repo = auth_repo + repo.split('https://')[-1] # clone repo and build tools check_output(shlex.split('git clone --recursive ' + repo + ' .'), stderr=STDOUT, close_fds=True).decode('utf-8') chdir(status[1]) status = (True, status[1]) except Exception as e: # pragma: no cover e_str = str(e) # scrub username and password from error message if e_str.find('@') >= 0: e_str = e_str[:e_str.find('//') + 2] + \ e_str[e_str.find('@') + 1:] self.logger.error('clone failed with error: ' + e_str) status = (False, e_str) self.logger.info('Status of clone: ' + str(status)) self.logger.info('Finished: clone') return status def available_tools(self, path, version='HEAD', groups=None): """ Return list of possible tools in repo for the given version and branch """ matches = [] if groups: groups = groups.split(',') for root, _, filenames in walk(path): files = fnmatch.filter(filenames, 'Dockerfile*') # append additional identifiers to tools if multiple in same # directory add_info = len(files) > 1 for f in files: # !! TODO deal with wild/etc.? addtl_info = '' if add_info: # @ will be delimiter symbol for multi-tools try: addtl_info = '@' + f.split('.')[1] except Exception as e: addtl_info = '@unspecified' if groups: if add_info and not addtl_info == '@unspecified': tool_template = addtl_info.split('@')[1] + '.template' else: tool_template = 'vent.template' try: template = Template(template=join(root, tool_template)) for group in groups: template_groups = template.option('info', 'groups') if (template_groups[0] and group in template_groups[1]): matches.append( (root.split(path)[1] + addtl_info, version)) except Exception as e: # pragma: no cover self.logger.info('error: ' + str(e)) else: matches.append((root.split(path)[1] + addtl_info, version)) return matches @staticmethod def tool_matches(tools=None, version='HEAD'): """ Get the tools paths and versions that were specified """ matches = [] if tools: for tool in tools: match_version = version if tool[1] != '': match_version = tool[1] match = '' if tool[0].endswith('/'): match = tool[0][:-1] elif tool[0] != '.': match = tool[0] if not match.startswith('/') and match != '': match = '/' + match matches.append((match, match_version)) return matches def start_sections(self, s, files, groups, enabled, branch, version): """ Run through sections for prep_start """ tool_d = {} status = (True, None) for section in s: # initialize needed vars c_name = s[section]['image_name'].replace(':', '-') c_name = c_name.replace('/', '-') instance_num = re.search(r'\d+$', s[section]['name']) if instance_num: c_name += instance_num.group() image_name = s[section]['image_name'] # checkout the right version and branch of the repo cwd = getcwd() self.logger.info('current directory is: ' + str(cwd)) # images built from registry won't have path if s[section]['path'] != '': chdir(join(s[section]['path'])) # TODO commenting out for now, should use update_repo #status = self.checkout(branch=branch, version=version) status = (True, None) self.logger.info(status) chdir(cwd) tool_d[c_name] = {'image': image_name, 'name': c_name} # get rid of all commented sections in various runtime # configurations manifest = Template(self.manifest) overall_dict = {} for setting in ['info', 'docker', 'gpu', 'settings', 'service']: option = manifest.option(section, setting) if option[0]: overall_dict[setting] = {} settings_dict = json.loads(option[1]) for opt in settings_dict: if not opt.startswith('#'): overall_dict[setting][opt] = \ settings_dict[opt] if 'docker' in overall_dict: options_dict = overall_dict['docker'] for option in options_dict: options = options_dict[option] # check for commands to evaluate if '`' in options: cmds = options.split('`') if len(cmds) > 2: i = 1 while i < len(cmds): try: cmds[i] = check_output( shlex.split(cmds[i]), stderr=STDOUT, close_fds=True).strip().decode('utf-8') except Exception as e: # pragma: no cover self.logger.error( 'unable to evaluate command specified in vent.template: ' + str(e)) i += 2 options = ''.join(cmds) # check for commands to evaluate # store options set for docker try: tool_d[c_name][option] = literal_eval(options) except Exception as e: # pragma: no cover self.logger.info('unable to literal_eval: ' + str(options)) tool_d[c_name][option] = options if 'labels' not in tool_d[c_name]: tool_d[c_name]['labels'] = {} # get the service uri info if 'service' in overall_dict: try: options_dict = overall_dict['service'] for option in options_dict: tool_d[c_name]['labels'][option] = options_dict[option] except Exception as e: # pragma: no cover self.logger.error('unable to store service options for ' 'docker: ' + str(e)) # check for gpu settings if 'gpu' in overall_dict: try: options_dict = json.loads(status[1]) for option in options_dict: tool_d[c_name]['labels']['gpu.' + option] = options_dict[option] except Exception as e: # pragma: no cover self.logger.error('unable to store gpu options for ' 'docker: ' + str(e)) # get temporary name for links, etc. plugin_c = Template(template=self.manifest) status, plugin_sections = plugin_c.sections() self.logger.info(status) for plugin_section in plugin_sections: status = plugin_c.option(plugin_section, 'link_name') self.logger.info(status) image_status = plugin_c.option(plugin_section, 'image_name') self.logger.info(image_status) if status[0] and image_status[0]: cont_name = image_status[1].replace(':', '-') cont_name = cont_name.replace('/', '-') if cont_name not in tool_d: tool_d[cont_name] = { 'image': image_status[1], 'name': cont_name, 'start': False } tool_d[cont_name]['tmp_name'] = status[1] # add extra labels tool_d[c_name]['labels']['vent'] = Version() tool_d[c_name]['labels']['vent.namespace'] = s[section][ 'namespace'] tool_d[c_name]['labels']['vent.branch'] = branch tool_d[c_name]['labels']['vent.version'] = version tool_d[c_name]['labels']['vent.name'] = s[section]['name'] tool_d[c_name]['labels']['vent.section'] = section tool_d[c_name]['labels']['vent.repo'] = s[section]['repo'] tool_d[c_name]['labels']['vent.type'] = s[section]['type'] # check for log_config settings in external-services externally_configured = False vent_config = Template(self.path_dirs.cfg_file) for ext_tool in vent_config.section('external-services')[1]: if ext_tool[0].lower() == 'syslog': try: log_dict = json.loads(ext_tool[1]) # configure if not locally active if ('locally_active' not in log_dict or log_dict['locally_active'] == 'no'): del log_dict['locally_active'] log_config = {} log_config['type'] = 'syslog' log_config['config'] = {} ip_address = '' port = '' for option in log_dict: if option == 'ip_address': ip_address = log_dict[option] elif option == 'port': port = log_dict['port'] syslog_address = 'tcp://' + ip_address + ':' + port syslog_config = { 'syslog-address': syslog_address, 'syslog-facility': 'daemon', 'tag': '{{.Name}}' } log_config['config'].update(syslog_config) externally_configured = True except Exception as e: # pragma: no cover self.logger.error('external settings for log_config' " couldn't be stored because: " + str(e)) externally_configured = False if not externally_configured: log_config = { 'type': 'syslog', 'config': { 'syslog-address': 'tcp://0.0.0.0:514', 'syslog-facility': 'daemon', 'tag': '{{.Name}}' } } if 'groups' in s[section]: # add labels for groups tool_d[c_name]['labels']['vent.groups'] = s[section]['groups'] # add restart=always to core containers if 'core' in s[section]['groups']: tool_d[c_name]['restart_policy'] = {'Name': 'always'} # map network names to environment variables if 'network' in s[section]['groups']: vent_config = Template(template=self.path_dirs.cfg_file) nic_mappings = vent_config.section('network-mapping') nics = '' if nic_mappings[0]: for nic in nic_mappings[1]: nics += nic[0] + ':' + nic[1] + ',' nics = nics[:-1] if nics: if 'environment' in tool_d[c_name]: tool_d[c_name]['environment'].append('VENT_NICS=' + nics) else: tool_d[c_name]['environment'] = [ 'VENT_NICS=' + nics ] # send logs to syslog if ('syslog' not in s[section]['groups'] and 'core' in s[section]['groups']): log_config['config']['tag'] = '{{.Name}}' tool_d[c_name]['log_config'] = log_config if 'syslog' not in s[section]['groups']: tool_d[c_name]['log_config'] = log_config # mount necessary directories if 'files' in s[section]['groups']: ulimits = [] ulimits.append( docker.types.Ulimit(name='nofile', soft=1048576, hard=1048576)) tool_d[c_name]['ulimits'] = ulimits # check if running in a docker container if 'VENT_CONTAINERIZED' in environ and environ[ 'VENT_CONTAINERIZED'] == 'true': if 'volumes_from' in tool_d[c_name]: tool_d[c_name]['volumes_from'].append( environ['HOSTNAME']) else: tool_d[c_name]['volumes_from'] = [ environ['HOSTNAME'] ] else: if 'volumes' in tool_d[c_name]: tool_d[c_name]['volumes'][ self.path_dirs.base_dir[:-1]] = { 'bind': '/vent', 'mode': 'ro' } else: tool_d[c_name]['volumes'] = { self.path_dirs.base_dir[:-1]: { 'bind': '/vent', 'mode': 'ro' } } if files[0]: if 'volumes' in tool_d[c_name]: tool_d[c_name]['volumes'][files[1]] = { 'bind': '/files', 'mode': 'rw' } else: tool_d[c_name]['volumes'] = { files[1]: { 'bind': '/files', 'mode': 'rw' } } else: tool_d[c_name]['log_config'] = log_config # add label for priority if 'settings' in overall_dict: try: options_dict = overall_dict['settings'] for option in options_dict: if option == 'priority': tool_d[c_name]['labels'][ 'vent.priority'] = options_dict[option] except Exception as e: # pragma: no cover self.logger.error('unable to store settings options ' 'for docker ' + str(e)) # only start tools that have been built if s[section]['built'] != 'yes': del tool_d[c_name] # store section information for adding info to manifest later else: tool_d[c_name]['section'] = section return status, tool_d def prep_start(self, repo=None, name=None, groups=None, enabled='yes', branch='master', version='HEAD'): """ Start a set of tools that match the parameters given, if no parameters are given, start all installed tools on the master branch at verison HEAD that are enabled """ args = locals() self.logger.info('Starting: prep_start') self.logger.info('Arguments: ' + str(args)) status = (False, None) try: options = [ 'name', 'namespace', 'built', 'groups', 'path', 'image_name', 'branch', 'repo', 'type', 'version' ] vent_config = Template(template=self.path_dirs.cfg_file) manifest = Template(self.manifest) files = vent_config.option('main', 'files') files = (files[0], expanduser(files[1])) s, _ = self.constraint_options(args, options) status, tool_d = self.start_sections(s, files, groups, enabled, branch, version) # look out for links to delete because they're defined externally links_to_delete = set() # get instances for each tool tool_instances = {} sections = manifest.sections()[1] for section in sections: settings = manifest.option(section, 'settings') if settings[0]: settings = json.loads(settings[1]) if 'instances' in settings: l_name = manifest.option(section, 'link_name') if l_name[0]: tool_instances[l_name[1]] = int( settings['instances']) # check and update links, volumes_from, network_mode for container in list(tool_d.keys()): if 'labels' not in tool_d[ container] or 'vent.groups' not in tool_d[container][ 'labels'] or 'core' not in tool_d[container][ 'labels']['vent.groups']: tool_d[container]['remove'] = True if 'links' in tool_d[container]: for link in list(tool_d[container]['links'].keys()): # add links to external services already running if # necessary, by default configure local services too configure_local = True ext = 'external-services' if link in vent_config.options(ext)[1]: try: lconf = json.loads( vent_config.option(ext, link)[1]) if ('locally_active' not in lconf or lconf['locally_active'] == 'no'): ip_adr = lconf['ip_address'] port = lconf['port'] tool_d[container]['extra_hosts'] = {} # containers use lowercase names for # connections tool_d[container]['extra_hosts'][ link.lower()] = ip_adr # create an environment variable for container # to access port later env_variable = link.upper() + \ '_CUSTOM_PORT=' + port if 'environment' not in tool_d[container]: tool_d[container]['environment'] = [] tool_d[container]['environment'].append( env_variable) # remove the entry from links because no # longer connecting to local container links_to_delete.add(link) configure_local = False except Exception as e: # pragma: no cover self.logger.error("couldn't load external" ' settings because: ' + str(e)) configure_local = True status = False if configure_local: for c in list(tool_d.keys()): if ('tmp_name' in tool_d[c] and tool_d[c]['tmp_name'] == link): tool_d[container]['links'][ tool_d[c]['name']] = tool_d[container][ 'links'].pop(link) if link in tool_instances and tool_instances[ link] > 1: for i in range( 2, tool_instances[link] + 1): tool_d[container]['links'][ tool_d[c]['name'] + str(i)] = tool_d[container][ 'links'][tool_d[c] ['name']] + str(i) if 'volumes_from' in tool_d[container]: tmp_volumes_from = tool_d[container]['volumes_from'] tool_d[container]['volumes_from'] = [] for volumes_from in list(tmp_volumes_from): for c in list(tool_d.keys()): if ('tmp_name' in tool_d[c] and tool_d[c]['tmp_name'] == volumes_from): tool_d[container]['volumes_from'].append( tool_d[c]['name']) tmp_volumes_from.remove(volumes_from) tool_d[container]['volumes_from'] += tmp_volumes_from if 'network_mode' in tool_d[container]: if tool_d[container]['network_mode'].startswith( 'container:'): network_c_name = tool_d[container][ 'network_mode'].split('container:')[1] for c in list(tool_d.keys()): if ('tmp_name' in tool_d[c] and tool_d[c]['tmp_name'] == network_c_name): tool_d[container]['network_mode'] = 'container:' + \ tool_d[c]['name'] # remove tmp_names for c in list(tool_d.keys()): if 'tmp_name' in tool_d[c]: del tool_d[c]['tmp_name'] # remove links section if all were externally configured for c in list(tool_d.keys()): if 'links' in tool_d[c]: for link in links_to_delete: if link in tool_d[c]['links']: del tool_d[c]['links'][link] # delete links if no more defined if not tool_d[c]['links']: del tool_d[c]['links'] # remove containers that shouldn't be started for c in list(tool_d.keys()): deleted = False if 'start' in tool_d[c] and not tool_d[c]['start']: del tool_d[c] deleted = True if not deleted: # look for tools services that are being done externally # tools are capitalized in vent.cfg, so make them lowercase # for comparison ext = 'external-services' external_tools = vent_config.section(ext)[1] name = tool_d[c]['labels']['vent.name'] for tool in external_tools: if name == tool[0].lower(): try: tool_config = json.loads(tool[1]) if ('locally_active' in tool_config and tool_config['locally_active'] == 'no'): del tool_d[c] except Exception as e: # pragma: no cover self.logger.warning( 'Locally running container ' + name + ' may be redundant') if status: status = (True, tool_d) else: status = (False, tool_d) except Exception as e: # pragma: no cover self.logger.error('prep_start failed with error: ' + str(e)) status = (False, e) self.logger.info('Status of prep_start: ' + str(status[0])) self.logger.info('Finished: prep_start') return status def start_priority_containers(self, groups, group_orders, tool_d): """ Select containers based on priorities to start """ vent_cfg = Template(self.path_dirs.cfg_file) cfg_groups = vent_cfg.option('groups', 'start_order') if cfg_groups[0]: cfg_groups = cfg_groups[1].split(',') else: cfg_groups = [] all_groups = sorted(set(groups)) s_conts = [] f_conts = [] # start tools in order of group defined in vent.cfg for group in cfg_groups: # remove from all_groups because already checked out if group in all_groups: all_groups.remove(group) if group in group_orders: for cont_t in sorted(group_orders[group]): if cont_t[1] not in s_conts: s_conts, f_conts = self.start_containers( cont_t[1], tool_d, s_conts, f_conts) # start tools that haven't been specified in the vent.cfg, if any for group in all_groups: if group in group_orders: for cont_t in sorted(group_orders[group]): if cont_t[1] not in s_conts: s_conts, f_conts = self.start_containers( cont_t[1], tool_d, s_conts, f_conts) return (s_conts, f_conts) def start_remaining_containers(self, containers_remaining, tool_d): """ Select remaining containers that didn't have priorities to start """ s_containers = [] f_containers = [] for container in containers_remaining: s_containers, f_containers = self.start_containers( container, tool_d, s_containers, f_containers) return (s_containers, f_containers) def start_containers(self, container, tool_d, s_containers, f_containers): """ Start container that was passed in and return status """ # use section to add info to manifest section = tool_d[container]['section'] del tool_d[container]['section'] manifest = Template(self.manifest) try: c = self.d_client.containers.get(container) c.start() s_containers.append(container) manifest.set_option(section, 'running', 'yes') self.logger.info('started ' + str(container) + ' with ID: ' + str(c.short_id)) except Exception as err: try: gpu = 'gpu.enabled' failed = False if (gpu in tool_d[container]['labels'] and tool_d[container]['labels'][gpu] == 'yes'): vent_config = Template(template=self.path_dirs.cfg_file) port = '' host = '' result = vent_config.option('nvidia-docker-plugin', 'port') if result[0]: port = result[1] else: port = '3476' result = vent_config.option('nvidia-docker-plugin', 'host') if result[0]: host = result[1] else: # now just requires ip, ifconfig try: route = check_output( ('ip', 'route')).decode('utf-8').split('\n') default = '' # grab the default network device. for device in route: if 'default' in device: default = device.split()[4] break # grab the IP address for the default device ip_addr = check_output( ('ifconfig', default)).decode('utf-8') ip_addr = ip_addr.split('\n')[1].split()[1] host = ip_addr except Exception as e: # pragma no cover self.logger.error('failed to grab ip. Ensure that \ ip and ifconfig are installed') nd_url = 'http://' + host + ':' + port + '/v1.0/docker/cli' params = {'vol': 'nvidia_driver'} r = requests.get(nd_url, params=params) if r.status_code == 200: options = r.text.split() for option in options: if option.startswith('--volume-driver='): tool_d[container][ 'volume_driver'] = option.split('=', 1)[1] elif option.startswith('--volume='): vol = option.split('=', 1)[1].split(':') if 'volumes' in tool_d[container]: if isinstance(tool_d[container]['volumes'], list): if len(vol) == 2: c_vol = vol[0] + \ ':' + vol[1] + ':rw' else: c_vol = vol[0] + ':' + \ vol[1] + ':' + vol[2] tool_d[container]['volumes'].append( c_vol) else: # Dictionary tool_d[container]['volumes'][ vol[0]] = { 'bind': vol[1], 'mode': vol[2] } else: tool_d[container]['volumes'] = { vol[0]: { 'bind': vol[1], 'mode': vol[2] } } elif option.startswith('--device='): dev = option.split('=', 1)[1] if 'devices' in tool_d[container]: tool_d[container]['devices'].append(dev + ':' + dev + ':rwm') else: tool_d[container]['devices'] = [ dev + ':' + dev + ':rwm' ] else: self.logger.error('Unable to parse ' + 'nvidia-docker option: ' + str(option)) else: failed = True f_containers.append(container) manifest.set_option(section, 'running', 'failed') self.logger.error('failed to start ' + str(container) + ' because nvidia-docker-plugin ' + 'failed with: ' + str(r.status_code)) if not failed: try: self.d_client.containers.remove(container, force=True) self.logger.info('removed old existing container: ' + str(container)) except Exception as e: pass cont_id = self.d_client.containers.run(detach=True, **tool_d[container]) s_containers.append(container) manifest.set_option(section, 'running', 'yes') self.logger.info('started ' + str(container) + ' with ID: ' + str(cont_id)) except Exception as e: # pragma: no cover f_containers.append(container) manifest.set_option(section, 'running', 'failed') self.logger.error('failed to start ' + str(container) + ' because: ' + str(e)) # save changes made to manifest manifest.write_config() return s_containers, f_containers
def __init__(self, **kargs): self.path_dirs = PathDirs(**kargs) self.manifest = os.path.join(self.path_dirs.meta_dir, "plugin_manifest.cfg") self.d_client = docker.from_env() self.logger = Logger(__name__)
class Plugin: """ Handle Plugins """ def __init__(self, **kargs): self.path_dirs = PathDirs(**kargs) self.manifest = os.path.join(self.path_dirs.meta_dir, "plugin_manifest.cfg") self.d_client = docker.from_env() self.logger = Logger(__name__) def apply_path(self, repo): """ Set path to where the repo is and return original path """ self.logger.info("Starting: apply_path") self.logger.info("repo given: "+str(repo)) status = (True, None) try: # rewrite repo for consistency if repo.endswith(".git"): repo = repo.split(".git")[0] # get org and repo name and path repo will be cloned to org, name = repo.split("/")[-2:] self.path = os.path.join(self.path_dirs.plugins_dir, org, name) self.logger.info("cloning to path: "+str(self.path)) # save current path cwd = os.getcwd() # set to new repo path os.chdir(self.path) status = (True, cwd) except Exception as e: self.logger.error("apply_path failed with error: "+str(e)) status = (False, e) self.logger.info("Status of apply_path: "+str(status)) self.logger.info("Finished: apply_path") return status def repo_branches(self, repo): """ Get the branches of a repository """ self.logger.info("Starting: repo_branches") self.logger.info("repo given: "+str(repo)) status = (True, None) branches = [] try: # switch to directory where repo will be cloned to status = self.apply_path(repo) if status[0]: cwd = status[1] else: self.logger.info("apply_path failed. Exiting repo_branches with status "+str(status)) return status junk = subprocess.check_output(shlex.split("git pull --all"), stderr=subprocess.STDOUT, close_fds=True) branch_output = subprocess.check_output(shlex.split("git branch -a"), stderr=subprocess.STDOUT, close_fds=True) branch_output = branch_output.split("\n") for branch in branch_output: b = branch.strip() if b.startswith('*'): b = b[2:] if "/" in b: branches.append(b.rsplit('/', 1)[1]) elif b: branches.append(b) branches = list(set(branches)) self.logger.info("branches found: "+str(branches)) for branch in branches: try: junk = subprocess.check_output(shlex.split("git checkout " + branch), stderr=subprocess.STDOUT, close_fds=True) except Exception as e: # pragma: no cover self.logger.error("repo_branches failed with error: "+str(e)+" on branch: "+str(branch)) status = (False, e) self.logger.info("Exiting repo_branches with status: "+str(status)) return status try: os.chdir(cwd) except Exception as e: # pragma: no cover self.logger.error("unable to change directory to: "+str(cwd)+"because: "+str(e)) status = (True, branches) except Exception as e: self.logger.error("repo_branches failed with error: "+str(e)) status = (False, e) self.logger.info("Status of repo_branches: "+str(status)) self.logger.info("Finished: repo_branches") return status def repo_commits(self, repo): """ Get the commit IDs for all of the branches of a repository """ self.logger.info("Starting: repo_commits") self.logger.info("repo given: "+str(repo)) status = (True, None) commits = [] try: status = self.apply_path(repo) # switch to directory where repo will be cloned to if status[0]: cwd = status[1] else: self.logger.info("apply_path failed. Exiting repo_commits with status: "+str(status)) return status status = self.repo_branches(repo) if status[0]: branches = status[1] for branch in branches: try: branch_output = subprocess.check_output(shlex.split("git rev-list " + branch), stderr=subprocess.STDOUT, close_fds=True) branch_output = ['HEAD'] + branch_output.split("\n")[:-1] commits.append((branch, branch_output)) except Exception as e: # pragma: no cover self.logger.error("repo_commits failed with error: "+str(e)+" on branch: "+str(branch)) status = (False, e) self.logger.info("Exiting repo_commits with status: "+str(status)) return status else: self.logger.info("repo_branches failed. Exiting repo_commits with status: "+str(status)) return status try: os.chdir(cwd) except Exception as e: # pragma: no cover self.logger.error("unable to change directory to: "+str(cwd)+" because: "+str(e)) status = (True, commits) except Exception as e: self.logger.error("repo_commits failed with error: "+str(e)) status = (False, e) self.logger.info("Status of repo_commits: "+str(status)) self.logger.info("Finished: repo_commits") return status def repo_tools(self, repo, branch, version): """ Get available tools for a repository branch at a version """ self.logger.info("Starting: repo_tools") self.logger.info("repo given: "+str(repo)) self.logger.info("branch given: "+str(branch)) self.logger.info("version given: "+str(version)) status = (True, None) try: tools = [] status = self.apply_path(repo) # switch to directory where repo will be cloned to if status[0]: cwd = status[1] else: self.logger.info("apply_path failed. Exiting repo_tools with status: "+str(status)) return status self.branch = branch self.version = version status = self.checkout() if status[0]: tools = self._available_tools() else: self.logger.info("checkout failed. Exiting repo_tools with status: "+str(status)) return status try: os.chdir(cwd) except Exception as e: # pragma: no cover self.logger.error("unable to change directory to: "+str(cwd)+" because: "+str(e)) status = (True, tools) except Exception as e: self.logger.error("repo_tools failed with error: "+str(e)) status = (False, e) self.logger.info("Status of repo_tools: "+str(status)) self.logger.info("Finished: repo_tools") return status def clone(self, repo, user=None, pw=None): """ Clone the repository """ self.logger.info("Starting: clone") self.logger.info("repo given: "+str(repo)) self.logger.info("user given: "+str(user)) status = (True, None) try: self.org = None self.name = None self.repo = repo # save current path cwd = os.getcwd() self.logger.info("current working directory: "+str(cwd)) # rewrite repo for consistency if self.repo.endswith(".git"): self.repo = self.repo.split(".git")[0] # get org and repo name and path repo will be cloned to self.org, self.name = self.repo.split("/")[-2:] self.logger.info("org name found: "+str(self.org)) self.logger.info("repo name found: "+str(self.name)) self.path = os.path.join(self.path_dirs.plugins_dir, self.org, self.name) self.logger.info("path to clone to: "+str(self.path)) # check if the directory exists, if so return now status = self.path_dirs.ensure_dir(self.path) if not status[0]: self.logger.info("ensure_dir failed. Exiting clone with status: "+str(status)) return status # set to new repo path os.chdir(self.path) # if path already exists, try git checkout to update if status[0] and status[1] == 'exists': try: response = subprocess.check_output(shlex.split("git -C "+self.path+" rev-parse"), stderr=subprocess.STDOUT, close_fds=True) self.logger.info("path already exists: "+str(self.path)) status = (True, cwd) self.logger.info("Status of clone: "+str(status)) self.logger.info("Finished: clone") return status except Exception as e: # pragma: no cover self.logger.error("unable to checkout: "+str(path)+" because: "+str(e)) status = (False, e) self.logger.info("Exiting clone with status: "+str(status)) return status # ensure cloning still works even if ssl is broken...probably should be improved response = subprocess.check_output(shlex.split("git config --global http.sslVerify false"), stderr=subprocess.STDOUT, close_fds=True) # check if user and pw were supplied, typically for private repos if user and pw: # only https is supported when using user/pw repo = 'https://'+user+':'+pw+'@'+self.repo.split("https://")[-1] # clone repo and build tools response = subprocess.check_output(shlex.split("git clone --recursive " + repo + " ."), stderr=subprocess.STDOUT, close_fds=True) status = (True, cwd) except Exception as e: self.logger.error("clone failed with error: "+str(e)) status = (False, e) self.logger.info("Status of clone: "+str(status)) self.logger.info("Finished: clone") return status def add(self, repo, tools=None, overrides=None, version="HEAD", branch="master", build=True, user=None, pw=None, groups=None, version_alias=None, wild=None, remove_old=True, disable_old=True, limit_groups=None): """ Adds a plugin of tool(s) tools is a list of tuples, where the pair is a tool name (path to Dockerfile) and version tools are for explicitly limiting which tools and versions (if version in tuple is '', then defaults to version) overrides is a list of tuples, where the pair is a tool name (path to Dockerfile) and a version overrides are for explicitly removing tools and overriding versions of tools (if version in tuple is '', then tool is removed, otherwise that tool is checked out at the specific version in the tuple) if tools and overrides are left as empty lists, then all tools in the repo are pulled down at the version and branch specified or defaulted to version is globally set for all tools, unless overridden in tools or overrides branch is globally set for all tools build is a boolean of whether or not to build the tools now user is the username for a private repo if needed pw is the password to go along with the username for a private repo groups is globally set for all tools version_alias is globally set for all tools and is a mapping from a friendly version tag to the real version commit ID wild lets you specify individual overrides for additional values in the tuple of tools or overrides. wild is a list containing one or more of the following: branch, build, groups, version_alias the order of the items in the wild list will expect values to be tacked on in the same order to the tuple for tools and overrides in additional to the tool name and version remove_old lets you specify whether or not to remove previously found tools that match to ones being added currently (note does not stop currently running instances of the older version) disable_old lets you specify whether or not to disable previously found tools that match to ones being added currently (note does not stop currently running instances of the older version) limit_groups is a list of groups to build tools for that match group names in vent.template of each tool if exists Examples: repo=fe (get all tools from repo 'fe' at version 'HEAD' on branch 'master') repo=foo, version="3d1f", branch="foo" (get all tools from repo 'foo' at verion '3d1f' on branch 'foo') repo=foo, tools=[('bar', ''), ('baz', '1d32')] (get only 'bar' from repo 'foo' at version 'HEAD' on branch 'master' and 'baz' from repo 'foo' at version '1d32' on branch 'master', ignore all other tools in repo 'foo') repo=foo overrides=[('baz/bar', ''), ('.', '1c4e')], version='4fad' (get all tools from repo 'foo' at verion '4fad' on branch 'master' except 'baz/bar' and for tool '.' get version '1c4e') repo=foo tools=[('bar', '1a2d')], overrides=[('baz', 'f2a1')] (not a particularly useful example, but get 'bar' from 'foo' at version '1a2d' and get 'baz' from 'foo' at version 'f2a1' on branch 'master', ignore all other tools) """ # initialize and store class objects self.tools = tools self.overrides = overrides self.version = version self.branch = branch self.build = build self.groups = groups # TODO these need to be implemented self.version_alias = version_alias self.wild = wild self.remove_old = remove_old self.disable_old = disable_old self.limit_groups = limit_groups status = (True, None) status_code, cwd = self.clone(repo, user=user, pw=pw) status = self._build_tools(status_code) # set back to original path try: os.chdir(cwd) except Exception as e: # pragma: no cover pass return status @ErrorHandler def builder(self, template, match_path, image_name, section, build=None, branch=None, version=None): """ Build tools """ self.logger.info("Starting: builder") self.logger.info("install path: "+str(match_path)) self.logger.info("image name: "+str(image_name)) self.logger.info("build: "+str(build)) self.logger.info("branch: "+str(branch)) self.logger.info("version: "+str(version)) if build: self.build = build elif not hasattr(self, 'build'): self.build = True if branch: self.branch = branch elif not hasattr(self, 'branch'): self.branch = 'master' if version: self.version = version elif not hasattr(self, 'version'): self.version = 'HEAD' cwd = os.getcwd() self.logger.info("current working directory: "+str(cwd)) try: os.chdir(match_path) except Exception as e: self.logger.error("unable to change to directory: "+str(match_path)+" because: "+str(e)) return None template = self._build_image(template, match_path, image_name, section) try: os.chdir(cwd) except Exception as e: # pragma: no cover self.logger.error("unable to change to directory: "+str(cwd)+" because: "+str(e)) self.logger.info("template of builder: "+str(template)) self.logger.info("Finished: builder") return template def _build_tools(self, status): """ Create list of tools, paths, and versions to be built and sends them to build_manifest """ response = (True, None) # !! TODO implement features: wild, remove_old, disable_old, limit_groups # check result of clone, ensure successful or that it already exists if status: response = self.checkout() if response[0]: matches = [] if self.tools is None and self.overrides is None: # get all tools matches = self._available_tools() elif self.tools is None: # there's only something in overrides # grab all the tools then apply overrides matches = self._available_tools() # !! TODO apply overrides to matches elif self.overrides is None: # there's only something in tools # only grab the tools specified matches = self.get_tool_matches() else: # both tools and overrides were specified # grab only the tools specified, with the overrides applied orig_matches = self.get_tool_matches() matches = orig_matches for override in self.overrides: override_t = None if override[0] == '.': override_t = ('', override[1]) else: override_t = override for match in orig_matches: if override_t[0] == match[0]: matches.remove(match) matches.append(override_t) if len(matches) > 0: self._build_manifest(matches) else: response = (False, status) return response def get_tool_matches(self): """ Get the tools paths and versions that were specified by self.tools and self.version """ matches = [] if not hasattr(self, 'tools'): self.tools = [] if not hasattr(self, 'version'): self.version = 'HEAD' for tool in self.tools: match_version = self.version if tool[1] != '': match_version = tool[1] match = '' if tool[0].endswith('/'): match = tool[0][:-1] elif tool[0] != '.': match = tool[0] if not match.startswith('/') and match != '': match = '/'+match matches.append((match, match_version)) return matches def _build_manifest(self, matches): """ Builds and writes the manifest for the tools being added """ # !! TODO check for pre-existing that conflict with request and disable and/or remove image for match in matches: template = Template(template=self.manifest) # !! TODO check for special settings here first for the specific match self.version = match[1] response = self.checkout() if response[0]: section = self.org + ":" + self.name + ":" + match[0] + ":" + self.branch + ":" + self.version match_path = self.path + match[0] image_name = self.org + "-" + self.name + "-" if match[0] != '': # if tool is in a subdir, add that to the name of the image image_name += '-'.join(match[0].split('/')[1:]) + "-" image_name += self.branch + ":" + self.version # check if the section already exists exists, options = template.section(section) previous_commit = None previous_commits = None head = False if exists: for option in options: # TODO check if tool name but a different version exists - then disable/remove if set if option[0] == 'version' and option[1] == 'HEAD': head = True if option[0] == 'built' and option[1] == 'yes': # !! TODO remove pre-existing image pass if option[0] == 'commit_id': previous_commit = option[1] if option[0] == 'previous_versions': previous_commits = option[1] # !! TODO # check if section should be removed from config - i.e. all tools, # but new commit removed one that was in a previous commit # set template section and options for tool at version and branch template.add_section(section) template.set_option(section, "name", match[0].split('/')[-1]) template.set_option(section, "namespace", self.org+'/'+self.name) template.set_option(section, "path", match_path) template.set_option(section, "repo", self.repo) template.set_option(section, "enabled", "yes") template.set_option(section, "branch", self.branch) template.set_option(section, "version", self.version) template.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") template.set_option(section, "image_name", image_name) vent_template = Template(template=os.path.join(match_path, 'vent.template')) vent_status, response = vent_template.option("info", "name") if vent_status: template.set_option(section, "link_name", response) else: template.set_option(section, "link_name", match[0].split('/')[-1]) commit_id = None if self.version == 'HEAD': os.chdir(match_path) commit_id = subprocess.check_output(shlex.split("git rev-parse --short HEAD"), stderr=subprocess.STDOUT, close_fds=True).strip() template.set_option(section, "commit_id", commit_id) if head: # no need to store previous commits if not HEAD, since # the version will always be the same commit ID if previous_commit and previous_commit != commit_id: if previous_commits and previous_commit not in previous_commits: previous_commits = previous_commit+','+previous_commits elif not previous_commits: previous_commits = previous_commit if previous_commits and previous_commits != commit_id: template.set_option(section, "previous_versions", previous_commits) if self.version_alias: template.set_option(section, "version_alias", self.version_alias) if self.groups: template.set_option(section, "groups", self.groups) else: vent_template = os.path.join(match_path, 'vent.template') if os.path.exists(vent_template): v_template = Template(template=vent_template) groups = v_template.option("info", "groups") if groups[0]: template.set_option(section, "groups", groups[1]) template = self._build_image(template, match_path, image_name, section) # write out configuration to the manifest file template.write_config() # reset to repo directory os.chdir(self.path) return def _build_image(self, template, match_path, image_name, section): """ Build docker images and store results in template """ # !! TODO return status of whether it built successfully or not if self.build: try: os.chdir(match_path) # currently can't use docker-py because it doesn't support labels on images yet name = template.option(section, "name") groups = template.option(section, "groups") if groups[1] == "" or not groups[0]: groups = (True, "none") if not name[0]: name = (True, image_name) # pull if '/' in image_name, fallback to build pull = False if '/' in image_name: try: self.logger.info("Trying to pull "+image_name) output = subprocess.check_output(shlex.split("docker pull "+image_name), stderr=subprocess.STDOUT, close_fds=True) self.logger.info("Pulling "+name[1]+"\n"+str(output)) for line in output.split('\n'): if line.startswith("Digest: sha256:"): image_id = line.split("Digest: sha256:")[1][:12] if image_id: template.set_option(section, "built", "yes") template.set_option(section, "image_id", image_id) template.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") status = (True, "Pulled "+image_name) self.logger.info(str(status)) else: template.set_option(section, "built", "failed") template.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") status = (False, "Failed to pull image "+str(output.split('\n')[-1])) self.logger.warning(str(status)) pull = True except Exception as e: # pragma: no cover self.logger.warning("Failed to pull image, going to build instead: "+str(e)) if not pull: output = subprocess.check_output(shlex.split("docker build --label vent --label vent.name="+name[1]+" --label vent.groups="+groups[1]+" -t " + image_name + " ."), stderr=subprocess.STDOUT, close_fds=True) self.logger.info("Building "+name[1]+"\n"+str(output)) image_id = "" for line in output.split("\n"): if line.startswith("Successfully built "): image_id = line.split("Successfully built ")[1].strip() template.set_option(section, "built", "yes") template.set_option(section, "image_id", image_id) template.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") except Exception as e: # pragma: no cover self.logger.error("unable to build image: "+str(image_name)+" because: "+str(e)) template.set_option(section, "built", "failed") template.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") else: template.set_option(section, "built", "no") template.set_option(section, "last_updated", str(datetime.datetime.utcnow()) + " UTC") return template def _available_tools(self, groups=None): """ Return list of possible tools in repo for the given version and branch """ matches = [] if not hasattr(self, 'path'): return matches if groups: groups = groups.split(",") for root, dirnames, filenames in os.walk(self.path): for filename in fnmatch.filter(filenames, 'Dockerfile'): # !! TODO deal with wild/etc.? if groups: try: template = Template(template=os.path.join(root, 'vent.template')) for group in groups: template_groups = template.option("info", "groups") if template_groups[0] and group in template_groups[1]: matches.append((root.split(self.path)[1], self.version)) except Exception as e: # pragma: no cover pass else: matches.append((root.split(self.path)[1], self.version)) return matches def checkout(self): """ Checkout a specific version and branch of a repo """ if not hasattr(self, 'branch'): self.branch = 'master' if not hasattr(self, 'version'): self.version = 'HEAD' response = (True, None) try: status = subprocess.check_output(shlex.split("git checkout " + self.branch), stderr=subprocess.STDOUT, close_fds=True) status = subprocess.check_output(shlex.split("git pull"), stderr=subprocess.STDOUT, close_fds=True) status = subprocess.check_output(shlex.split("git reset --hard " + self.version), stderr=subprocess.STDOUT, close_fds=True) response = (True, status) except Exception as e: # pragma: no cover response = (False, os.getcwd()+str(e)) return response @staticmethod def add_image(image, tag="latest"): """ Add an image from a registry/hub rather than building from a repository """ # !! TODO return def constraint_options(self, constraint_dict, options): """ Return result of constraints and options against a template """ constraints = {} template = Template(template=self.manifest) for constraint in constraint_dict: if constraint != 'self': if constraint_dict[constraint] or constraint_dict[constraint] == '': constraints[constraint] = constraint_dict[constraint] results = template.constrained_sections(constraints=constraints, options=options) return results, template def tools(self): """ Return list of tuples of all tools """ tools = [] template = Template(template=self.manifest) exists, sections = template.sections() if exists: for section in sections: options = {'section':section, 'enabled':None, 'built':None, 'version':None, 'repo':None, 'branch':None, 'name':None, 'groups':None, 'image_name':None} for option in options.keys(): exists, value = template.option(section, option) if exists: options[option] = value tools.append(options) return tools def remove(self, name=None, repo=None, namespace=None, branch="master", groups=None, enabled="yes", version="HEAD", built="yes"): """ Remove tool (name) or repository, repository is the url. If no arguments are specified, all tools will be removed for the defaults. """ # initialize args = locals() status = (True, None) # get resulting dictionary of sections with options that match constraints results, template = self.constraint_options(args, []) for result in results: response, image_name = template.option(result, 'image_name') # check for container and remove container_name = image_name.replace(':', '-').replace('/', '-') try: container = self.d_client.containers.get(container_name) response = container.remove(v=True, force=True) self.logger.info(response) self.logger.info("Removing plugin container: "+container_name) except Exception as e: # pragma: no cover self.logger.warn("Unable to remove the plugin container: " + container_name + " because: " + str(e)) # check for image and remove try: response = self.d_client.images.remove(image_name) self.logger.info(response) self.logger.info("Removing plugin image: "+image_name) except Exception as e: # pragma: no cover self.logger.warn("Unable to remove the plugin image: " + image_name + " because: " + str(e)) # remove tool from the manifest status = template.del_section(result) self.logger.info("Removing plugin tool: "+result) # TODO if all tools from a repo have been removed, remove the repo template.write_config() return status def update(self, name=None, repo=None, namespace=None, branch=None, groups=None): """ Update tool (name) or repository, repository is the url. If no arguments are specified, all tools will be updated """ # initialize args = locals() status = (False, None) options = ['branch', 'groups', 'image_name'] # get resulting dictionary of sections with options that match constraints results, template = self.constraint_options(args, options) for result in results: # check for container and remove try: container_name = results['image_name'].replace(':', '-') \ .replace('/', '-') container = self.d_client.containers.get(container_name) response = container.remove(v=True, force=True) except Exception as e: # pragma: no cover pass # TODO git pull # TODO build # TODO docker pull # TODO update tool in the manifest self.logger.info("Updating plugin tool: "+result) template.write_config() return status # !! TODO name or group ? def versions(self, name, namespace=None, branch="master"): """ Return available versions of a tool """ # initialize args = locals() versions = [] options = ['version', 'previous_versions'] # get resulting dictionary of sections with options that match constraints results, _ = self.constraint_options(args, options) for result in results: version_list = [results[result]['version']] if 'previous_versions' in results[result]: version_list = version_list+(results[result]['previous_versions']).split(',') versions.append((result, version_list)) return versions # !! TODO name or group ? def current_version(self, name, namespace=None, branch="master"): """ Return current version for a given tool """ # initialize args = locals() versions = [] options = ['version'] # get resulting dictionary of sections with options that match constraints results, _ = self.constraint_options(args, options) for result in results: versions.append((result, results[result]['version'])) return versions # !! TODO name or group ? def state(self, name, namespace=None, branch="master"): """ Return state of a tool, disabled/enabled for each version """ # initialize args = locals() states = [] options = ['enabled'] # get resulting dictionary of sections with options that match constraints results, _ = self.constraint_options(args, options) for result in results: if results[result]['enabled'] == 'yes': states.append((result, 'enabled')) else: states.append((result, 'disabled')) return states # !! TODO name or group ? def enable(self, name, namespace=None, branch="master", version="HEAD"): """ Enable tool at a specific version, default to head """ # initialize args = locals() status = (False, None) # get resulting dictionary of sections with options that match constraints results, template = self.constraint_options(args, []) for result in results: status = template.set_option(result, 'enabled', 'yes') template.write_config() return status # !! TODO name or group ? def disable(self, name, namespace=None, branch="master", version="HEAD"): """ Disable tool at a specific version, default to head """ # initialize args = locals() status = (False, None) # get resulting dictionary of sections with options that match constraints results, template = self.constraint_options(args, []) for result in results: status = template.set_option(result, 'enabled', 'no') template.write_config() return status
class System: def __init__(self, *args, **kwargs): self.d_client = docker.from_env() self.path_dirs = PathDirs(**kwargs) self.manifest = join(self.path_dirs.meta_dir, 'plugin_manifest.cfg') self.vent_config = self.path_dirs.cfg_file self.startup_file = self.path_dirs.startup_file self.logger = Logger(__name__) self._auto_install() def _auto_install(self): """ Automatically detects images and installs them in the manifest if they are not there already """ template = Template(template=self.manifest) sections = template.sections() images = self.d_client.images.list(filters={'label': 'vent'}) add_sections = [] status = (True, None) for image in images: ignore = False if ('Labels' in image.attrs['Config'] and 'vent.section' in image.attrs['Config']['Labels'] and not image.attrs['Config']['Labels']['vent.section'] in sections[1]): section = image.attrs['Config']['Labels']['vent.section'] section_str = image.attrs['Config']['Labels']['vent.section'].split( ':') template.add_section(section) if 'vent.name' in image.attrs['Config']['Labels']: template.set_option(section, 'name', image.attrs['Config']['Labels']['vent.name']) if 'vent.repo' in image.attrs['Config']['Labels']: template.set_option(section, 'repo', image.attrs['Config']['Labels']['vent.repo']) git_path = join(self.path_dirs.plugins_dir, '/'.join(section_str[:2])) # TODO clone it down template.set_option(section, 'path', join( git_path, section_str[-3][1:])) # get template settings # TODO account for template files not named vent.template v_template = Template(template=join( git_path, section_str[-3][1:], 'vent.template')) tool_sections = v_template.sections() if tool_sections[0]: for s in tool_sections[1]: section_dict = {} options = v_template.options(s) if options[0]: for option in options[1]: option_name = option if option == 'name': # get link name template.set_option(section, 'link_name', v_template.option(s, option)[1]) option_name = 'link_name' opt_val = v_template.option(s, option)[1] section_dict[option_name] = opt_val if section_dict: template.set_option(section, s, json.dumps(section_dict)) if ('vent.type' in image.attrs['Config']['Labels'] and image.attrs['Config']['Labels']['vent.type'] == 'repository'): template.set_option( section, 'namespace', '/'.join(section_str[:2])) template.set_option(section, 'branch', section_str[-2]) template.set_option(section, 'version', section_str[-1]) template.set_option(section, 'last_updated', str( datetime.utcnow()) + ' UTC') if image.attrs['RepoTags']: template.set_option( section, 'image_name', image.attrs['RepoTags'][0]) else: # image with none tag is outdated, don't add it ignore = True template.set_option(section, 'type', 'repository') if 'vent.groups' in image.attrs['Config']['Labels']: template.set_option(section, 'groups', image.attrs['Config']['Labels']['vent.groups']) template.set_option(section, 'built', 'yes') template.set_option(section, 'image_id', image.attrs['Id'].split(':')[1][:12]) template.set_option(section, 'running', 'no') # check if image is running as a container containers = self.d_client.containers.list( filters={'label': 'vent'}) for container in containers: if container.attrs['Image'] == image.attrs['Id']: template.set_option(section, 'running', 'yes') if not ignore: add_sections.append(section) template.write_config() # TODO this check will always be true, need to actually validate the above logic if status[0]: status = (True, add_sections) return status def backup(self): """ Saves the configuration information of the current running vent instance to be used for restoring at a later time """ status = (True, None) # initialize all needed variables (names for backup files, etc.) backup_name = ('.vent-backup-' + '-'.join(Timestamp().split(' '))) backup_dir = join(expanduser('~'), backup_name) backup_manifest = join(backup_dir, 'backup_manifest.cfg') backup_vcfg = join(backup_dir, 'backup_vcfg.cfg') manifest = self.manifest # create new backup directory try: mkdir(backup_dir) except Exception as e: # pragma: no cover self.logger.error(str(e)) return (False, str(e)) # create new files in backup directory try: # backup manifest with open(backup_manifest, 'w') as bmanifest: with open(manifest, 'r') as manifest_file: bmanifest.write(manifest_file.read()) # backup vent.cfg with open(backup_vcfg, 'w') as bvcfg: with open(self.vent_config, 'r') as vcfg_file: bvcfg.write(vcfg_file.read()) self.logger.info('Backup information written to ' + backup_dir) status = (True, backup_dir) except Exception as e: # pragma: no cover self.logger.error("Couldn't backup vent: " + str(e)) status = (False, str(e)) # TODO #266 return status def configure(self): # TODO return def gpu(self): # TODO return def history(self): # TODO #255 return def restore(self, backup_dir): """ Restores a vent configuration from a previously backed up version """ # TODO #266 status = (True, None) return status def reset(self): """ Factory reset all of Vent's user data, containers, and images """ status = (True, None) error_message = '' # remove containers try: c_list = set(self.d_client.containers.list( filters={'label': 'vent'}, all=True)) for c in c_list: c.remove(force=True) except Exception as e: # pragma: no cover error_message += 'Error removing Vent containers: ' + str(e) + '\n' # remove images try: i_list = set(self.d_client.images.list(filters={'label': 'vent'}, all=True)) for i in i_list: # delete tagged images only because they are the parents for # the untagged images. Remove the parents and the children get # removed automatically if i.attrs['RepoTags']: self.d_client.images.remove(image=i.id, force=True) except Exception as e: # pragma: no cover error_message += 'Error deleting Vent images: ' + str(e) + '\n' # remove .vent folder try: cwd = getcwd() if cwd.startswith(join(expanduser('~'), '.vent')): chdir(expanduser('~')) shutil.rmtree(join(expanduser('~'), '.vent')) except Exception as e: # pragma: no cover error_message += 'Error deleting Vent data: ' + str(e) + '\n' if error_message: status = (False, error_message) return status def rollback(self): # TODO #266 return def start(self): status = (True, None) vent_bridge = None # create vent network bridge if it doesn't already exist try: vent_bridge = self.d_client.networks.create( 'vent', check_duplicate=True, driver='bridge') except docker.errors.APIError as e: # pragma: no cover if str(e) != '409 Client Error: Conflict ("network with name vent already exists")': self.logger.error( 'Unable to create network bridge because: {0}'.format(str(e))) status = (False, str(e)) else: vent_bridge = self.d_client.networks.list('vent')[0] if status[0]: # add vent to the vent network bridge try: vent_bridge.connect(environ['HOSTNAME']) except Exception as e: # pragma: no coverr self.logger.error( 'Unable to connect vent to the network bridge because: {0}'.format(str(e))) status = (False, str(e)) if status[0]: # remove vent to the default network bridge try: default_bridge = self.d_client.networks.list('bridge')[0] default_bridge.disconnect(environ['HOSTNAME']) except Exception as e: # pragma: no coverr self.logger.error( 'Unable to disconnect vent from the default network bridge because: {0}'.format(str(e))) status = (False, str(e)) if status[0]: # startup based on startup file if exists(self.startup_file): status = self._startup() else: tools = Tools() status = tools.new('core', None) if status[0]: status = tools.start( 'https://github.com/cyberreboot/vent', None) return status def _startup(self): """ Automatically detect if a startup file is specified and stand up a vent host with all necessary tools based on the specifications in that file """ status = (True, None) try: s_dict = {} # rewrite the yml file to exclusively lowercase with open(self.startup_file, 'r') as sup: vent_startup = sup.read() with open(self.startup_file, 'w') as sup: for line in vent_startup: sup.write(line.lower()) with open(self.startup_file, 'r') as sup: s_dict = yaml.safe_load(sup.read()) if 'vent.cfg' in s_dict: v_cfg = Template(self.vent_config) for section in s_dict['vent.cfg']: for option in s_dict['vent.cfg'][section]: val = ('no', 'yes')[ s_dict['vent.cfg'][section][option]] v_status = v_cfg.add_option(section, option, value=val) if not v_status[0]: v_cfg.set_option(section, option, val) v_cfg.write_config() del s_dict['vent.cfg'] tool_d = {} extra_options = ['info', 'service', 'settings', 'docker', 'gpu'] s_dict_c = copy.deepcopy(s_dict) # TODO check for repo or image type for repo in s_dict_c: repository = Repository(System().manifest) repository.repo = repo repository._clone() repo_path, org, r_name = self.path_dirs.get_path(repo) get_tools = [] for tool in s_dict_c[repo]: t_branch = 'master' t_version = 'HEAD' if 'branch' in s_dict[repo][tool]: t_branch = s_dict[repo][tool]['branch'] if 'version' in s_dict[repo][tool]: t_version = s_dict[repo][tool]['version'] get_tools.append((tool, t_branch, t_version)) available_tools = AvailableTools(repo_path, tools=get_tools) for tool in s_dict_c[repo]: # if we can't find the tool in that repo, skip over this # tool and notify in the logs t_path, t_path_cased = PathDirs.rel_path( tool, available_tools) if t_path is None: self.logger.error("Couldn't find tool " + tool + ' in' ' repo ' + repo) continue # ensure no NoneType iteration errors if s_dict_c[repo][tool] is None: s_dict[repo][tool] = {} # check if we need to configure instances along the way instances = 1 if 'settings' in s_dict[repo][tool]: if 'instances' in s_dict[repo][tool]['settings']: instances = int(s_dict[repo][tool] ['settings']['instances']) # add the tool t_branch = 'master' t_version = 'HEAD' t_image = None add_tools = None add_tools = [(t_path_cased, '')] if 'branch' in s_dict[repo][tool]: t_branch = s_dict[repo][tool]['branch'] if 'version' in s_dict[repo][tool]: t_version = s_dict[repo][tool]['version'] if 'image' in s_dict[repo][tool]: t_image = s_dict[repo][tool]['image'] repository.add( repo, tools=add_tools, branch=t_branch, version=t_version, image_name=t_image) manifest = Template(self.manifest) # update the manifest with extra defined runtime settings base_section = ':'.join([org, r_name, t_path, t_branch, t_version]) for option in extra_options: if option in s_dict[repo][tool]: opt_dict = manifest.option(base_section, option) # add new values defined into default options for # that tool, don't overwrite them if opt_dict[0]: opt_dict = json.loads(opt_dict[1]) else: opt_dict = {} # stringify values for vent for v in s_dict[repo][tool][option]: pval = s_dict[repo][tool][option][v] s_dict[repo][tool][option][v] = json.dumps( pval) opt_dict.update(s_dict[repo][tool][option]) manifest.set_option(base_section, option, json.dumps(opt_dict)) # copy manifest info into new sections if necessary if instances > 1: for i in range(2, instances + 1): i_section = base_section.rsplit(':', 2) i_section[0] += str(i) i_section = ':'.join(i_section) manifest.add_section(i_section) for opt_val in manifest.section(base_section)[1]: if opt_val[0] == 'name': manifest.set_option(i_section, opt_val[0], opt_val[1] + str(i)) else: manifest.set_option(i_section, opt_val[0], opt_val[1]) manifest.write_config() tool_d = {} tools = Tools() # start tools, if necessary for repo in s_dict: for tool in s_dict[repo]: if 'start' in s_dict[repo][tool]: if s_dict[repo][tool]['start']: local_instances = 1 if 'settings' in s_dict[repo][tool] and 'instances' in s_dict[repo][tool]['settings']: local_instances = int( s_dict[repo][tool]['settings']['instances']) t_branch = 'master' t_version = 'HEAD' if 'branch' in s_dict[repo][tool]: t_branch = s_dict[repo][tool]['branch'] if 'version' in s_dict[repo][tool]: t_version = s_dict[repo][tool]['version'] for i in range(1, local_instances + 1): i_name = tool + str(i) if i != 1 else tool i_name = i_name.replace('@', '') tool_d.update( tools._prep_start(repo, i_name)[1]) if tool_d: tools.start(tool_d, None, is_tool_d=True) except Exception as e: # pragma: no cover self.logger.error('Startup failed because: {0}'.format(str(e))) status = (False, str(e)) return status def stop(self): status = (True, None) # remove containers try: c_list = set(self.d_client.containers.list( filters={'label': 'vent'}, all=True)) for c in c_list: c.remove(force=True) except Exception as e: # pragma: no cover status = (False, str(e)) return status def get_configure(self, repo=None, name=None, groups=None, main_cfg=False): """ Get the vent.template settings for a given tool by looking at the plugin_manifest """ constraints = locals() del constraints['main_cfg'] status = (True, None) template_dict = {} return_str = '' if main_cfg: vent_cfg = Template(self.vent_config) for section in vent_cfg.sections()[1]: template_dict[section] = {} for vals in vent_cfg.section(section)[1]: template_dict[section][vals[0]] = vals[1] else: # all possible vent.template options stored in plugin_manifest options = ['info', 'service', 'settings', 'docker', 'gpu'] tools = Template(System().manifest).constrain_opts( constraints, options)[0] if tools: # should only be one tool tool = list(tools.keys())[0] # load all vent.template options into dict for section in tools[tool]: template_dict[section] = json.loads(tools[tool][section]) else: status = (False, "Couldn't get vent.template information") if status[0]: # display all those options as they would in the file for section in template_dict: return_str += '[' + section + ']\n' # ensure instances shows up in configuration for option in template_dict[section]: if option.startswith('#'): return_str += option + '\n' else: return_str += option + ' = ' return_str += template_dict[section][option] + '\n' return_str += '\n' # only one newline at end of file status = (True, return_str[:-1]) return status def save_configure(self, repo=None, name=None, groups=None, config_val='', from_registry=False, main_cfg=False, instances=1): """ Save changes made to vent.template through npyscreen to the template and to plugin_manifest """ def template_to_manifest(vent_template, manifest, tool, instances): """ Helper function to transfer information from vent.template to plugin_manifest """ sections = vent_template.sections() if sections[0]: for section in sections[1]: section_dict = {} if section == 'settings': section_dict.update({'instances': str(instances)}) options = vent_template.options(section) if options[0]: for option in options[1]: option_name = option if option == 'name': option_name = 'link_name' opt_val = vent_template.option(section, option)[1] section_dict[option_name] = opt_val if section_dict: manifest.set_option(tool, section, json.dumps(section_dict)) elif manifest.option(tool, section)[0]: manifest.del_option(tool, section) constraints = locals() del constraints['config_val'] del constraints['from_registry'] del constraints['main_cfg'] del constraints['instances'] del constraints['template_to_manifest'] status = (True, None) fd = None # ensure instances is an int and remove instances from config_val to # ensure correct info instances = int(instances) config_val = re.sub(r'instances\ *=\ *\d+\n', '', config_val) api_system = System() manifest = api_system.manifest if not main_cfg: if not from_registry: # creating new instances if instances > 1: fd, template_path = tempfile.mkstemp(suffix='.template') # scrub name for clean section name if re.search(r'\d+$', name): name = re.sub(r'\d+$', '', name) t_identifier = {'name': name, 'branch': branch, 'version': version} result = Template(manifest).constrain_opts( t_identifier, []) tools = result[0] tool = list(tools.keys())[0] else: options = ['path', 'multi_tool', 'name'] tools, _ = Template(manifest).constrain_opts( constraints, options) # only one tool in tools because perform this function for # every tool if tools: tool = list(tools.keys())[0] if ('multi_tool' in tools[tool] and tools[tool]['multi_tool'] == 'yes'): name = tools[tool]['name'] if name == 'unspecified': name = 'vent' template_path = join(tools[tool]['path'], name+'.template') else: template_path = join(tools[tool]['path'], 'vent.template') else: status = (False, "Couldn't save configuration") else: fd, template_path = tempfile.mkstemp(suffix='.template') options = ['namespace'] constraints.update({'type': 'registry'}) tools, _ = Template(manifest).constrain_opts(constraints, options) if tools: tool = list(tools.keys())[0] else: status = (False, "Couldn't save configuration") if status[0]: try: # save in vent.template with open(template_path, 'w') as f: f.write(config_val) # save in plugin_manifest vent_template = Template(template_path) manifest = Template(manifest) if instances > 1: # add instances as needed for i in range(1, instances + 1): i_section = tool.rsplit(':', 2) i_section[0] += str(i) if i != 1 else '' i_section = ':'.join(i_section) if not manifest.section(i_section)[0]: manifest.add_section(i_section) for val_pair in manifest.section(tool)[1]: name = val_pair[0] val = val_pair[1] if name == 'name': val += str(i) elif name == 'last_updated': val = Timestamp() elif name == 'running': val = 'no' manifest.set_option(i_section, name, val) template_to_manifest(vent_template, manifest, i_section, instances) else: settings = manifest.option(i_section, 'settings') if settings[0]: settings_dict = json.loads(settings[1]) settings_dict['instances'] = str(instances) manifest.set_option(i_section, 'settings', json.dumps( settings_dict)) else: inst = str(instances) settings_dict = {'instances': inst} manifest.set_option(i_section, 'settings', json.dumps( settings_dict)) else: try: settings_str = manifest.option(tool, 'settings')[1] settings_dict = json.loads(settings_str) old_instances = int(settings_dict['instances']) except Exception: old_instances = 1 template_to_manifest(vent_template, manifest, tool, old_instances) manifest.write_config() status = (True, manifest) except Exception as e: # pragma: no cover self.logger.error('save_configure error: ' + str(e)) status = (False, str(e)) # close os file handle and remove temp file if from_registry or instances > 1: try: close(fd) remove(template_path) except Exception as e: # pragma: no cover self.logger.error('save_configure error: ' + str(e)) else: with open(self.vent_config, 'w') as f: f.write(config_val) return status def restart_tools(self, repo=None, name=None, groups=None, branch='master', version='HEAD', main_cfg=False, old_val='', new_val=''): """ Restart necessary tools based on changes that have been made either to vent.cfg or to vent.template. This includes tools that need to be restarted because they depend on other tools that were changed. """ status = (True, None) if not main_cfg: try: t_identifier = {'name': name, 'branch': branch, 'version': version} result = Template(System().manifest).constrain_opts(t_identifier, ['running', 'link_name']) tools = result[0] tool = list(tools.keys())[0] if ('running' in tools[tool] and tools[tool]['running'] == 'yes'): start_tools = [t_identifier] dependent_tools = [tools[tool]['link_name']] start_tools += Dependencies(dependent_tools) # TODO start_d = {} for tool_identifier in start_tools: self.clean(**tool_identifier) start_d.update(self.prep_start(**tool_identifier)[1]) if start_d: Tools().start(start_d, '', is_tool_d=True) except Exception as e: # pragma: no cover self.logger.error('Trouble restarting tool ' + name + ' because: ' + str(e)) status = (False, str(e)) else: try: # string manipulation to get tools into arrays ext_start = old_val.find('[external-services]') if ext_start >= 0: ot_str = old_val[old_val.find('[external-services]') + 20:] else: ot_str = '' old_tools = [] for old_tool in ot_str.split('\n'): if old_tool != '': old_tools.append(old_tool.split('=')[0].strip()) ext_start = new_val.find('[external-services]') if ext_start >= 0: nt_str = new_val[new_val.find('[external-services]') + 20:] else: nt_str = '' new_tools = [] for new_tool in nt_str.split('\n'): if new_tool != '': new_tools.append(new_tool.split('=')[0].strip()) # find tools changed tool_changes = [] for old_tool in old_tools: if old_tool not in new_tools: tool_changes.append(old_tool) for new_tool in new_tools: if new_tool not in old_tools: tool_changes.append(new_tool) else: # tool name will be the same oconf = old_val[old_val.find(new_tool):].split('\n')[0] nconf = new_val[new_val.find(new_tool):].split('\n')[0] if oconf != nconf: tool_changes.append(new_tool) # put link names in a dictionary for finding dependencies dependent_tools = [] for i, entry in enumerate(tool_changes): dependent_tools.append(entry) # change names to lowercase for use in clean, prep_start tool_changes[i] = {'name': entry.lower().replace('-', '_')} dependencies = Dependencies(dependent_tools) # restart tools restart = tool_changes + dependencies tool_d = {} for tool in restart: self.clean(**tool) tool_d.update(self.prep_start(**tool)[1]) if tool_d: # TODO fix the arguments Tools().start(tool_d) except Exception as e: # pragma: no cover self.logger.error('Problem restarting tools: ' + str(e)) status = (False, str(e)) return status def upgrade(self): ''' Upgrades Vent itself, and core containers ''' # TODO return
def Services(core, vent=True, external=False, **kargs): """ Get services that have exposed ports, expects param core to be True or False based on which type of services to return, by default limit to vent containers and processes not running externally, if not limited by vent containers, then core is ignored. """ services = [] path_dirs = PathDirs(**kargs) template = Template(template=path_dirs.cfg_file) services_uri = template.option("main", "services_uri") try: # look for internal services if not external: d_client = docker.from_env() if vent: c_filter = {'label': 'vent'} containers = d_client.containers.list(filters=c_filter) else: containers = d_client.containers.list() for c in containers: uri_prefix = '' uri_postfix = '' uri_user = '' uri_pw = '' name = None if vent and 'vent.name' in c.attrs['Config']['Labels']: if ((core and 'vent.groups' in c.attrs['Config']['Labels'] and 'core' in c.attrs['Config']['Labels']['vent.groups']) or (not core and 'vent.groups' in c.attrs['Config']['Labels'] and 'core' not in c.attrs['Config']['Labels']['vent.groups'])): name = c.attrs['Config']['Labels']['vent.name'] if 'uri_prefix' in c.attrs['Config']['Labels']: uri_prefix = c.attrs['Config']['Labels'][ 'uri_prefix'] if 'uri_postfix' in c.attrs['Config']['Labels']: uri_postfix = c.attrs['Config']['Labels'][ 'uri_postfix'] if 'uri_user' in c.attrs['Config']['Labels']: uri_user = "******" uri_user += c.attrs['Config']['Labels']['uri_user'] if 'uri_pw' in c.attrs['Config']['Labels']: uri_pw = " pw:" uri_pw += c.attrs['Config']['Labels']['uri_pw'] else: name = c.name ports = c.attrs['NetworkSettings']['Ports'] p = [] for port in ports: if ports[port]: uri_creds = '' if uri_user or uri_pw: uri_creds = " - (" + uri_user + uri_pw + " )" host = ports[port][0]['HostIp'] if services_uri[0] and host == '0.0.0.0': host = services_uri[1] p.append(uri_prefix + host + ":" + ports[port][0]['HostPort'] + uri_postfix + uri_creds) if p and name: services.append((name, p)) # look for external services else: ext_tools = template.section('external-services')[1] for ext_tool in ext_tools: try: name = ext_tool[0].lower() p = [] settings_dict = json.loads(ext_tool[1]) if ('locally_active' in settings_dict and settings_dict['locally_active'] == 'no'): # default protocol to display will be http protocol = 'http' ip_address = '' port = '' for setting in settings_dict: if setting == 'ip_address': ip_address = settings_dict[setting] if setting == 'port': port = settings_dict[setting] if setting == 'protocol': protocol = settings_dict[setting] p.append(protocol + '://' + ip_address + ':' + port) if p and name: services.append((name, p)) except Exception: # pragma: no cover p = None except Exception as e: # pragma: no cover pass return services
def _startup(self): """ Automatically detect if a startup file is specified and stand up a vent host with all necessary tools based on the specifications in that file """ status = (True, None) try: s_dict = {} # rewrite the yml file to exclusively lowercase with open(self.startup_file, 'r') as sup: vent_startup = sup.read() with open(self.startup_file, 'w') as sup: for line in vent_startup: sup.write(line.lower()) with open(self.startup_file, 'r') as sup: s_dict = yaml.safe_load(sup.read()) if 'vent.cfg' in s_dict: v_cfg = Template(self.vent_config) for section in s_dict['vent.cfg']: for option in s_dict['vent.cfg'][section]: val = ('no', 'yes')[ s_dict['vent.cfg'][section][option]] v_status = v_cfg.add_option(section, option, value=val) if not v_status[0]: v_cfg.set_option(section, option, val) v_cfg.write_config() del s_dict['vent.cfg'] tool_d = {} extra_options = ['info', 'service', 'settings', 'docker', 'gpu'] s_dict_c = copy.deepcopy(s_dict) # TODO check for repo or image type for repo in s_dict_c: repository = Repository(System().manifest) repository.repo = repo repository._clone() repo_path, org, r_name = self.path_dirs.get_path(repo) get_tools = [] for tool in s_dict_c[repo]: t_branch = 'master' t_version = 'HEAD' if 'branch' in s_dict[repo][tool]: t_branch = s_dict[repo][tool]['branch'] if 'version' in s_dict[repo][tool]: t_version = s_dict[repo][tool]['version'] get_tools.append((tool, t_branch, t_version)) available_tools = AvailableTools(repo_path, tools=get_tools) for tool in s_dict_c[repo]: # if we can't find the tool in that repo, skip over this # tool and notify in the logs t_path, t_path_cased = PathDirs.rel_path( tool, available_tools) if t_path is None: self.logger.error("Couldn't find tool " + tool + ' in' ' repo ' + repo) continue # ensure no NoneType iteration errors if s_dict_c[repo][tool] is None: s_dict[repo][tool] = {} # check if we need to configure instances along the way instances = 1 if 'settings' in s_dict[repo][tool]: if 'instances' in s_dict[repo][tool]['settings']: instances = int(s_dict[repo][tool] ['settings']['instances']) # add the tool t_branch = 'master' t_version = 'HEAD' t_image = None add_tools = None add_tools = [(t_path_cased, '')] if 'branch' in s_dict[repo][tool]: t_branch = s_dict[repo][tool]['branch'] if 'version' in s_dict[repo][tool]: t_version = s_dict[repo][tool]['version'] if 'image' in s_dict[repo][tool]: t_image = s_dict[repo][tool]['image'] repository.add( repo, tools=add_tools, branch=t_branch, version=t_version, image_name=t_image) manifest = Template(self.manifest) # update the manifest with extra defined runtime settings base_section = ':'.join([org, r_name, t_path, t_branch, t_version]) for option in extra_options: if option in s_dict[repo][tool]: opt_dict = manifest.option(base_section, option) # add new values defined into default options for # that tool, don't overwrite them if opt_dict[0]: opt_dict = json.loads(opt_dict[1]) else: opt_dict = {} # stringify values for vent for v in s_dict[repo][tool][option]: pval = s_dict[repo][tool][option][v] s_dict[repo][tool][option][v] = json.dumps( pval) opt_dict.update(s_dict[repo][tool][option]) manifest.set_option(base_section, option, json.dumps(opt_dict)) # copy manifest info into new sections if necessary if instances > 1: for i in range(2, instances + 1): i_section = base_section.rsplit(':', 2) i_section[0] += str(i) i_section = ':'.join(i_section) manifest.add_section(i_section) for opt_val in manifest.section(base_section)[1]: if opt_val[0] == 'name': manifest.set_option(i_section, opt_val[0], opt_val[1] + str(i)) else: manifest.set_option(i_section, opt_val[0], opt_val[1]) manifest.write_config() tool_d = {} tools = Tools() # start tools, if necessary for repo in s_dict: for tool in s_dict[repo]: if 'start' in s_dict[repo][tool]: if s_dict[repo][tool]['start']: local_instances = 1 if 'settings' in s_dict[repo][tool] and 'instances' in s_dict[repo][tool]['settings']: local_instances = int( s_dict[repo][tool]['settings']['instances']) t_branch = 'master' t_version = 'HEAD' if 'branch' in s_dict[repo][tool]: t_branch = s_dict[repo][tool]['branch'] if 'version' in s_dict[repo][tool]: t_version = s_dict[repo][tool]['version'] for i in range(1, local_instances + 1): i_name = tool + str(i) if i != 1 else tool i_name = i_name.replace('@', '') tool_d.update( tools._prep_start(repo, i_name)[1]) if tool_d: tools.start(tool_d, None, is_tool_d=True) except Exception as e: # pragma: no cover self.logger.error('Startup failed because: {0}'.format(str(e))) status = (False, str(e)) return status
class Repository: def __init__(self, manifest, *args, **kwargs): self.path_dirs = PathDirs(**kwargs) self.manifest = manifest self.d_client = docker.from_env() self.logger = Logger(__name__) def add(self, repo, tools=None, overrides=None, version='HEAD', core=False, image_name=None, branch='master', build=True, user=None, pw=None): status = (True, None) self.repo = repo.lower() self.tools = tools self.overrides = overrides self.branch = branch self.version = version self.image_name = image_name self.core = core status = self._clone(user=user, pw=pw) if status[0] and build: status = self._build() return status def _build(self): status = (True, None) status = self._get_tools() matches = status[1] status = self.path_dirs.apply_path(self.repo) original_path = status[1] if status[0] and len(matches) > 0: repo, org, name = self.path_dirs.get_path(self.repo) cmd = 'git rev-parse --short ' + self.version commit_id = '' try: commit_id = check_output(shlex.split(cmd), stderr=STDOUT, close_fds=True).strip().decode('utf-8') except Exception as e: # pragma: no cover self.logger.error( 'Unable to get commit ID because: {0}'.format(str(e))) template = Template(template=self.manifest) for match in matches: status, template, match_path, image_name, section = self._build_manifest( match, template, repo, org, name, commit_id) if not status[0]: break status, template = self._build_image(template, match_path, image_name, section) if not status[0]: break if status[0]: # write out configuration to the manifest file template.write_config() chdir(original_path) return status def _get_tools(self): status = (True, None) matches = [] path, _, _ = self.path_dirs.get_path(self.repo) status = Checkout(path, branch=self.branch, version=self.version) if status[0]: if self.tools is None and self.overrides is None: # get all tools matches = AvailableTools( path, branch=self.branch, version=self.version, core=self.core) elif self.tools is None: # there's only something in overrides # grab all the tools then apply overrides matches = AvailableTools( path, branch=self.branch, version=self.version, core=self.core) # !! TODO apply overrides to matches elif self.overrides is None: # there's only something in tools # only grab the tools specified matches = ToolMatches(tools=self.tools, version=self.version) else: # both tools and overrides were specified # grab only the tools specified, with the overrides applied o_matches = ToolMatches(tools=self.tools, version=self.version) matches = o_matches for override in self.overrides: override_t = None if override[0] == '.': override_t = ('', override[1]) else: override_t = override for match in o_matches: if override_t[0] == match[0]: matches.remove(match) matches.append(override_t) status = (True, matches) return status def _build_manifest(self, match, template, repo, org, name, commit_id): status = (True, None) # keep track of whether or not to write an additional manifest # entry for multiple instances, and how many additional entries # to write addtl_entries = 1 # remove @ in match for template setting purposes if match[0].find('@') >= 0: true_name = match[0].split('@')[1] else: true_name = match[0] # TODO check for special settings here first for the specific match self.version = match[1] section = org + ':' + name + ':' + true_name + ':' section += self.branch + ':' + self.version # need to get rid of temp identifiers for tools in same repo match_path = repo + match[0].split('@')[0] if self.image_name: image_name = self.image_name elif not self.core: image_name = org + '/' + name if match[0] != '': # if tool is in a subdir, add that to the name of the # image image_name += '-' + '-'.join(match[0].split('/')[1:]) image_name += ':' + self.branch else: image_name = ('cyberreboot/vent-' + match[0].split('/')[-1] + ':' + self.branch) image_name = image_name.replace('_', '-') # check if the section already exists is_there, options = template.section(section) previous_commit = None previous_commits = None head = False if is_there: for option in options: # TODO check if tool name but a different version # exists - then disable/remove if set if option[0] == 'version' and option[1] == 'HEAD': head = True if option[0] == 'built' and option[1] == 'yes': # !! TODO remove pre-existing image pass if option[0] == 'commit_id': previous_commit = option[1] if option[0] == 'previous_versions': previous_commits = option[1] # check if tool comes from multi directory multi_tool = 'no' if match[0].find('@') >= 0: multi_tool = 'yes' # !! TODO # check if section should be removed from config i.e. all tools # but new commit removed one that was in a previous commit image_name = image_name.lower() image_name = image_name.replace('@', '-') # special case for vent images if image_name.startswith('cyberreboot/vent'): image_name = image_name.replace('vent-vent-core-', 'vent-') image_name = image_name.replace('vent-vent-extras-', 'vent-') # set template section & options for tool at version and branch template.add_section(section) template.set_option(section, 'name', true_name.split('/')[-1]) template.set_option(section, 'namespace', org + '/' + name) template.set_option(section, 'path', match_path) template.set_option(section, 'repo', self.repo) template.set_option(section, 'multi_tool', multi_tool) template.set_option(section, 'branch', self.branch) template.set_option(section, 'version', self.version) template.set_option(section, 'last_updated', str(datetime.utcnow()) + ' UTC') template.set_option(section, 'image_name', image_name) template.set_option(section, 'type', 'repository') # save settings in vent.template to plugin_manifest # watch for multiple tools in same directory # just wanted to store match path with @ for path for use in # other actions tool_template = 'vent.template' if match[0].find('@') >= 0: tool_template = match[0].split('@')[1] + '.template' vent_template_path = join(match_path, tool_template) if exists(vent_template_path): with open(vent_template_path, 'r') as f: vent_template_val = f.read() else: vent_template_val = '' settings_dict = ParsedSections(vent_template_val) for setting in settings_dict: template.set_option(section, setting, json.dumps(settings_dict[setting])) # TODO do we need this if we save as a dictionary? vent_template = Template(vent_template_path) vent_status, response = vent_template.option('info', 'name') instances = vent_template.option('settings', 'instances') if instances[0]: addtl_entries = int(instances[1]) if vent_status: template.set_option(section, 'link_name', response) else: template.set_option(section, 'link_name', true_name.split('/')[-1]) if self.version == 'HEAD': template.set_option(section, 'commit_id', commit_id) if head: # no need to store previous commits if not HEAD, since # the version will always be the same commit ID if previous_commit and previous_commit != commit_id: if (previous_commits and previous_commit not in previous_commits): previous_commits = (previous_commit + ',' + previous_commits) elif not previous_commits: previous_commits = previous_commit if previous_commits and previous_commits != commit_id: template.set_option(section, 'previous_versions', previous_commits) groups = vent_template.option('info', 'groups') if groups[0]: template.set_option(section, 'groups', groups[1]) # set groups to empty string if no groups defined for tool else: template.set_option(section, 'groups', '') # write additional entries for multiple instances if addtl_entries > 1: # add 2 for naming conventions for i in range(2, addtl_entries + 1): addtl_section = section.rsplit(':', 2) addtl_section[0] += str(i) addtl_section = ':'.join(addtl_section) template.add_section(addtl_section) orig_vals = template.section(section)[1] for val in orig_vals: template.set_option(addtl_section, val[0], val[1]) template.set_option(addtl_section, 'name', true_name.split('/')[-1]+str(i)) return status, template, match_path, image_name, section def _build_image(self, template, match_path, image_name, section, build_local=False): status = (True, None) output = '' def set_instances(template, section, built, image_id=None): """ Set build information for multiple instances """ i = 2 while True: addtl_section = section.rsplit(':', 2) addtl_section[0] += str(i) addtl_section = ':'.join(addtl_section) if template.section(addtl_section)[0]: template.set_option(addtl_section, 'built', built) if image_id: template.set_option(addtl_section, 'image_id', image_id) template.set_option(addtl_section, 'last_updated', Timestamp()) else: break i += 1 # determine whether a tool should be considered a multi instance multi_instance = False try: settings = template.option(section, 'settings') if settings[0]: settings_dict = json.loads(settings[1]) if 'instances' in settings_dict and int(settings_dict['instances']) > 1: multi_instance = True except Exception as e: # pragma: no cover self.logger.error( 'Failed to check for multi instance because: {0}'.format(str(e))) status = (False, str(e)) cwd = getcwd() chdir(match_path) try: name = template.option(section, 'name') groups = template.option(section, 'groups') t_type = template.option(section, 'type') path = template.option(section, 'path') status, config_override = self.path_dirs.override_config(path[1]) if groups[1] == '' or not groups[0]: groups = (True, 'none') if not name[0]: name = (True, image_name) pull = False image_exists = False cfg_template = Template(template=self.path_dirs.cfg_file) use_existing_image = False result = cfg_template.option('build-options', 'use_existing_images') if result[0]: use_existing_image = result[1] if use_existing_image == 'yes' and not config_override: try: self.d_client.images.get(image_name) i_attrs = self.d_client.images.get(image_name).attrs image_id = i_attrs['Id'].split(':')[1][:12] template.set_option(section, 'built', 'yes') template.set_option(section, 'image_id', image_id) template.set_option(section, 'last_updated', str(datetime.utcnow()) + ' UTC') # set other instances too if multi_instance: set_instances(template, section, 'yes', image_id) status = (True, 'Found {0}'.format(image_name)) self.logger.info(str(status)) image_exists = True except docker.errors.ImageNotFound: image_exists = False except Exception as e: # pragma: no cover self.logger.warning( 'Failed to query Docker for images because: {0}'.format(str(e))) if not image_exists: # pull if '/' in image_name, fallback to build if '/' in image_name and not build_local and not config_override: try: image = self.d_client.images.pull(image_name) i_attrs = self.d_client.images.get( image_name).attrs image_id = i_attrs['Id'].split(':')[1][:12] if image_id: template.set_option(section, 'built', 'yes') template.set_option(section, 'image_id', image_id) template.set_option(section, 'last_updated', str(datetime.utcnow()) + ' UTC') # set other instances too if multi_instance: set_instances(template, section, 'yes', image_id) status = (True, 'Pulled {0}'.format(image_name)) self.logger.info(str(status)) else: template.set_option(section, 'built', 'failed') template.set_option(section, 'last_updated', str(datetime.utcnow()) + ' UTC') # set other instances too if multi_instance: set_instances(template, section, 'failed') status = (False, 'Failed to pull image {0}'.format( str(output.split('\n')[-1]))) self.logger.warning(str(status)) pull = True except Exception as e: # pragma: no cover self.logger.warning( 'Failed to pull image, going to build instead: {0}'.format(str(e))) status = ( False, 'Failed to pull image because: {0}'.format(str(e))) if not pull and not image_exists: # get username to label built image with username = getpass.getuser() # see if additional file arg needed for building multiple # images from same directory file_tag = 'Dockerfile' multi_tool = template.option(section, 'multi_tool') if multi_tool[0] and multi_tool[1] == 'yes': specific_file = template.option(section, 'name')[1] if specific_file != 'unspecified': file_tag = 'Dockerfile.' + specific_file # update image name with new version for update image_name = image_name.rsplit(':', 1)[0]+':'+self.branch labels = {} labels['vent'] = '' labels['vent.section'] = section labels['vent.repo'] = self.repo labels['vent.type'] = t_type[1] labels['vent.name'] = name[1] labels['vent.groups'] = groups[1] labels['built-by'] = username image = self.d_client.images.build(path='.', dockerfile=file_tag, tag=image_name, labels=labels, rm=True) image_id = image[0].id.split(':')[1][:12] template.set_option(section, 'built', 'yes') template.set_option(section, 'image_id', image_id) template.set_option(section, 'last_updated', str(datetime.utcnow()) + ' UTC') # set other instances too if multi_instance: set_instances(template, section, 'yes', image_id) status = (True, 'Built {0}'.format(image_name)) except Exception as e: # pragma: no cover self.logger.error('Unable to build image {0} because: {1} | {2}'.format( str(image_name), str(e), str(output))) template.set_option(section, 'built', 'failed') template.set_option(section, 'last_updated', str(datetime.utcnow()) + ' UTC') if multi_instance: set_instances(template, section, 'failed') status = ( False, 'Failed to build image because: {0}'.format(str(e))) chdir(cwd) template.set_option(section, 'running', 'no') return status, template def _clone(self, user=None, pw=None): status = (True, None) try: # if path already exists, ignore try: path, _, _ = self.path_dirs.get_path(self.repo) chdir(path) return status except Exception as e: # pragma: no cover self.logger.debug("Repo doesn't exist, attempting to clone.") status = self.path_dirs.apply_path(self.repo) if not status[0]: self.logger.error( 'Unable to clone because: {0}'.format(str(status[1]))) return status repo = self.repo # check if user and pw were supplied, typically for private repos if user and pw: # only https is supported when using user/pw auth_repo = 'https://' + user + ':' + pw + '@' repo = auth_repo + repo.split('https://')[-1] # clone repo check_output(shlex.split('env GIT_SSL_NO_VERIFY=true git clone --recursive ' + repo + ' .'), stderr=STDOUT, close_fds=True).decode('utf-8') chdir(status[1]) status = (True, 'Successfully cloned: {0}'.format(self.repo)) except Exception as e: # pragma: no cover e_str = str(e) # scrub username and password from error message if e_str.find('@') >= 0: e_str = e_str[:e_str.find('//') + 2] + \ e_str[e_str.find('@') + 1:] self.logger.error('Clone failed with error: {0}'.format(e_str)) status = (False, e_str) return status def update(self, repo, tools=None): # TODO return
def create(self): """ Override method for creating FormBaseNewWithMenu form """ try: self.api_action = System() except DockerException as de: # pragma: no cover notify_confirm(str(de), title='Docker Error', form_color='DANGER', wrap=True) MainForm.exit() self.add_handlers({'^T': self.help_form, '^Q': MainForm.exit}) # all forms that can toggle view by group self.view_togglable = ['inventory', 'remove'] ####################### # MAIN SCREEN WIDGETS # ####################### self.addfield = self.add(npyscreen.TitleFixedText, name='Date:', labelColor='DEFAULT', value=Timestamp()) self.addfield2 = self.add(npyscreen.TitleFixedText, name='Uptime:', labelColor='DEFAULT', value=Uptime()) self.cpufield = self.add(npyscreen.TitleFixedText, name='Logical CPUs:', labelColor='DEFAULT', value=Cpu()) self.gpufield = self.add(npyscreen.TitleFixedText, name='GPUs:', labelColor='DEFAULT', value=Gpu()[1]) self.location = self.add(npyscreen.TitleFixedText, name='User Data:', value=PathDirs().meta_dir, labelColor='DEFAULT') self.file_drop = self.add(npyscreen.TitleFixedText, name='File Drop:', value=DropLocation()[1], labelColor='DEFAULT') self.addfield3 = self.add(npyscreen.TitleFixedText, name='Containers:', labelColor='DEFAULT', value='0 ' + ' running') self.multifield1 = self.add(npyscreen.MultiLineEdit, max_height=22, editable=False, value=""" '., 'b * '$ #. $: #: *# @): :@,@): ,.**:' , :@@*: ..**' '#o. .:(@'.@*"' 'bq,..:,@@*' ,* ,p$q8,:@)' .p*' ' '@@Pp@@*' Y7'.' :@):. .:@:'. .::(@:. _ __ _____ _ __ | |_ \ \ / / _ \ '_ \| __| \ V / __/ | | | |_ \_/ \___|_| |_|\__| """) ################ # MENU OPTIONS # ################ # Tool Menu Items self.m3 = self.add_menu(name='Tools', shortcut='p') self.m3.addItem(text='Add New Tool', onSelect=self.perform_action, arguments=['add'], shortcut='a') self.m3.addItem(text='Configure Tools', onSelect=self.perform_action, arguments=['configure'], shortcut='t') self.m3.addItem(text='Inventory', onSelect=self.perform_action, arguments=['inventory'], shortcut='i') self.m3.addItem(text='Remove Tools', onSelect=self.perform_action, arguments=['remove'], shortcut='r') self.m3.addItem(text='Start Tools', onSelect=self.perform_action, arguments=['start'], shortcut='s') self.m3.addItem(text='Stop Tools', onSelect=self.perform_action, arguments=['stop'], shortcut='p') # Services Menu Items self.m5 = self.add_menu(name='Services Running', shortcut='s') self.m5.addItem(text='External Services', onSelect=self.perform_action, arguments=['services_external'], shortcut='e') self.m5.addItem(text='Tool Services', onSelect=self.perform_action, arguments=['services'], shortcut='t') # System Commands Menu Items self.m6 = self.add_menu(name='System Commands', shortcut='y') self.m6.addItem(text='Backup', onSelect=self.system_commands, arguments=['backup'], shortcut='b') self.m6.addItem(text='Change Vent Configuration', onSelect=self.system_commands, arguments=['configure'], shortcut='c') self.m6.addItem(text='Detect GPUs', onSelect=self.system_commands, arguments=['gpu'], shortcut='g') self.m6.addItem(text='Factory Reset', onSelect=self.system_commands, arguments=['reset'], shortcut='r') self.m6.addItem(text='Restore (To Be Implemented...', onSelect=self.system_commands, arguments=['restore'], shortcut='t') # TODO this should be either or depending on whether or not it's running already self.m6.addItem(text='Start', onSelect=self.system_commands, arguments=['start'], shortcut='s') self.m6.addItem(text='Stop', onSelect=self.system_commands, arguments=['stop'], shortcut='o') self.m6.addItem(text='Upgrade (To Be Implemented...)', onSelect=self.system_commands, arguments=['upgrade'], shortcut='u') # Tutorial Menu Items self.m7 = self.add_menu(name='Tutorials', shortcut='t') self.s1 = self.m7.addNewSubmenu(name='About Vent', shortcut='v') self.s1.addItem(text='Background', onSelect=self.switch_tutorial, arguments=['background'], shortcut='b') self.s1.addItem(text='Terminology', onSelect=self.switch_tutorial, arguments=['terminology'], shortcut='t') self.s1.addItem(text='Getting Setup', onSelect=self.switch_tutorial, arguments=['setup'], shortcut='s') self.s2 = self.m7.addNewSubmenu(name='Working with Tools', shortcut='c') self.s2.addItem(text='Starting Tools', onSelect=self.switch_tutorial, arguments=['starting_tools'], shortcut='s') self.s3 = self.m7.addNewSubmenu(name='Working with Plugins', shortcut='p') self.s3.addItem(text='Adding Tools', onSelect=self.switch_tutorial, arguments=['adding_tools'], shortcut='a') self.s4 = self.m7.addNewSubmenu(name='Files', shortcut='f') self.s4.addItem(text='Adding Files', onSelect=self.switch_tutorial, arguments=['adding_files'], shortcut='a') self.s5 = self.m7.addNewSubmenu(name='Help', shortcut='s') self.s5.addItem(text='Basic Troubleshooting', onSelect=self.switch_tutorial, arguments=['basic_troubleshooting'], shortcut='t')
class System: def __init__(self, *args, **kwargs): self.d_client = docker.from_env() self.path_dirs = PathDirs(**kwargs) self.manifest = join(self.path_dirs.meta_dir, 'plugin_manifest.cfg') self.vent_config = self.path_dirs.cfg_file self.startup_file = self.path_dirs.startup_file self.logger = Logger(__name__) self._auto_install() def _auto_install(self): """ Automatically detects images and installs them in the manifest if they are not there already """ template = Template(template=self.manifest) sections = template.sections() images = self.d_client.images.list(filters={'label': 'vent'}) add_sections = [] status = (True, None) for image in images: ignore = False if ('Labels' in image.attrs['Config'] and 'vent.section' in image.attrs['Config']['Labels'] and not image.attrs['Config']['Labels']['vent.section'] in sections[1]): section = image.attrs['Config']['Labels']['vent.section'] section_str = image.attrs['Config']['Labels']['vent.section'].split( ':') template.add_section(section) if 'vent.name' in image.attrs['Config']['Labels']: template.set_option(section, 'name', image.attrs['Config']['Labels']['vent.name']) if 'vent.repo' in image.attrs['Config']['Labels']: template.set_option(section, 'repo', image.attrs['Config']['Labels']['vent.repo']) git_path = join(self.path_dirs.plugins_dir, '/'.join(section_str[:2])) # TODO clone it down template.set_option(section, 'path', join( git_path, section_str[-3][1:])) # get template settings # TODO account for template files not named vent.template v_template = Template(template=join( git_path, section_str[-3][1:], 'vent.template')) tool_sections = v_template.sections() if tool_sections[0]: for s in tool_sections[1]: section_dict = {} options = v_template.options(s) if options[0]: for option in options[1]: option_name = option if option == 'name': # get link name template.set_option(section, 'link_name', v_template.option(s, option)[1]) option_name = 'link_name' opt_val = v_template.option(s, option)[1] section_dict[option_name] = opt_val if section_dict: template.set_option(section, s, json.dumps(section_dict)) if ('vent.type' in image.attrs['Config']['Labels'] and image.attrs['Config']['Labels']['vent.type'] == 'repository'): template.set_option( section, 'namespace', '/'.join(section_str[:2])) template.set_option(section, 'branch', section_str[-2]) template.set_option(section, 'version', section_str[-1]) template.set_option(section, 'last_updated', str( datetime.utcnow()) + ' UTC') if image.attrs['RepoTags']: template.set_option( section, 'image_name', image.attrs['RepoTags'][0]) else: # image with none tag is outdated, don't add it ignore = True template.set_option(section, 'type', 'repository') if 'vent.groups' in image.attrs['Config']['Labels']: template.set_option(section, 'groups', image.attrs['Config']['Labels']['vent.groups']) template.set_option(section, 'built', 'yes') template.set_option(section, 'image_id', image.attrs['Id'].split(':')[1][:12]) template.set_option(section, 'running', 'no') # check if image is running as a container containers = self.d_client.containers.list( filters={'label': 'vent'}) for container in containers: if container.attrs['Image'] == image.attrs['Id']: template.set_option(section, 'running', 'yes') if not ignore: add_sections.append(section) template.write_config() # TODO this check will always be true, need to actually validate the above logic if status[0]: status = (True, add_sections) return status def backup(self): """ Saves the configuration information of the current running vent instance to be used for restoring at a later time """ status = (True, None) # initialize all needed variables (names for backup files, etc.) backup_name = ('.vent-backup-' + '-'.join(Timestamp().split(' '))) backup_dir = join(expanduser('~'), backup_name) backup_manifest = join(backup_dir, 'backup_manifest.cfg') backup_vcfg = join(backup_dir, 'backup_vcfg.cfg') manifest = self.manifest # create new backup directory try: mkdir(backup_dir) except Exception as e: # pragma: no cover self.logger.error(str(e)) return (False, str(e)) # create new files in backup directory try: # backup manifest with open(backup_manifest, 'w') as bmanifest: with open(manifest, 'r') as manifest_file: bmanifest.write(manifest_file.read()) # backup vent.cfg with open(backup_vcfg, 'w') as bvcfg: with open(self.vent_config, 'r') as vcfg_file: bvcfg.write(vcfg_file.read()) self.logger.info('Backup information written to ' + backup_dir) status = (True, backup_dir) except Exception as e: # pragma: no cover self.logger.error("Couldn't backup vent: " + str(e)) status = (False, str(e)) # TODO #266 return status def configure(self): # TODO return def gpu(self): # TODO return def history(self): # TODO #255 return def restore(self, backup_dir): """ Restores a vent configuration from a previously backed up version """ # TODO #266 status = (True, None) return status def reset(self): """ Factory reset all of Vent's user data, containers, and images """ status = (True, None) error_message = '' # remove containers try: c_list = set(self.d_client.containers.list( filters={'label': 'vent'}, all=True)) for c in c_list: c.remove(force=True) except Exception as e: # pragma: no cover error_message += 'Error removing Vent containers: ' + str(e) + '\n' # remove images try: i_list = set(self.d_client.images.list(filters={'label': 'vent'}, all=True)) for i in i_list: # delete tagged images only because they are the parents for # the untagged images. Remove the parents and the children get # removed automatically if i.attrs['RepoTags']: self.d_client.images.remove(image=i.id, force=True) except Exception as e: # pragma: no cover error_message += 'Error deleting Vent images: ' + str(e) + '\n' # remove .vent folder try: cwd = getcwd() if cwd.startswith(join(expanduser('~'), '.vent')): chdir(expanduser('~')) shutil.rmtree(join(expanduser('~'), '.vent')) except Exception as e: # pragma: no cover error_message += 'Error deleting Vent data: ' + str(e) + '\n' if error_message: status = (False, error_message) return status def rollback(self): # TODO #266 return def start(self): status = (True, None) # startup based on startup file if exists(self.startup_file): status = self._startup() else: tools = Tools() status = tools.new('core', None) if status[0]: status = tools.start( 'https://github.com/cyberreboot/vent', None) return status def _startup(self): """ Automatically detect if a startup file is specified and stand up a vent host with all necessary tools based on the specifications in that file """ status = (True, None) try: s_dict = {} # rewrite the yml file to exclusively lowercase with open(self.startup_file, 'r') as sup: vent_startup = sup.read() with open(self.startup_file, 'w') as sup: for line in vent_startup: sup.write(line.lower()) with open(self.startup_file, 'r') as sup: s_dict = yaml.safe_load(sup.read()) if 'vent.cfg' in s_dict: v_cfg = Template(self.vent_config) for section in s_dict['vent.cfg']: for option in s_dict['vent.cfg'][section]: val = ('no', 'yes')[ s_dict['vent.cfg'][section][option]] v_status = v_cfg.add_option(section, option, value=val) if not v_status[0]: v_cfg.set_option(section, option, val) v_cfg.write_config() del s_dict['vent.cfg'] tool_d = {} extra_options = ['info', 'service', 'settings', 'docker', 'gpu'] s_dict_c = copy.deepcopy(s_dict) # TODO check for repo or image type for repo in s_dict_c: repository = Repository(System().manifest) repository.repo = repo repository._clone() repo_path, org, r_name = self.path_dirs.get_path(repo) get_tools = [] for tool in s_dict_c[repo]: t_branch = 'master' t_version = 'HEAD' if 'branch' in s_dict[repo][tool]: t_branch = s_dict[repo][tool]['branch'] if 'version' in s_dict[repo][tool]: t_version = s_dict[repo][tool]['version'] get_tools.append((tool, t_branch, t_version)) available_tools = AvailableTools(repo_path, tools=get_tools) for tool in s_dict_c[repo]: # if we can't find the tool in that repo, skip over this # tool and notify in the logs t_path, t_path_cased = PathDirs.rel_path( tool, available_tools) if t_path is None: self.logger.error("Couldn't find tool " + tool + ' in' ' repo ' + repo) continue # ensure no NoneType iteration errors if s_dict_c[repo][tool] is None: s_dict[repo][tool] = {} # check if we need to configure instances along the way instances = 1 if 'settings' in s_dict[repo][tool]: if 'instances' in s_dict[repo][tool]['settings']: instances = int(s_dict[repo][tool] ['settings']['instances']) # add the tool t_branch = 'master' t_version = 'HEAD' t_image = None add_tools = None add_tools = [(t_path_cased, '')] if 'branch' in s_dict[repo][tool]: t_branch = s_dict[repo][tool]['branch'] if 'version' in s_dict[repo][tool]: t_version = s_dict[repo][tool]['version'] if 'image' in s_dict[repo][tool]: t_image = s_dict[repo][tool]['image'] repository.add( repo, add_tools, branch=t_branch, version=t_version, image_name=t_image) manifest = Template(self.manifest) # update the manifest with extra defined runtime settings base_section = ':'.join([org, r_name, t_path, t_branch, t_version]) for option in extra_options: if option in s_dict[repo][tool]: opt_dict = manifest.option(base_section, option) # add new values defined into default options for # that tool, don't overwrite them if opt_dict[0]: opt_dict = json.loads(opt_dict[1]) else: opt_dict = {} # stringify values for vent for v in s_dict[repo][tool][option]: pval = s_dict[repo][tool][option][v] s_dict[repo][tool][option][v] = json.dumps( pval) opt_dict.update(s_dict[repo][tool][option]) manifest.set_option(base_section, option, json.dumps(opt_dict)) # copy manifest info into new sections if necessary if instances > 1: for i in range(2, instances + 1): i_section = base_section.rsplit(':', 2) i_section[0] += str(i) i_section = ':'.join(i_section) manifest.add_section(i_section) for opt_val in manifest.section(base_section)[1]: if opt_val[0] == 'name': manifest.set_option(i_section, opt_val[0], opt_val[1] + str(i)) else: manifest.set_option(i_section, opt_val[0], opt_val[1]) manifest.write_config() tool_d = {} tools = Tools() # start tools, if necessary for repo in s_dict: for tool in s_dict[repo]: if 'start' in s_dict[repo][tool]: if s_dict[repo][tool]['start']: local_instances = 1 if 'settings' in s_dict[repo][tool] and 'instances' in s_dict[repo][tool]['settings']: local_instances = int( s_dict[repo][tool]['settings']['instances']) t_branch = 'master' t_version = 'HEAD' if 'branch' in s_dict[repo][tool]: t_branch = s_dict[repo][tool]['branch'] if 'version' in s_dict[repo][tool]: t_version = s_dict[repo][tool]['version'] for i in range(1, local_instances + 1): i_name = tool + str(i) if i != 1 else tool i_name = i_name.replace('@', '') tool_d.update( tools._prep_start(repo, i_name)[1]) if tool_d: tools.start(tool_d, None, is_tool_d=True) except Exception as e: # pragma: no cover self.logger.error('Startup failed because: {0}'.format(str(e))) status = (False, str(e)) return status def stop(self): status = (True, None) # remove containers try: c_list = set(self.d_client.containers.list( filters={'label': 'vent'}, all=True)) for c in c_list: c.remove(force=True) except Exception as e: # pragma: no cover status = (False, str(e)) return status def get_configure(self, repo=None, name=None, groups=None, main_cfg=False): """ Get the vent.template settings for a given tool by looking at the plugin_manifest """ constraints = locals() del constraints['main_cfg'] status = (True, None) template_dict = {} return_str = '' if main_cfg: vent_cfg = Template(self.vent_config) for section in vent_cfg.sections()[1]: template_dict[section] = {} for vals in vent_cfg.section(section)[1]: template_dict[section][vals[0]] = vals[1] else: # all possible vent.template options stored in plugin_manifest options = ['info', 'service', 'settings', 'docker', 'gpu'] tools = Template(System().manifest).constrain_opts( constraints, options)[0] if tools: # should only be one tool tool = list(tools.keys())[0] # load all vent.template options into dict for section in tools[tool]: template_dict[section] = json.loads(tools[tool][section]) else: status = (False, "Couldn't get vent.template information") if status[0]: # display all those options as they would in the file for section in template_dict: return_str += '[' + section + ']\n' # ensure instances shows up in configuration for option in template_dict[section]: if option.startswith('#'): return_str += option + '\n' else: return_str += option + ' = ' return_str += template_dict[section][option] + '\n' return_str += '\n' # only one newline at end of file status = (True, return_str[:-1]) return status def save_configure(self, repo=None, name=None, groups=None, config_val='', from_registry=False, main_cfg=False, instances=1): """ Save changes made to vent.template through npyscreen to the template and to plugin_manifest """ def template_to_manifest(vent_template, manifest, tool, instances): """ Helper function to transfer information from vent.template to plugin_manifest """ sections = vent_template.sections() if sections[0]: for section in sections[1]: section_dict = {} if section == 'settings': section_dict.update({'instances': str(instances)}) options = vent_template.options(section) if options[0]: for option in options[1]: option_name = option if option == 'name': option_name = 'link_name' opt_val = vent_template.option(section, option)[1] section_dict[option_name] = opt_val if section_dict: manifest.set_option(tool, section, json.dumps(section_dict)) elif manifest.option(tool, section)[0]: manifest.del_option(tool, section) constraints = locals() del constraints['config_val'] del constraints['from_registry'] del constraints['main_cfg'] del constraints['instances'] del constraints['template_to_manifest'] status = (True, None) fd = None # ensure instances is an int and remove instances from config_val to # ensure correct info instances = int(instances) config_val = re.sub(r'instances\ *=\ *\d+\n', '', config_val) api_system = System() manifest = api_system.manifest if not main_cfg: if not from_registry: # creating new instances if instances > 1: fd, template_path = tempfile.mkstemp(suffix='.template') # scrub name for clean section name if re.search(r'\d+$', name): name = re.sub(r'\d+$', '', name) t_identifier = {'name': name, 'branch': branch, 'version': version} result = Template(manifest).constrain_opts( t_identifier, []) tools = result[0] tool = list(tools.keys())[0] else: options = ['path', 'multi_tool', 'name'] tools, _ = Template(manifest).constrain_opts( constraints, options) # only one tool in tools because perform this function for # every tool if tools: tool = list(tools.keys())[0] if ('multi_tool' in tools[tool] and tools[tool]['multi_tool'] == 'yes'): name = tools[tool]['name'] if name == 'unspecified': name = 'vent' template_path = join(tools[tool]['path'], name+'.template') else: template_path = join(tools[tool]['path'], 'vent.template') else: status = (False, "Couldn't save configuration") else: fd, template_path = tempfile.mkstemp(suffix='.template') options = ['namespace'] constraints.update({'type': 'registry'}) tools, _ = Template(manifest).constrain_opts(constraints, options) if tools: tool = list(tools.keys())[0] else: status = (False, "Couldn't save configuration") if status[0]: try: # save in vent.template with open(template_path, 'w') as f: f.write(config_val) # save in plugin_manifest vent_template = Template(template_path) manifest = Template(manifest) if instances > 1: # add instances as needed for i in range(1, instances + 1): i_section = tool.rsplit(':', 2) i_section[0] += str(i) if i != 1 else '' i_section = ':'.join(i_section) if not manifest.section(i_section)[0]: manifest.add_section(i_section) for val_pair in manifest.section(tool)[1]: name = val_pair[0] val = val_pair[1] if name == 'name': val += str(i) elif name == 'last_updated': val = Timestamp() elif name == 'running': val = 'no' manifest.set_option(i_section, name, val) template_to_manifest(vent_template, manifest, i_section, instances) else: settings = manifest.option(i_section, 'settings') if settings[0]: settings_dict = json.loads(settings[1]) settings_dict['instances'] = str(instances) manifest.set_option(i_section, 'settings', json.dumps( settings_dict)) else: inst = str(instances) settings_dict = {'instances': inst} manifest.set_option(i_section, 'settings', json.dumps( settings_dict)) else: try: settings_str = manifest.option(tool, 'settings')[1] settings_dict = json.loads(settings_str) old_instances = int(settings_dict['instances']) except Exception: old_instances = 1 template_to_manifest(vent_template, manifest, tool, old_instances) manifest.write_config() status = (True, manifest) except Exception as e: # pragma: no cover self.logger.error('save_configure error: ' + str(e)) status = (False, str(e)) # close os file handle and remove temp file if from_registry or instances > 1: try: close(fd) remove(template_path) except Exception as e: # pragma: no cover self.logger.error('save_configure error: ' + str(e)) else: with open(self.vent_config, 'w') as f: f.write(config_val) return status def restart_tools(self, repo=None, name=None, groups=None, branch='master', version='HEAD', main_cfg=False, old_val='', new_val=''): """ Restart necessary tools based on changes that have been made either to vent.cfg or to vent.template. This includes tools that need to be restarted because they depend on other tools that were changed. """ status = (True, None) if not main_cfg: try: t_identifier = {'name': name, 'branch': branch, 'version': version} result = Template(System().manifest).constrain_opts(t_identifier, ['running', 'link_name']) tools = result[0] tool = list(tools.keys())[0] if ('running' in tools[tool] and tools[tool]['running'] == 'yes'): start_tools = [t_identifier] dependent_tools = [tools[tool]['link_name']] start_tools += Dependencies(dependent_tools) # TODO start_d = {} for tool_identifier in start_tools: self.clean(**tool_identifier) start_d.update(self.prep_start(**tool_identifier)[1]) if start_d: Tools().start(start_d, '', is_tool_d=True) except Exception as e: # pragma: no cover self.logger.error('Trouble restarting tool ' + name + ' because: ' + str(e)) status = (False, str(e)) else: try: # string manipulation to get tools into arrays ext_start = old_val.find('[external-services]') if ext_start >= 0: ot_str = old_val[old_val.find('[external-services]') + 20:] else: ot_str = '' old_tools = [] for old_tool in ot_str.split('\n'): if old_tool != '': old_tools.append(old_tool.split('=')[0].strip()) ext_start = new_val.find('[external-services]') if ext_start >= 0: nt_str = new_val[new_val.find('[external-services]') + 20:] else: nt_str = '' new_tools = [] for new_tool in nt_str.split('\n'): if new_tool != '': new_tools.append(new_tool.split('=')[0].strip()) # find tools changed tool_changes = [] for old_tool in old_tools: if old_tool not in new_tools: tool_changes.append(old_tool) for new_tool in new_tools: if new_tool not in old_tools: tool_changes.append(new_tool) else: # tool name will be the same oconf = old_val[old_val.find(new_tool):].split('\n')[0] nconf = new_val[new_val.find(new_tool):].split('\n')[0] if oconf != nconf: tool_changes.append(new_tool) # put link names in a dictionary for finding dependencies dependent_tools = [] for i, entry in enumerate(tool_changes): dependent_tools.append(entry) # change names to lowercase for use in clean, prep_start tool_changes[i] = {'name': entry.lower().replace('-', '_')} dependencies = Dependencies(dependent_tools) # restart tools restart = tool_changes + dependencies tool_d = {} for tool in restart: self.clean(**tool) tool_d.update(self.prep_start(**tool)[1]) if tool_d: # TODO fix the arguments Tools().start(tool_d) except Exception as e: # pragma: no cover self.logger.error('Problem restarting tools: ' + str(e)) status = (False, str(e)) return status def upgrade(self): ''' Upgrades Vent itself, and core containers ''' # TODO return
def test_pathdirs(): """ Test the pathdirs class """ path = PathDirs() path.host_config()
def _startup(self): """ Automatically detect if a startup file is specified and stand up a vent host with all necessary tools based on the specifications in that file """ status = (True, None) try: s_dict = {} # rewrite the yml file to exclusively lowercase with open(self.startup_file, 'r') as sup: vent_startup = sup.read() with open(self.startup_file, 'w') as sup: for line in vent_startup: sup.write(line.lower()) with open(self.startup_file, 'r') as sup: s_dict = yaml.safe_load(sup.read()) if 'vent.cfg' in s_dict: v_cfg = Template(self.vent_config) for section in s_dict['vent.cfg']: for option in s_dict['vent.cfg'][section]: val = ('no', 'yes')[ s_dict['vent.cfg'][section][option]] v_status = v_cfg.add_option(section, option, value=val) if not v_status[0]: v_cfg.set_option(section, option, val) v_cfg.write_config() del s_dict['vent.cfg'] tool_d = {} extra_options = ['info', 'service', 'settings', 'docker', 'gpu'] s_dict_c = copy.deepcopy(s_dict) # TODO check for repo or image type for repo in s_dict_c: repository = Repository(System().manifest) repository.repo = repo repository._clone() repo_path, org, r_name = self.path_dirs.get_path(repo) get_tools = [] for tool in s_dict_c[repo]: t_branch = 'master' t_version = 'HEAD' if 'branch' in s_dict[repo][tool]: t_branch = s_dict[repo][tool]['branch'] if 'version' in s_dict[repo][tool]: t_version = s_dict[repo][tool]['version'] get_tools.append((tool, t_branch, t_version)) available_tools = AvailableTools(repo_path, tools=get_tools) for tool in s_dict_c[repo]: # if we can't find the tool in that repo, skip over this # tool and notify in the logs t_path, t_path_cased = PathDirs.rel_path( tool, available_tools) if t_path is None: self.logger.error("Couldn't find tool " + tool + ' in' ' repo ' + repo) continue # ensure no NoneType iteration errors if s_dict_c[repo][tool] is None: s_dict[repo][tool] = {} # check if we need to configure instances along the way instances = 1 if 'settings' in s_dict[repo][tool]: if 'instances' in s_dict[repo][tool]['settings']: instances = int(s_dict[repo][tool] ['settings']['instances']) # add the tool t_branch = 'master' t_version = 'HEAD' t_image = None add_tools = None add_tools = [(t_path_cased, '')] if 'branch' in s_dict[repo][tool]: t_branch = s_dict[repo][tool]['branch'] if 'version' in s_dict[repo][tool]: t_version = s_dict[repo][tool]['version'] if 'image' in s_dict[repo][tool]: t_image = s_dict[repo][tool]['image'] repository.add( repo, add_tools, branch=t_branch, version=t_version, image_name=t_image) manifest = Template(self.manifest) # update the manifest with extra defined runtime settings base_section = ':'.join([org, r_name, t_path, t_branch, t_version]) for option in extra_options: if option in s_dict[repo][tool]: opt_dict = manifest.option(base_section, option) # add new values defined into default options for # that tool, don't overwrite them if opt_dict[0]: opt_dict = json.loads(opt_dict[1]) else: opt_dict = {} # stringify values for vent for v in s_dict[repo][tool][option]: pval = s_dict[repo][tool][option][v] s_dict[repo][tool][option][v] = json.dumps( pval) opt_dict.update(s_dict[repo][tool][option]) manifest.set_option(base_section, option, json.dumps(opt_dict)) # copy manifest info into new sections if necessary if instances > 1: for i in range(2, instances + 1): i_section = base_section.rsplit(':', 2) i_section[0] += str(i) i_section = ':'.join(i_section) manifest.add_section(i_section) for opt_val in manifest.section(base_section)[1]: if opt_val[0] == 'name': manifest.set_option(i_section, opt_val[0], opt_val[1] + str(i)) else: manifest.set_option(i_section, opt_val[0], opt_val[1]) manifest.write_config() tool_d = {} tools = Tools() # start tools, if necessary for repo in s_dict: for tool in s_dict[repo]: if 'start' in s_dict[repo][tool]: if s_dict[repo][tool]['start']: local_instances = 1 if 'settings' in s_dict[repo][tool] and 'instances' in s_dict[repo][tool]['settings']: local_instances = int( s_dict[repo][tool]['settings']['instances']) t_branch = 'master' t_version = 'HEAD' if 'branch' in s_dict[repo][tool]: t_branch = s_dict[repo][tool]['branch'] if 'version' in s_dict[repo][tool]: t_version = s_dict[repo][tool]['version'] for i in range(1, local_instances + 1): i_name = tool + str(i) if i != 1 else tool i_name = i_name.replace('@', '') tool_d.update( tools._prep_start(repo, i_name)[1]) if tool_d: tools.start(tool_d, None, is_tool_d=True) except Exception as e: # pragma: no cover self.logger.error('Startup failed because: {0}'.format(str(e))) status = (False, str(e)) return status
def create(self): """ Override method for creating FormBaseNewWithMenu form """ try: self.api_action = Action() except DockerException as de: # pragma: no cover notify_confirm(str(de), title='Docker Error', form_color='DANGER', wrap=True) MainForm.exit() self.add_handlers({'^T': self.help_form, '^Q': MainForm.exit}) # all forms that can toggle view by group self.view_togglable = [ 'inventory', 'remove', 'update', 'enable', 'disable', 'build' ] ####################### # MAIN SCREEN WIDGETS # ####################### self.addfield = self.add(npyscreen.TitleFixedText, name='Date:', labelColor='DEFAULT', value=Timestamp()) self.addfield2 = self.add(npyscreen.TitleFixedText, name='Uptime:', labelColor='DEFAULT', value=Uptime()) self.cpufield = self.add(npyscreen.TitleFixedText, name='Logical CPUs:', labelColor='DEFAULT', value=Cpu()) self.gpufield = self.add(npyscreen.TitleFixedText, name='GPUs:', labelColor='DEFAULT', value=Gpu()[1]) self.location = self.add(npyscreen.TitleFixedText, name='User Data:', value=PathDirs().meta_dir, labelColor='DEFAULT') self.file_drop = self.add(npyscreen.TitleFixedText, name='File Drop:', value=DropLocation()[1], labelColor='DEFAULT') self.addfield3 = self.add(npyscreen.TitleFixedText, name='Containers:', labelColor='DEFAULT', value='0 ' + ' running') self.addfield4 = self.add(npyscreen.TitleFixedText, name='Status:', labelColor='CAUTION', value='Idle') self.addfield5 = self.add(npyscreen.TitleFixedText, name='Core Tools:', labelColor='DANGER', value='Not built') self.addfield6 = self.add(npyscreen.TitleFixedText, name='Plugin Tools:', labelColor='DEFAULT', value='Not built') self.addfield7 = self.add(npyscreen.TitleFixedText, name='Jobs:', value='0 jobs running (0 tool containers),' ' 0 completed jobs', labelColor='DEFAULT') self.multifield1 = self.add(npyscreen.MultiLineEdit, max_height=22, editable=False, value=""" '., 'b * '$ #. $: #: *# @): :@,@): ,.**:' , :@@*: ..**' '#o. .:(@'.@*"' 'bq,..:,@@*' ,* ,p$q8,:@)' .p*' ' '@@Pp@@*' Y7'.' :@):. .:@:'. .::(@:. _ __ _____ _ __ | |_ \ \ / / _ \ '_ \| __| \ V / __/ | | | |_ \_/ \___|_| |_|\__| """) ################ # MENU OPTIONS # ################ # Core Tools Menu Items self.m2 = self.add_menu(name='Core Tools', shortcut='c') self.m2.addItem(text='Add all latest core tools', onSelect=MainForm.core_tools, arguments=['install'], shortcut='i') self.m2.addItem(text='Build core tools', onSelect=self.perform_action, arguments=['build_core'], shortcut='b') self.m2.addItem(text='Clean core tools', onSelect=self.perform_action, arguments=['clean_core'], shortcut='c') self.m2.addItem(text='Configure core tools', onSelect=self.perform_action, arguments=['configure_core'], shortcut='t') self.m2.addItem(text='Disable core tools', onSelect=self.perform_action, arguments=['disable_core'], shortcut='d') self.m2.addItem(text='Enable core tools', onSelect=self.perform_action, arguments=['enable_core'], shortcut='e') self.m2.addItem(text='Inventory of core tools', onSelect=self.perform_action, arguments=['inventory_core'], shortcut='v') self.m2.addItem(text='Remove core tools', onSelect=self.perform_action, arguments=['remove_core'], shortcut='r') self.m2.addItem(text='Start core tools', onSelect=self.perform_action, arguments=['start_core'], shortcut='s') self.m2.addItem(text='Stop core tools', onSelect=self.perform_action, arguments=['stop_core'], shortcut='p') self.m2.addItem(text='Update core tools', onSelect=self.perform_action, arguments=['update_core'], shortcut='u') # Plugin Menu Items self.m3 = self.add_menu(name='Plugins', shortcut='p') self.m3.addItem(text='Add new plugin', onSelect=self.perform_action, arguments=['add'], shortcut='a') self.m3.addItem(text='Build plugin tools', onSelect=self.perform_action, arguments=['build'], shortcut='b') self.m3.addItem(text='Clean plugin tools', onSelect=self.perform_action, arguments=['clean'], shortcut='c') self.m3.addItem(text='Configure plugin tools', onSelect=self.perform_action, arguments=['configure'], shortcut='t') self.m3.addItem(text='Disable plugin tools', onSelect=self.perform_action, arguments=['disable'], shortcut='d') self.m3.addItem(text='Enable plugin tools', onSelect=self.perform_action, arguments=['enable'], shortcut='e') self.m3.addItem(text='Inventory of installed plugins', onSelect=self.perform_action, arguments=['inventory'], shortcut='i') self.m3.addItem(text='Remove plugins', onSelect=self.perform_action, arguments=['remove'], shortcut='r') self.m3.addItem(text='Start plugin tools', onSelect=self.perform_action, arguments=['start'], shortcut='s') self.m3.addItem(text='Stop plugin tools', onSelect=self.perform_action, arguments=['stop'], shortcut='p') self.m3.addItem(text='Update plugins', onSelect=self.perform_action, arguments=['update'], shortcut='u') # Log Menu Items self.m4 = self.add_menu(name='Logs', shortcut='l') self.m4.addItem(text='Get container logs', arguments=['logs'], onSelect=self.perform_action) # Services Menu Items self.m5 = self.add_menu(name='Services Running', shortcut='s') self.m5.addItem(text='Core Services', onSelect=self.perform_action, arguments=['services_core'], shortcut='c') self.m5.addItem(text='External Services', onSelect=self.perform_action, arguments=['services_external'], shortcut='e') self.m5.addItem(text='Plugin Services', onSelect=self.perform_action, arguments=['services'], shortcut='p') # System Commands Menu Items self.m6 = self.add_menu(name='System Commands', shortcut='y') self.m6.addItem(text='Backup', onSelect=self.system_commands, arguments=['backup'], shortcut='b') self.m6.addItem(text='Change vent configuration', onSelect=self.system_commands, arguments=['configure'], shortcut='c') self.m6.addItem(text='Detect GPUs', onSelect=self.system_commands, arguments=['gpu'], shortcut='g') self.m6.addItem(text='Enable Swarm Mode (To Be Implemented...)', onSelect=self.system_commands, arguments=['swarm'], shortcut='s') self.m6.addItem(text='Factory reset', onSelect=self.system_commands, arguments=['reset'], shortcut='r') self.s6 = self.m6.addNewSubmenu(name='Network Tap Interface', shortcut='n') self.m6.addItem(text='Restore', onSelect=self.system_commands, arguments=['restore'], shortcut='t') self.m6.addItem(text='Upgrade (To Be Implemented...)', onSelect=self.system_commands, arguments=['upgrade'], shortcut='u') self.s6.addItem(text='Create', onSelect=self.system_commands, shortcut='c', arguments=['ntapcreate']) self.s6.addItem(text='Delete', onSelect=self.system_commands, shortcut='d', arguments=['ntapdelete']) self.s6.addItem(text='List', onSelect=self.system_commands, shortcut='l', arguments=['ntaplist']) self.s6.addItem(text='NICs', onSelect=self.system_commands, shortcut='n', arguments=['ntapnics']) self.s6.addItem(text='Start', onSelect=self.system_commands, shortcut='s', arguments=['ntapstart']) self.s6.addItem(text='Stop', onSelect=self.system_commands, shortcut='t', arguments=['ntapstop']) # Tutorial Menu Items self.m7 = self.add_menu(name='Tutorials', shortcut='t') self.s1 = self.m7.addNewSubmenu(name='About Vent', shortcut='v') self.s1.addItem(text='Background', onSelect=self.switch_tutorial, arguments=['background'], shortcut='b') self.s1.addItem(text='Terminology', onSelect=self.switch_tutorial, arguments=['terminology'], shortcut='t') self.s1.addItem(text='Getting Setup', onSelect=self.switch_tutorial, arguments=['setup'], shortcut='s') self.s2 = self.m7.addNewSubmenu(name='Working with Cores', shortcut='c') self.s2.addItem(text='Building Cores', onSelect=self.switch_tutorial, arguments=['building_cores'], shortcut='b') self.s2.addItem(text='Starting Cores', onSelect=self.switch_tutorial, arguments=['starting_cores'], shortcut='c') self.s3 = self.m7.addNewSubmenu(name='Working with Plugins', shortcut='p') self.s3.addItem(text='Adding Plugins', onSelect=self.switch_tutorial, arguments=['adding_plugins'], shortcut='a') self.s4 = self.m7.addNewSubmenu(name='Files', shortcut='f') self.s4.addItem(text='Adding Files', onSelect=self.switch_tutorial, arguments=['adding_files'], shortcut='a') self.s5 = self.m7.addNewSubmenu(name='Help', shortcut='s') self.s5.addItem(text='Basic Troubleshooting', onSelect=self.switch_tutorial, arguments=['basic_troubleshooting'], shortcut='t')