def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) # snippets_dir = Path(os.path.join(settings.SRC_PATH, 'panhandler', 'snippets')) snippets_dir = Path(os.path.join(os.path.expanduser('~/.pan_cnc'), 'panhandler', 'repositories')) repos = cnc_utils.get_long_term_cached_value(self.app_dir, 'imported_repositories') if repos is not None: context['repos'] = repos else: repos = list() for d in snippets_dir.iterdir(): # git_dir = os.path.join(d, '.git') git_dir = d.joinpath('.git') if git_dir.exists() and git_dir.is_dir(): repo_detail = git_utils.get_repo_details(d.name, d, self.app_dir) repos.append(repo_detail) continue # cache the repos list for 1 week. this will be cleared when we import a new repository or otherwise # change the repo list somehow cnc_utils.set_long_term_cached_value(self.app_dir, 'imported_repositories', repos, 604800, 'imported_git_repos') context['repos'] = repos return context
def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) # snippets_dir = Path(os.path.join(settings.SRC_PATH, 'panhandler', 'snippets')) snippets_dir = Path( os.path.join(os.path.expanduser('~/.pan_cnc'), 'panhandler', 'repositories')) try: if not snippets_dir.exists(): messages.add_message( self.request, messages.ERROR, 'Could not load repositories from directory as it does not exists' ) context['repos'] = list() return context except PermissionError as pe: print(pe) context['repos'] = list() return context except OSError as oe: print(oe) context['repos'] = list() return context repos = cnc_utils.get_long_term_cached_value(self.app_dir, 'imported_repositories') if repos is not None: print(f'Returning cached repos') context['repos'] = repos else: repos = list() for d in snippets_dir.iterdir(): # git_dir = os.path.join(d, '.git') git_dir = d.joinpath('.git') if git_dir.exists() and git_dir.is_dir(): repo_detail = git_utils.get_repo_details( d.name, d, self.app_dir) repos.append(repo_detail) continue # cache the repos list for 1 week. this will be cleared when we import a new repository or otherwise # change the repo list somehow cnc_utils.set_long_term_cached_value(self.app_dir, 'imported_repositories', repos, 604800, 'imported_git_repos') context['repos'] = repos return context
def get_context_data(self, **kwargs): repo_name = self.kwargs['repo_name'] # we are going to keep the snippets in the snippets dir in the panhandler app # get the dir where all apps are installed src_dir = settings.SRC_PATH # get the panhandler app dir panhandler_dir = os.path.join(src_dir, 'panhandler') # get the snippets dir under that # snippets_dir = os.path.join(panhandler_dir, 'snippets') # repo_dir = os.path.join(snippets_dir, repo_name) user_dir = os.path.expanduser('~') repo_dir = os.path.join(user_dir, '.pan_cnc', 'panhandler', 'repositories', repo_name) if os.path.exists(repo_dir): repo_detail = git_utils.get_repo_details(repo_name, repo_dir, self.app_dir) else: repo_detail = dict() repo_detail['name'] = 'Repository directory not found' try: snippets_from_repo = snippet_utils.load_snippets_of_type_from_dir( self.app_dir, repo_dir) except CCFParserError: messages.add_message( self.request, messages.ERROR, 'Could not read all snippets from repo. Parser error') snippets_from_repo = list() # get a list of all collections found in this repo collections = list() for skillet in snippets_from_repo: if 'labels' in skillet and 'collection' in skillet['labels']: collection = skillet['labels']['collection'] if type(collection) is str: if collection not in collections: collections.append(collection) elif type(collection) is list: for collection_member in collection: if collection_member not in collections: collections.append(collection_member) # create our docker command to pass to git context = super().get_context_data(**kwargs) context['repo_detail'] = repo_detail context['repo_name'] = repo_name context['snippets'] = snippets_from_repo context['collections'] = collections return context
def get_context_data(self, **kwargs): repo_name = self.kwargs['repo_name'] user_dir = os.path.expanduser('~') repo_dir = os.path.join(user_dir, '.pan_cnc', 'panhandler', 'repositories', repo_name) if os.path.exists(repo_dir): repo_detail = git_utils.get_repo_details(repo_name, repo_dir, self.app_dir) else: repo_detail = dict() repo_detail['name'] = 'repo_name' repo_detail['error'] = 'Repository directory not found' if 'error' in repo_detail: messages.add_message(self.request, messages.ERROR, repo_detail['error']) try: snippets_from_repo = snippet_utils.load_snippets_of_type_from_dir(self.app_dir, repo_dir) except CCFParserError: messages.add_message(self.request, messages.ERROR, 'Could not read all snippets from repo. Parser error') snippets_from_repo = list() # get a list of all collections found in this repo collections = list() for skillet in snippets_from_repo: if 'labels' in skillet and 'collection' in skillet['labels']: collection = skillet['labels']['collection'] if type(collection) is str: if collection not in collections: collections.append(collection) elif type(collection) is list: for collection_member in collection: if collection_member not in collections: collections.append(collection_member) context = super().get_context_data(**kwargs) context['repo_detail'] = repo_detail context['repo_name'] = repo_name context['snippets'] = snippets_from_repo context['collections'] = collections return context
def form_valid(self, form): workflow = self.get_workflow() # get the values from the user submitted form here url = workflow.get('url') repo_name = workflow.get('repo_name') if not re.match(r'^[a-zA-Z0-9-_ \.]*$', repo_name): print('Repository name is invalid!') messages.add_message(self.request, messages.ERROR, 'Invalid Repository Name') return HttpResponseRedirect('repos') user_dir = os.path.expanduser('~/.pan_cnc') snippets_dir = os.path.join(user_dir, 'panhandler/repositories') repo_dir = os.path.join(snippets_dir, repo_name) if os.path.exists(repo_dir): if os.path.isdir(repo_dir) and len(os.listdir(repo_dir)) == 0: print('Reusing existing repository directory') else: messages.add_message( self.request, messages.ERROR, 'A Repository with this name already exists') return HttpResponseRedirect('repos') else: try: os.makedirs(repo_dir, mode=0o700) except PermissionError as pe: messages.add_message( self.request, messages.ERROR, 'Could not create repository directory, Permission Denied') return HttpResponseRedirect('repos') except OSError as ose: messages.add_message(self.request, messages.ERROR, 'Could not create repository directory') return HttpResponseRedirect('repos') # where to clone from clone_url = url.strip() if 'github' in url.lower(): details = git_utils.get_repo_upstream_details( repo_name, url, self.app_dir) if 'clone_url' in details: clone_url = details['clone_url'] try: message = git_utils.clone_repository(repo_dir, repo_name, clone_url) print(message) except ImportRepositoryException as ire: messages.add_message(self.request, messages.ERROR, f'Could not Import Repository: {ire}') else: print('Invalidating snippet cache') snippet_utils.invalidate_snippet_caches(self.app_dir) # no need to evict all these items, just grab the new repo details and append it to list and re-cache # cnc_utils.evict_cache_items_of_type(self.app_dir, 'imported_git_repos') repos = cnc_utils.get_long_term_cached_value( self.app_dir, 'imported_repositories') repo_detail = git_utils.get_repo_details(repo_name, repo_dir, self.app_dir) repos.append(repo_detail) cnc_utils.set_long_term_cached_value(self.app_dir, 'imported_repositories', repos, 604800, 'imported_git_repos') debug_errors = snippet_utils.debug_snippets_in_repo( Path(repo_dir), list()) # check each snippet found for dependencies loaded_skillets = snippet_utils.load_snippets_of_type_from_dir( self.app_dir, repo_dir) for skillet in loaded_skillets: for depends in skillet['depends']: url = depends.get('url', None) name = depends.get('name', None) branch = depends.get('branch', 'master') # now check each repo to see if we already have it, add an error if not found = False for repo in repos: if repo['url'] == url and repo['branch'] == branch: found = True break if not found: messages.add_message( self.request, messages.ERROR, f'Unresolved Dependency found!! Please ensure you import the following' f'repository: {url} with branch: {branch}') if debug_errors: messages.add_message( self.request, messages.ERROR, 'Found Skillets with errors! Please open an issue on ' 'this repository to help resolve this issue') for d in debug_errors: if 'err_list' in d and 'path' in d and 'severity' in d: for e in d['err_list']: if d['severity'] == 'warn': level = messages.WARNING else: level = messages.ERROR messages.add_message( self.request, level, f'Skillet: {d["path"]}\n\nError: {e}') else: messages.add_message(self.request, messages.INFO, 'Imported Repository Successfully') # return render(self.request, 'pan_cnc/results.html', context) return HttpResponseRedirect('repos')
def get_redirect_url(self, *args, **kwargs): repo_name = kwargs['repo_name'] branch = kwargs['branch'] user_dir = os.path.expanduser('~') repo_dir = os.path.join(user_dir, '.pan_cnc', 'panhandler', 'repositories', repo_name) if not os.path.exists(repo_dir): messages.add_message(self.request, messages.ERROR, 'Repository directory does not exist!') return f'/panhandler/repo_detail/{repo_name}' msg = git_utils.update_repo(repo_dir, branch) if 'Error' in msg: level = messages.ERROR cnc_utils.evict_cache_items_of_type(self.app_dir, 'imported_git_repos') elif 'updated' in msg or 'Checked out new' in msg: # msg updated will catch both switching branches as well as new commits # since this repoo has been updated, we need to ensure the caches are all in sync print('Invalidating snippet cache') snippet_utils.invalidate_snippet_caches(self.app_dir) git_utils.update_repo_in_cache(repo_name, repo_dir, self.app_dir) # remove all python3 init touch files if there is an update task_utils.python3_reset_init(repo_dir) level = messages.INFO else: level = messages.INFO messages.add_message(self.request, level, msg) # check if there are new branches available repo_detail = git_utils.get_repo_details(repo_name, repo_dir, self.app_dir) repo_branches = git_utils.get_repo_branches_from_dir(repo_dir) if repo_detail['branches'] != repo_branches: messages.add_message(self.request, messages.INFO, 'New Branches are available') git_utils.update_repo_in_cache(repo_name, repo_dir, self.app_dir) # check each snippet found for dependencies repos = cnc_utils.get_long_term_cached_value(self.app_dir, 'imported_repositories') loaded_skillets = snippet_utils.load_snippets_of_type_from_dir( self.app_dir, repo_dir) for skillet in loaded_skillets: for depends in skillet['depends']: url = depends.get('url', None) branch = depends.get('branch', 'master') # now check each repo to see if we already have it, add an error if not found = False for repo in repos: if repo['url'] == url and repo['branch'] == branch: found = True break if not found: messages.add_message( self.request, messages.ERROR, f'Unresolved Dependency found!! Please ensure you import the following' f' repository: {url} with branch: {branch}') debug_errors = snippet_utils.debug_snippets_in_repo( Path(repo_dir), list()) if debug_errors: messages.add_message( self.request, messages.ERROR, 'Found Skillets with errors! Please open an issue on ' 'this repository to help resolve this issue') for d in debug_errors: if 'err_list' in d and 'path' in d: for e in d['err_list']: if d['severity'] == 'warn': level = messages.WARNING else: level = messages.ERROR messages.add_message( self.request, level, f'Skillet: {d["path"]}\n\nError: {e}') return f'/panhandler/repo_detail/{repo_name}'