def diff_status(): # get latest commits of repos git_folder = app.config['GIT_REPOS_FOLDER'] rep = Repo(git_folder + '/terraform/') repowlkr = rep.get_walker(max_entries=1) lastfcommit = next(iter(repowlkr)).commit modules_repo = Repo(git_folder + '/terraform-cognite-modules') repowlkr_modules = modules_repo.get_walker(max_entries=1) lastfcommit_modules = next(iter(repowlkr_modules)).commit all_files = porcelain.ls_files(rep) state_map = {} diff_module_map = {} diff_utils.set_state_map(state_map, all_files, lastfcommit) count = 0 # For now, we are comparing everything against Greenfield. We assume Greenfield is the most up to date project for folder_path, modules in state_map['cognitedata-greenfield'].items(): for module, module_info in modules.items(): for full_module_path, git_ref in module_info.items(): dirfnm = full_module_path.encode('utf-8') repowlkr_subpath = modules_repo.get_walker(paths=[dirfnm]) repowlkr = modules_repo.get_walker() iterator = iter(repowlkr) iterator_subpath = iter(repowlkr_subpath) all_commits = [] subpath_commits = [] diff_utils.create_all_commits_list(iterator, all_commits) diff_utils.create_subpath_commits_list(iterator_subpath, subpath_commits) greenfield_commit = diff_utils.get_commit_in_subpath(git_ref, all_commits, subpath_commits) for project in state_map: if project != 'cognitedata-greenfield': project_commit = None if folder_path in state_map[project]: if module in state_map[project][folder_path]: project_commit = project + '_commit' project_commit = diff_utils.get_commit_in_subpath(state_map[project][folder_path][module][full_module_path], all_commits, subpath_commits) else: continue else: continue if project_commit is None: continue diff_utils.set_diff_module_map(project_commit, project, greenfield_commit, folder_path, module, full_module_path, subpath_commits, diff_module_map, count) count += 1 cache.set('diff_module_map', diff_module_map) return json.dumps(diff_module_map)
def _remove_untracked_files(requirements: List[_Requirement]) -> None: """ Remove files downloaded by pip which are not tracked by git. """ target_directories = set(requirement.target_directory for requirement in requirements) repo_files = ls_files(repo='.') for target_directory in target_directories: git_paths = [ item.decode() for item in repo_files if item.decode().startswith(str(target_directory)) ] for item in target_directory.iterdir(): if not [path for path in git_paths if path.startswith(str(item))]: shutil.rmtree(path=str(item))
def _remove_existing_files(requirements: List[_Requirement]) -> None: """ Remove existing files in vendored target directories. """ target_directories = set(requirement.target_directory for requirement in requirements) repo_files = ls_files(repo='.') for target_directory in target_directories: git_paths = [ item.decode() for item in repo_files if item.decode().startswith(str(target_directory)) ] remove(paths=git_paths) try: shutil.rmtree(path=str(target_directory)) except FileNotFoundError: pass
def get_latest_commits(git_folder, module_state_map): """Gets latest commit in /terraform and sets module_state_map Args: git_folder: A string of the repo path module_state_map: An empty map Returns: A nested dictionary of all terraform modules being used by all projects, path name within that project and path to that specific module. See diff_utils.set_module_state_map for example """ rep = Repo(git_folder + '/terraform/') repowlkr = rep.get_walker(max_entries=1) lastfcommit = next(iter(repowlkr)).commit all_files = porcelain.ls_files(rep) module_state_map = cache.get('module_state_map') if module_state_map is None: module_state_map = {} diff_utils.set_module_state_map(module_state_map, all_files, lastfcommit) cache.set('module_state_map', module_state_map) return module_state_map
def performCommit(localRepo, bookid, bookinfo, bookroot, bookfiles): has_error = False staged = [] added = [] ignored = [] # convert url paths to os specific paths repo_home = pathof(localRepo) repo_home = repo_home.replace("/", os.sep) repo_path = os.path.join(repo_home, "epub_" + bookid) book_home = pathof(bookroot) book_home = book_home.replace("/", os.sep) # convert from bookpaths to os relative file paths filepaths = [] for bkpath in bookfiles: afile = pathof(bkpath) afile = afile.replace("/", os.sep) filepaths.append(afile) cdir = os.getcwd() if os.path.exists(repo_path): # handle updating the staged files and commiting and tagging # first collect info to determine files to delete form repo # current tag, etc os.chdir(repo_path) # determine the new tag tags = porcelain.list_tags(repo='.') tagname = "V%04d" % (len(tags) + 1) tagmessage = "Tag: " + tagname message = "updating to " + tagname # extra parameters must be passed as bytes if annotated is true tagname = utf8_str(tagname) message = utf8_str(message) tagmessage = utf8_str(tagmessage) # delete files that are no longer needed from staging area tracked = [] tracked = porcelain.ls_files(repo='.') files_to_delete = [] for afile in tracked: afile = pathof(afile) if afile not in filepaths: if afile not in ["mimetype", ".gitignore", ".bookinfo"]: files_to_delete.append(afile) if len(files_to_delete) > 0: porcelain.rm(repo='.', paths=files_to_delete) # copy over current files copy_book_contents_to_destination(book_home, filepaths, repo_path) (staged, unstaged, untracked) = porcelain.status(repo='.') files_to_update = [] for afile in unstaged: afile = pathof(afile) files_to_update.append(afile) for afile in untracked: afile = pathof(afile) files_to_update.append(afile) (added, ignored) = porcelain.add(repo='.', paths=files_to_update) commit_sha1 = porcelain.commit(repo='.', message=message, author=_SIGIL, committer=_SIGIL) # create annotated tags so we can get a date history tag = porcelain.tag_create(repo='.', tag=tagname, message=tagmessage, annotated=True, author=_SIGIL) os.chdir(cdir) add_bookinfo(repo_path, bookinfo, bookid, unicode_str(tagname)) else: # this will be an initial commit to this repo tagname = b"V0001" tagmessage = b'First Tag' message = b"Initial Commit" os.makedirs(repo_path) add_gitignore(repo_path) add_gitattributes(repo_path) cdir = os.getcwd() os.chdir(repo_path) r = porcelain.init(path='.', bare=False) staged = copy_book_contents_to_destination(book_home, filepaths, repo_path) (added, ignored) = porcelain.add(repo='.', paths=staged) # it seems author, committer, messages, and tagname only work with bytes if annotated=True commit_sha1 = porcelain.commit(repo='.', message=message, author=_SIGIL, committer=_SIGIL) tag = porcelain.tag_create(repo='.', tag=tagname, message=tagmessage, annotated=True, author=_SIGIL) os.chdir(cdir) add_bookinfo(repo_path, bookinfo, bookid, unicode_str(tagname)) result = "\n".join(added) result = result + "***********" + "\n".join(ignored) if not has_error: return result return ''
def run(self, args): parser = optparse.OptionParser() options, args = parser.parse_args(args) for name in porcelain.ls_files('.'): print(name)
def display_diff(): git_folder = app.config['GIT_REPOS_FOLDER'] rep = Repo(git_folder + '/terraform/') repowlkr = rep.get_walker(max_entries=1) lastfcommit = next(iter(repowlkr)).commit modules_repo = Repo(git_folder + '/terraform-cognite-modules') repowlkr_modules = modules_repo.get_walker(max_entries=1) lastfcommit_modules = next(iter(repowlkr_modules)).commit all_files = porcelain.ls_files(rep) state_map = {} ret = cache.get('ret') if ret is None: ret = {} diff_utils.set_state_map(state_map, all_files, lastfcommit) ret["all_subpath_commits"] = {} for folder_path, modules in state_map['cognitedata-greenfield'].items(): for module, module_info in modules.items(): ret["all_subpath_commits"][module] = {} ret["all_subpath_commits"][module]["module_commits"] = [] for full_module_path, git_ref in module_info.items(): dirfnm = full_module_path.encode('utf-8') repowlkr_subpath = modules_repo.get_walker(paths=[dirfnm]) repowlkr = modules_repo.get_walker() iterator = iter(repowlkr) iterator_subpath = iter(repowlkr_subpath) all_commits = [] subpath_commits = [] diff_utils.create_all_commits_list(iterator, all_commits) diff_utils.create_subpath_commits_list(iterator_subpath, subpath_commits) for commit in subpath_commits: parsed_commit = commit.as_pretty_string().decode().split( '\n') committer = parsed_commit[2] description = parsed_commit[-2] # sometimes them commit description has 2 new lines at the end if len(description) == 0: description = parsed_commit[-3] sha_commit = commit.id.decode() commit_info = {} commit_info["committer"] = committer commit_info["description"] = description commit_info["sha_commit"] = sha_commit ret["all_subpath_commits"][module][ "module_commits"].append(commit_info) greenfield_commit = diff_utils.get_commit_in_subpath( git_ref, all_commits, subpath_commits) ret["all_subpath_commits"][module][ "cognitedata-greenfield"] = greenfield_commit.id.decode() for project in state_map: if project != 'cognitedata-greenfield': project_commit = None if folder_path in state_map[project]: if module in state_map[project][folder_path]: project_commit = project + '_commit' project_commit = diff_utils.get_commit_in_subpath( state_map[project][folder_path][module] [full_module_path], all_commits, subpath_commits) if module in ret["all_subpath_commits"]: ret["all_subpath_commits"][module][ project] = project_commit.id.decode() if module not in ret: ret[module] = {} ret[module][ "greenfield"] = greenfield_commit.id.decode( ) ret[module][ project] = project_commit.id.decode() else: continue else: continue if project_commit is None: # raise a flag if equinor's git ref is invalid continue return jsonify(ret)