def diff_status():
# get latest commits of repos
git_folder = app.config['GIT_REPOS_FOLDER']
rep = Repo(git_folder + '/terraform/')
repowlkr = rep.get_walker(max_entries=1)
lastfcommit = next(iter(repowlkr)).commit
modules_repo = Repo(git_folder + '/terraform-cognite-modules')
repowlkr_modules = modules_repo.get_walker(max_entries=1)
lastfcommit_modules = next(iter(repowlkr_modules)).commit
all_files = porcelain.ls_files(rep)
state_map = {}
diff_module_map = {}
diff_utils.set_state_map(state_map, all_files, lastfcommit)
count = 0
# For now, we are comparing everything against Greenfield. We assume Greenfield is the most up to date project
for folder_path, modules in state_map['cognitedata-greenfield'].items():
for module, module_info in modules.items():
for full_module_path, git_ref in module_info.items():
dirfnm = full_module_path.encode('utf-8')
repowlkr_subpath = modules_repo.get_walker(paths=[dirfnm])
repowlkr = modules_repo.get_walker()
iterator = iter(repowlkr)
iterator_subpath = iter(repowlkr_subpath)
all_commits = []
subpath_commits = []
diff_utils.create_all_commits_list(iterator, all_commits)
diff_utils.create_subpath_commits_list(iterator_subpath, subpath_commits)
greenfield_commit = diff_utils.get_commit_in_subpath(git_ref, all_commits, subpath_commits)
for project in state_map:
if project != 'cognitedata-greenfield':
project_commit = None
if folder_path in state_map[project]:
if module in state_map[project][folder_path]:
project_commit = project + '_commit'
project_commit = diff_utils.get_commit_in_subpath(state_map[project][folder_path][module][full_module_path],
all_commits, subpath_commits)
else:
continue
else:
continue
if project_commit is None:
continue
diff_utils.set_diff_module_map(project_commit, project, greenfield_commit, folder_path,
module, full_module_path, subpath_commits, diff_module_map, count)
count += 1
cache.set('diff_module_map', diff_module_map)
return json.dumps(diff_module_map)
def _remove_untracked_files(requirements: List[_Requirement]) -> None:
"""
Remove files downloaded by pip which are not tracked by git.
"""
target_directories = set(requirement.target_directory
for requirement in requirements)
repo_files = ls_files(repo='.')
for target_directory in target_directories:
git_paths = [
item.decode() for item in repo_files
if item.decode().startswith(str(target_directory))
]
for item in target_directory.iterdir():
if not [path for path in git_paths if path.startswith(str(item))]:
shutil.rmtree(path=str(item))
def _remove_existing_files(requirements: List[_Requirement]) -> None:
"""
Remove existing files in vendored target directories.
"""
target_directories = set(requirement.target_directory
for requirement in requirements)
repo_files = ls_files(repo='.')
for target_directory in target_directories:
git_paths = [
item.decode() for item in repo_files
if item.decode().startswith(str(target_directory))
]
remove(paths=git_paths)
try:
shutil.rmtree(path=str(target_directory))
except FileNotFoundError:
pass
def get_latest_commits(git_folder, module_state_map):
"""Gets latest commit in /terraform and sets module_state_map
Args:
git_folder: A string of the repo path
module_state_map: An empty map
Returns:
A nested dictionary of all terraform modules being used by all projects,
path name within that project and path to that specific module.
See diff_utils.set_module_state_map for example
"""
rep = Repo(git_folder + '/terraform/')
repowlkr = rep.get_walker(max_entries=1)
lastfcommit = next(iter(repowlkr)).commit
all_files = porcelain.ls_files(rep)
module_state_map = cache.get('module_state_map')
if module_state_map is None:
module_state_map = {}
diff_utils.set_module_state_map(module_state_map, all_files,
lastfcommit)
cache.set('module_state_map', module_state_map)
return module_state_map
def performCommit(localRepo, bookid, bookinfo, bookroot, bookfiles):
has_error = False
staged = []
added = []
ignored = []
# convert url paths to os specific paths
repo_home = pathof(localRepo)
repo_home = repo_home.replace("/", os.sep)
repo_path = os.path.join(repo_home, "epub_" + bookid)
book_home = pathof(bookroot)
book_home = book_home.replace("/", os.sep)
# convert from bookpaths to os relative file paths
filepaths = []
for bkpath in bookfiles:
afile = pathof(bkpath)
afile = afile.replace("/", os.sep)
filepaths.append(afile)
cdir = os.getcwd()
if os.path.exists(repo_path):
# handle updating the staged files and commiting and tagging
# first collect info to determine files to delete form repo
# current tag, etc
os.chdir(repo_path)
# determine the new tag
tags = porcelain.list_tags(repo='.')
tagname = "V%04d" % (len(tags) + 1)
tagmessage = "Tag: " + tagname
message = "updating to " + tagname
# extra parameters must be passed as bytes if annotated is true
tagname = utf8_str(tagname)
message = utf8_str(message)
tagmessage = utf8_str(tagmessage)
# delete files that are no longer needed from staging area
tracked = []
tracked = porcelain.ls_files(repo='.')
files_to_delete = []
for afile in tracked:
afile = pathof(afile)
if afile not in filepaths:
if afile not in ["mimetype", ".gitignore", ".bookinfo"]:
files_to_delete.append(afile)
if len(files_to_delete) > 0:
porcelain.rm(repo='.', paths=files_to_delete)
# copy over current files
copy_book_contents_to_destination(book_home, filepaths, repo_path)
(staged, unstaged, untracked) = porcelain.status(repo='.')
files_to_update = []
for afile in unstaged:
afile = pathof(afile)
files_to_update.append(afile)
for afile in untracked:
afile = pathof(afile)
files_to_update.append(afile)
(added, ignored) = porcelain.add(repo='.', paths=files_to_update)
commit_sha1 = porcelain.commit(repo='.',
message=message,
author=_SIGIL,
committer=_SIGIL)
# create annotated tags so we can get a date history
tag = porcelain.tag_create(repo='.',
tag=tagname,
message=tagmessage,
annotated=True,
author=_SIGIL)
os.chdir(cdir)
add_bookinfo(repo_path, bookinfo, bookid, unicode_str(tagname))
else:
# this will be an initial commit to this repo
tagname = b"V0001"
tagmessage = b'First Tag'
message = b"Initial Commit"
os.makedirs(repo_path)
add_gitignore(repo_path)
add_gitattributes(repo_path)
cdir = os.getcwd()
os.chdir(repo_path)
r = porcelain.init(path='.', bare=False)
staged = copy_book_contents_to_destination(book_home, filepaths,
repo_path)
(added, ignored) = porcelain.add(repo='.', paths=staged)
# it seems author, committer, messages, and tagname only work with bytes if annotated=True
commit_sha1 = porcelain.commit(repo='.',
message=message,
author=_SIGIL,
committer=_SIGIL)
tag = porcelain.tag_create(repo='.',
tag=tagname,
message=tagmessage,
annotated=True,
author=_SIGIL)
os.chdir(cdir)
add_bookinfo(repo_path, bookinfo, bookid, unicode_str(tagname))
result = "\n".join(added)
result = result + "***********" + "\n".join(ignored)
if not has_error:
return result
return ''
def run(self, args):
parser = optparse.OptionParser()
options, args = parser.parse_args(args)
for name in porcelain.ls_files('.'):
print(name)
def display_diff():
git_folder = app.config['GIT_REPOS_FOLDER']
rep = Repo(git_folder + '/terraform/')
repowlkr = rep.get_walker(max_entries=1)
lastfcommit = next(iter(repowlkr)).commit
modules_repo = Repo(git_folder + '/terraform-cognite-modules')
repowlkr_modules = modules_repo.get_walker(max_entries=1)
lastfcommit_modules = next(iter(repowlkr_modules)).commit
all_files = porcelain.ls_files(rep)
state_map = {}
ret = cache.get('ret')
if ret is None:
ret = {}
diff_utils.set_state_map(state_map, all_files, lastfcommit)
ret["all_subpath_commits"] = {}
for folder_path, modules in state_map['cognitedata-greenfield'].items():
for module, module_info in modules.items():
ret["all_subpath_commits"][module] = {}
ret["all_subpath_commits"][module]["module_commits"] = []
for full_module_path, git_ref in module_info.items():
dirfnm = full_module_path.encode('utf-8')
repowlkr_subpath = modules_repo.get_walker(paths=[dirfnm])
repowlkr = modules_repo.get_walker()
iterator = iter(repowlkr)
iterator_subpath = iter(repowlkr_subpath)
all_commits = []
subpath_commits = []
diff_utils.create_all_commits_list(iterator, all_commits)
diff_utils.create_subpath_commits_list(iterator_subpath,
subpath_commits)
for commit in subpath_commits:
parsed_commit = commit.as_pretty_string().decode().split(
'\n')
committer = parsed_commit[2]
description = parsed_commit[-2]
# sometimes them commit description has 2 new lines at the end
if len(description) == 0:
description = parsed_commit[-3]
sha_commit = commit.id.decode()
commit_info = {}
commit_info["committer"] = committer
commit_info["description"] = description
commit_info["sha_commit"] = sha_commit
ret["all_subpath_commits"][module][
"module_commits"].append(commit_info)
greenfield_commit = diff_utils.get_commit_in_subpath(
git_ref, all_commits, subpath_commits)
ret["all_subpath_commits"][module][
"cognitedata-greenfield"] = greenfield_commit.id.decode()
for project in state_map:
if project != 'cognitedata-greenfield':
project_commit = None
if folder_path in state_map[project]:
if module in state_map[project][folder_path]:
project_commit = project + '_commit'
project_commit = diff_utils.get_commit_in_subpath(
state_map[project][folder_path][module]
[full_module_path], all_commits,
subpath_commits)
if module in ret["all_subpath_commits"]:
ret["all_subpath_commits"][module][
project] = project_commit.id.decode()
if module not in ret:
ret[module] = {}
ret[module][
"greenfield"] = greenfield_commit.id.decode(
)
ret[module][
project] = project_commit.id.decode()
else:
continue
else:
continue
if project_commit is None:
# raise a flag if equinor's git ref is invalid
continue
return jsonify(ret)