Esempio n. 1
0
def diff_status():
    # get latest commits of repos
    git_folder = app.config['GIT_REPOS_FOLDER']
    rep = Repo(git_folder + '/terraform/')
    repowlkr = rep.get_walker(max_entries=1)
    lastfcommit = next(iter(repowlkr)).commit

    modules_repo = Repo(git_folder + '/terraform-cognite-modules')
    repowlkr_modules = modules_repo.get_walker(max_entries=1)
    lastfcommit_modules = next(iter(repowlkr_modules)).commit

    all_files = porcelain.ls_files(rep)
    state_map = {}
    diff_module_map = {}

    diff_utils.set_state_map(state_map, all_files, lastfcommit)
    count = 0
    
    # For now, we are comparing everything against Greenfield. We assume Greenfield is the most up to date project
    for folder_path, modules in state_map['cognitedata-greenfield'].items():
        for module, module_info in modules.items():
            for full_module_path, git_ref in module_info.items():
                dirfnm = full_module_path.encode('utf-8')
                repowlkr_subpath = modules_repo.get_walker(paths=[dirfnm])
                repowlkr = modules_repo.get_walker()

                iterator = iter(repowlkr)
                iterator_subpath = iter(repowlkr_subpath)

                all_commits = []
                subpath_commits = []

                diff_utils.create_all_commits_list(iterator, all_commits)
                diff_utils.create_subpath_commits_list(iterator_subpath, subpath_commits)

                greenfield_commit = diff_utils.get_commit_in_subpath(git_ref, all_commits, subpath_commits)
                for project in state_map:
                    if project != 'cognitedata-greenfield':
                        project_commit = None
                        if folder_path in state_map[project]:
                            if module in state_map[project][folder_path]:
                                project_commit = project + '_commit'
                                project_commit = diff_utils.get_commit_in_subpath(state_map[project][folder_path][module][full_module_path],
                                                                             all_commits, subpath_commits)

                            else:
                                continue

                        else:
                            continue

                        if project_commit is None:
                            continue

                        diff_utils.set_diff_module_map(project_commit, project, greenfield_commit, folder_path,
                                            module, full_module_path, subpath_commits, diff_module_map, count)
                        count += 1

    cache.set('diff_module_map', diff_module_map)
    return json.dumps(diff_module_map)
Esempio n. 2
0
def _remove_untracked_files(requirements: List[_Requirement]) -> None:
    """
    Remove files downloaded by pip which are not tracked by git.
    """
    target_directories = set(requirement.target_directory
                             for requirement in requirements)
    repo_files = ls_files(repo='.')
    for target_directory in target_directories:
        git_paths = [
            item.decode() for item in repo_files
            if item.decode().startswith(str(target_directory))
        ]

        for item in target_directory.iterdir():
            if not [path for path in git_paths if path.startswith(str(item))]:
                shutil.rmtree(path=str(item))
Esempio n. 3
0
def _remove_existing_files(requirements: List[_Requirement]) -> None:
    """
    Remove existing files in vendored target directories.
    """
    target_directories = set(requirement.target_directory
                             for requirement in requirements)

    repo_files = ls_files(repo='.')
    for target_directory in target_directories:
        git_paths = [
            item.decode() for item in repo_files
            if item.decode().startswith(str(target_directory))
        ]
        remove(paths=git_paths)
        try:
            shutil.rmtree(path=str(target_directory))
        except FileNotFoundError:
            pass
Esempio n. 4
0
def get_latest_commits(git_folder, module_state_map):
    """Gets latest commit in /terraform and sets module_state_map

    Args:
        git_folder: A string of the repo path
        module_state_map: An empty map

    Returns:
        A nested dictionary of all terraform modules being used by all projects,
        path name within that project and path to that specific module.
        See diff_utils.set_module_state_map for example
    """
    rep = Repo(git_folder + '/terraform/')
    repowlkr = rep.get_walker(max_entries=1)
    lastfcommit = next(iter(repowlkr)).commit
    all_files = porcelain.ls_files(rep)
    module_state_map = cache.get('module_state_map')
    if module_state_map is None:
        module_state_map = {}
        diff_utils.set_module_state_map(module_state_map, all_files,
                                        lastfcommit)
    cache.set('module_state_map', module_state_map)
    return module_state_map
Esempio n. 5
0
def performCommit(localRepo, bookid, bookinfo, bookroot, bookfiles):
    has_error = False
    staged = []
    added = []
    ignored = []
    # convert url paths to os specific paths
    repo_home = pathof(localRepo)
    repo_home = repo_home.replace("/", os.sep)
    repo_path = os.path.join(repo_home, "epub_" + bookid)
    book_home = pathof(bookroot)
    book_home = book_home.replace("/", os.sep)
    # convert from bookpaths to os relative file paths
    filepaths = []
    for bkpath in bookfiles:
        afile = pathof(bkpath)
        afile = afile.replace("/", os.sep)
        filepaths.append(afile)

    cdir = os.getcwd()
    if os.path.exists(repo_path):
        # handle updating the staged files and commiting and tagging
        # first collect info to determine files to delete form repo
        # current tag, etc
        os.chdir(repo_path)
        # determine the new tag
        tags = porcelain.list_tags(repo='.')
        tagname = "V%04d" % (len(tags) + 1)
        tagmessage = "Tag: " + tagname
        message = "updating to " + tagname
        # extra parameters must be passed as bytes if annotated is true
        tagname = utf8_str(tagname)
        message = utf8_str(message)
        tagmessage = utf8_str(tagmessage)
        # delete files that are no longer needed from staging area
        tracked = []
        tracked = porcelain.ls_files(repo='.')
        files_to_delete = []
        for afile in tracked:
            afile = pathof(afile)
            if afile not in filepaths:
                if afile not in ["mimetype", ".gitignore", ".bookinfo"]:
                    files_to_delete.append(afile)
        if len(files_to_delete) > 0:
            porcelain.rm(repo='.', paths=files_to_delete)
        # copy over current files
        copy_book_contents_to_destination(book_home, filepaths, repo_path)
        (staged, unstaged, untracked) = porcelain.status(repo='.')
        files_to_update = []
        for afile in unstaged:
            afile = pathof(afile)
            files_to_update.append(afile)
        for afile in untracked:
            afile = pathof(afile)
            files_to_update.append(afile)
        (added, ignored) = porcelain.add(repo='.', paths=files_to_update)
        commit_sha1 = porcelain.commit(repo='.',
                                       message=message,
                                       author=_SIGIL,
                                       committer=_SIGIL)
        # create annotated tags so we can get a date history
        tag = porcelain.tag_create(repo='.',
                                   tag=tagname,
                                   message=tagmessage,
                                   annotated=True,
                                   author=_SIGIL)
        os.chdir(cdir)
        add_bookinfo(repo_path, bookinfo, bookid, unicode_str(tagname))
    else:
        # this will be an initial commit to this repo
        tagname = b"V0001"
        tagmessage = b'First Tag'
        message = b"Initial Commit"
        os.makedirs(repo_path)
        add_gitignore(repo_path)
        add_gitattributes(repo_path)
        cdir = os.getcwd()
        os.chdir(repo_path)
        r = porcelain.init(path='.', bare=False)
        staged = copy_book_contents_to_destination(book_home, filepaths,
                                                   repo_path)
        (added, ignored) = porcelain.add(repo='.', paths=staged)
        # it seems author, committer, messages, and tagname only work with bytes if annotated=True
        commit_sha1 = porcelain.commit(repo='.',
                                       message=message,
                                       author=_SIGIL,
                                       committer=_SIGIL)
        tag = porcelain.tag_create(repo='.',
                                   tag=tagname,
                                   message=tagmessage,
                                   annotated=True,
                                   author=_SIGIL)
        os.chdir(cdir)
        add_bookinfo(repo_path, bookinfo, bookid, unicode_str(tagname))
    result = "\n".join(added)
    result = result + "***********" + "\n".join(ignored)
    if not has_error:
        return result
    return ''
Esempio n. 6
0
 def run(self, args):
     parser = optparse.OptionParser()
     options, args = parser.parse_args(args)
     for name in porcelain.ls_files('.'):
         print(name)
Esempio n. 7
0
def display_diff():
    git_folder = app.config['GIT_REPOS_FOLDER']
    rep = Repo(git_folder + '/terraform/')
    repowlkr = rep.get_walker(max_entries=1)
    lastfcommit = next(iter(repowlkr)).commit

    modules_repo = Repo(git_folder + '/terraform-cognite-modules')
    repowlkr_modules = modules_repo.get_walker(max_entries=1)
    lastfcommit_modules = next(iter(repowlkr_modules)).commit

    all_files = porcelain.ls_files(rep)
    state_map = {}

    ret = cache.get('ret')
    if ret is None:
        ret = {}

    diff_utils.set_state_map(state_map, all_files, lastfcommit)
    ret["all_subpath_commits"] = {}

    for folder_path, modules in state_map['cognitedata-greenfield'].items():
        for module, module_info in modules.items():
            ret["all_subpath_commits"][module] = {}
            ret["all_subpath_commits"][module]["module_commits"] = []
            for full_module_path, git_ref in module_info.items():
                dirfnm = full_module_path.encode('utf-8')
                repowlkr_subpath = modules_repo.get_walker(paths=[dirfnm])
                repowlkr = modules_repo.get_walker()

                iterator = iter(repowlkr)
                iterator_subpath = iter(repowlkr_subpath)

                all_commits = []
                subpath_commits = []

                diff_utils.create_all_commits_list(iterator, all_commits)
                diff_utils.create_subpath_commits_list(iterator_subpath,
                                                       subpath_commits)

                for commit in subpath_commits:
                    parsed_commit = commit.as_pretty_string().decode().split(
                        '\n')
                    committer = parsed_commit[2]
                    description = parsed_commit[-2]
                    # sometimes them commit description has 2 new lines at the end
                    if len(description) == 0:
                        description = parsed_commit[-3]
                    sha_commit = commit.id.decode()
                    commit_info = {}
                    commit_info["committer"] = committer
                    commit_info["description"] = description
                    commit_info["sha_commit"] = sha_commit
                    ret["all_subpath_commits"][module][
                        "module_commits"].append(commit_info)
                greenfield_commit = diff_utils.get_commit_in_subpath(
                    git_ref, all_commits, subpath_commits)
                ret["all_subpath_commits"][module][
                    "cognitedata-greenfield"] = greenfield_commit.id.decode()

                for project in state_map:
                    if project != 'cognitedata-greenfield':
                        project_commit = None
                        if folder_path in state_map[project]:
                            if module in state_map[project][folder_path]:
                                project_commit = project + '_commit'
                                project_commit = diff_utils.get_commit_in_subpath(
                                    state_map[project][folder_path][module]
                                    [full_module_path], all_commits,
                                    subpath_commits)

                                if module in ret["all_subpath_commits"]:
                                    ret["all_subpath_commits"][module][
                                        project] = project_commit.id.decode()
                                if module not in ret:
                                    ret[module] = {}
                                ret[module][
                                    "greenfield"] = greenfield_commit.id.decode(
                                    )
                                ret[module][
                                    project] = project_commit.id.decode()
                            else:
                                continue

                        else:
                            continue

                        if project_commit is None:
                            # raise a flag if equinor's git ref is invalid
                            continue
    return jsonify(ret)