def prepare_git_repo(self, url, ref=None): """Clone and cache a Git repo.""" if not url: raise errors.GitError("Invalid URL.") RENKU_BRANCH = "renku-default-branch" def checkout(repo, ref): try: repo.git.checkout(ref) except GitCommandError: raise errors.ParameterError('Cannot find reference "{}" in Git repository: {}'.format(ref, url)) ref = ref or RENKU_BRANCH u = GitURL.parse(url) path = u.pathname if u.hostname == "localhost": path = str(Path(path).resolve()) url = path repo_name = os.path.splitext(os.path.basename(path))[0] path = os.path.dirname(path).lstrip("/") repo_path = self.renku_path / self.CACHE / u.hostname / path / repo_name if repo_path.exists(): repo = Repo(str(repo_path)) if repo.remotes.origin.url == url: try: repo.git.fetch(all=True) repo.git.checkout(ref) try: repo.git.pull() except GitError: # When ref is not a branch, an error is thrown pass except GitError: # ignore the error and try re-cloning pass else: return repo, repo_path try: shutil.rmtree(str(repo_path)) except PermissionError: raise errors.InvalidFileOperation("Cannot delete files in {}: Permission denied".format(repo_path)) repo = clone(url, path=str(repo_path), install_githooks=False) # Because the name of the default branch is not always 'master', we # create an alias of the default branch when cloning the repo. It # is used to refer to the default branch later. renku_ref = "refs/heads/" + RENKU_BRANCH try: repo.git.execute(["git", "symbolic-ref", renku_ref, repo.head.reference.path]) checkout(repo, ref) except GitCommandError as e: raise errors.GitError("Cannot clone remote Git repo: {}".format(url)) from e else: return repo, repo_path
def fetch_template(source, ref='master', tempdir=None): """Fetch the template and checkout the relevant data. :param source: url or full path of the templates repository :param ref: reference for git checkout - branch, commit or tag :param tempdir: temporary work directory path :return: template manifest Path """ if tempdir is None: tempdir = Path(tempfile.mkdtemp()) try: # clone the repo locally without checking out files template_repo = git.Repo.clone_from(source, tempdir, no_checkout=True) except git.exc.GitCommandError as e: raise errors.GitError( 'Cannot clone repo from {0}'.format(source) ) from e try: # fetch ref and set the HEAD template_repo.remotes.origin.fetch(ref) try: template_repo.head.reset(template_repo.commit(ref)) except git.exc.BadName: ref = 'origin/{0}'.format(ref) template_repo.head.reset(template_repo.commit(ref)) git_repo = git.Git(str(tempdir)) except git.exc.GitCommandError as e: raise errors.GitError( 'Cannot fetch and checkout reference {0}'.format(ref) ) from e # checkout the manifest try: git_repo.checkout(TEMPLATE_MANIFEST) except git.exc.GitCommandError as e: raise errors.GitError( 'Cannot checkout manifest file {0}'.format(TEMPLATE_MANIFEST) ) from e return tempdir / TEMPLATE_MANIFEST
def check_lfs_migrate_info(self, everything=False): """Return list of file groups in history should be in LFS.""" ref = ['--everything'] if everything else [ '--include-ref', self.repo.active_branch.name ] includes = [] excludes = [] for p in self.renku_lfs_ignore.patterns: if p.regex is None: continue pattern = p.pattern.replace(os.linesep, '').replace('\n', '') if pattern.startswith('!'): pattern.replace('!', '', 1) if p.include: # File ignored by LFS excludes.append(pattern) else: includes.append(pattern) if excludes: excludes = ['--exclude', ','.join(excludes)] if includes: includes = ['--include', ','.join(includes)] above = ['--above', str(self.minimum_lfs_file_size)] command = (self._CMD_STORAGE_MIGRATE_INFO + ref + above + includes + excludes) try: lfs_output = run(command, stdout=PIPE, stderr=STDOUT, cwd=self.path, universal_newlines=True) except (KeyboardInterrupt, OSError) as e: raise errors.GitError( 'Couldn\'t run \'git lfs migrate info\':\n{0}'.format(e)) groups = [] files_re = re.compile(r'(.*\s+[\d.]+\s+\S+).*') for line in lfs_output.stdout.split('\n'): match = files_re.match(line) if match: groups.append(match.groups()[0]) return groups
def clone( url, path=None, install_githooks=True, install_lfs=True, skip_smudge=True, recursive=True, depth=None, progress=None, config=None, raise_git_except=False, checkout_rev=None, ): """Clone Renku project repo, install Git hooks and LFS.""" from renku.core.management.client import LocalClient path = path or GitURL.parse(url).name if isinstance(path, Path): path = str(path) # Clone the project if skip_smudge: os.environ["GIT_LFS_SKIP_SMUDGE"] = "1" try: repo = Repo.clone_from(url, path, recursive=recursive, depth=depth, progress=progress) except GitCommandError as e: if not raise_git_except: raise errors.GitError("Cannot clone remote Renku project: {}".format(url)) from e raise e remote_refs = [Path(ref.abspath).name for ref in repo.remote().refs] if checkout_rev in remote_refs: repo.git.checkout(checkout_rev) elif checkout_rev: repo.git.checkout(checkout_rev, b=checkout_rev) if config: config_writer = repo.config_writer() for key, value in config.items(): key_path = key.split(".") key = key_path.pop() if not key_path or not key: raise errors.GitError("Cannot write to config. Section path or key is invalid.") config_writer.set_value(".".join(key_path), key, value) config_writer.release() client = LocalClient(path) if install_githooks: install(client=client, force=True) if install_lfs: command = ["git", "lfs", "install", "--local", "--force"] if skip_smudge: command += ["--skip-smudge"] try: repo.git.execute(command=command, with_exceptions=True) except GitCommandError as e: raise errors.GitError("Cannot install Git LFS") from e return repo
def repo_sync(repo, message=None, remote=None, paths=None): """Commit and push paths.""" origin = None saved_paths = [] # get branch that's pushed if repo.active_branch.tracking_branch(): pushed_branch = repo.active_branch.tracking_branch().name else: pushed_branch = repo.active_branch.name if remote: # get/setup supplied remote for pushing if repo.remotes: existing = next((r for r in repo.remotes if r.url == remote), None) if not existing: existing = next((r for r in repo.remotes if r.name == remote), None) origin = next((r for r in repo.remotes if r.name == "origin"), None) if existing: origin = existing elif origin: pushed_branch = uuid4().hex origin = repo.create_remote(pushed_branch, remote) if not origin: origin = repo.create_remote("origin", remote) elif not repo.active_branch.tracking_branch(): # No remote set on branch, push to available remote if only a single # one is available if len(repo.remotes) == 1: origin = repo.remotes[0] else: raise errors.ConfigurationError( "No remote has been set up for the current branch") else: # get remote that's set up to track the local branch origin = repo.remotes[repo.active_branch.tracking_branch().remote_name] if paths: # commit uncommitted changes try: repo.git.add(*paths) saved_paths = [d.b_path for d in repo.index.diff("HEAD")] if not message: # Show saved files in message max_len = 100 message = "Saved changes to: " paths_with_lens = reduce( lambda c, x: c + [(x, c[-1][1] + len(x))], saved_paths, [(None, len(message))])[1:] # limit first line to max_len characters message += " ".join(p if l < max_len else "\n\t" + p for p, l in paths_with_lens) repo.index.commit(message) except git.exc.GitCommandError as e: raise errors.GitError("Cannot commit changes") from e try: # push local changes to remote branch if origin.refs and repo.active_branch in origin.refs: origin.fetch() origin.pull(repo.active_branch) origin.push(repo.active_branch) except git.exc.GitCommandError as e: if "protected branches" not in e.stderr: raise errors.GitError("Cannot push changes") from e # push to new remote branch if original one is protected pushed_branch = uuid4().hex origin = repo.create_remote(pushed_branch, remote) origin.push(repo.active_branch) return saved_paths, pushed_branch
def clone( url, path=None, install_githooks=True, install_lfs=True, skip_smudge=True, recursive=True, depth=None, progress=None, config=None, raise_git_except=False, ): """Clone Renku project repo, install Git hooks and LFS.""" from renku.core.management.client import LocalClient path = path or GitURL.parse(url).name if isinstance(path, Path): path = str(path) # Clone the project if skip_smudge: os.environ['GIT_LFS_SKIP_SMUDGE'] = '1' try: repo = Repo.clone_from(url, path, recursive=recursive, depth=depth, progress=progress) except GitCommandError as e: if not raise_git_except: raise errors.GitError( 'Cannot clone remote Renku project: {}'.format(url)) from e raise e if config: config_writer = repo.config_writer() for key, value in config.items(): key_path = key.split('.') key = key_path.pop() if not key_path or not key: raise errors.GitError( 'Cannot write to config. Section path or key is invalid.') config_writer.set_value('.'.join(key_path), key, value) config_writer.release() client = LocalClient(path) if install_githooks: install(client=client, force=True) if install_lfs: command = ['git', 'lfs', 'install', '--local', '--force'] if skip_smudge: command += ['--skip-smudge'] try: repo.git.execute(command=command, with_exceptions=True) except GitCommandError as e: raise errors.GitError('Cannot install Git LFS') from e return repo