def prepare_git_repo(self, url, ref=None):
"""Clone and cache a Git repo."""
if not url:
raise errors.GitError("Invalid URL.")
RENKU_BRANCH = "renku-default-branch"
def checkout(repo, ref):
try:
repo.git.checkout(ref)
except GitCommandError:
raise errors.ParameterError('Cannot find reference "{}" in Git repository: {}'.format(ref, url))
ref = ref or RENKU_BRANCH
u = GitURL.parse(url)
path = u.pathname
if u.hostname == "localhost":
path = str(Path(path).resolve())
url = path
repo_name = os.path.splitext(os.path.basename(path))[0]
path = os.path.dirname(path).lstrip("/")
repo_path = self.renku_path / self.CACHE / u.hostname / path / repo_name
if repo_path.exists():
repo = Repo(str(repo_path))
if repo.remotes.origin.url == url:
try:
repo.git.fetch(all=True)
repo.git.checkout(ref)
try:
repo.git.pull()
except GitError:
# When ref is not a branch, an error is thrown
pass
except GitError:
# ignore the error and try re-cloning
pass
else:
return repo, repo_path
try:
shutil.rmtree(str(repo_path))
except PermissionError:
raise errors.InvalidFileOperation("Cannot delete files in {}: Permission denied".format(repo_path))
repo = clone(url, path=str(repo_path), install_githooks=False)
# Because the name of the default branch is not always 'master', we
# create an alias of the default branch when cloning the repo. It
# is used to refer to the default branch later.
renku_ref = "refs/heads/" + RENKU_BRANCH
try:
repo.git.execute(["git", "symbolic-ref", renku_ref, repo.head.reference.path])
checkout(repo, ref)
except GitCommandError as e:
raise errors.GitError("Cannot clone remote Git repo: {}".format(url)) from e
else:
return repo, repo_path
def fetch_template(source, ref='master', tempdir=None):
"""Fetch the template and checkout the relevant data.
:param source: url or full path of the templates repository
:param ref: reference for git checkout - branch, commit or tag
:param tempdir: temporary work directory path
:return: template manifest Path
"""
if tempdir is None:
tempdir = Path(tempfile.mkdtemp())
try:
# clone the repo locally without checking out files
template_repo = git.Repo.clone_from(source, tempdir, no_checkout=True)
except git.exc.GitCommandError as e:
raise errors.GitError(
'Cannot clone repo from {0}'.format(source)
) from e
try:
# fetch ref and set the HEAD
template_repo.remotes.origin.fetch(ref)
try:
template_repo.head.reset(template_repo.commit(ref))
except git.exc.BadName:
ref = 'origin/{0}'.format(ref)
template_repo.head.reset(template_repo.commit(ref))
git_repo = git.Git(str(tempdir))
except git.exc.GitCommandError as e:
raise errors.GitError(
'Cannot fetch and checkout reference {0}'.format(ref)
) from e
# checkout the manifest
try:
git_repo.checkout(TEMPLATE_MANIFEST)
except git.exc.GitCommandError as e:
raise errors.GitError(
'Cannot checkout manifest file {0}'.format(TEMPLATE_MANIFEST)
) from e
return tempdir / TEMPLATE_MANIFEST
def check_lfs_migrate_info(self, everything=False):
"""Return list of file groups in history should be in LFS."""
ref = ['--everything'] if everything else [
'--include-ref', self.repo.active_branch.name
]
includes = []
excludes = []
for p in self.renku_lfs_ignore.patterns:
if p.regex is None:
continue
pattern = p.pattern.replace(os.linesep, '').replace('\n', '')
if pattern.startswith('!'):
pattern.replace('!', '', 1)
if p.include: # File ignored by LFS
excludes.append(pattern)
else:
includes.append(pattern)
if excludes:
excludes = ['--exclude', ','.join(excludes)]
if includes:
includes = ['--include', ','.join(includes)]
above = ['--above', str(self.minimum_lfs_file_size)]
command = (self._CMD_STORAGE_MIGRATE_INFO + ref + above + includes +
excludes)
try:
lfs_output = run(command,
stdout=PIPE,
stderr=STDOUT,
cwd=self.path,
universal_newlines=True)
except (KeyboardInterrupt, OSError) as e:
raise errors.GitError(
'Couldn\'t run \'git lfs migrate info\':\n{0}'.format(e))
groups = []
files_re = re.compile(r'(.*\s+[\d.]+\s+\S+).*')
for line in lfs_output.stdout.split('\n'):
match = files_re.match(line)
if match:
groups.append(match.groups()[0])
return groups
def clone(
url,
path=None,
install_githooks=True,
install_lfs=True,
skip_smudge=True,
recursive=True,
depth=None,
progress=None,
config=None,
raise_git_except=False,
checkout_rev=None,
):
"""Clone Renku project repo, install Git hooks and LFS."""
from renku.core.management.client import LocalClient
path = path or GitURL.parse(url).name
if isinstance(path, Path):
path = str(path)
# Clone the project
if skip_smudge:
os.environ["GIT_LFS_SKIP_SMUDGE"] = "1"
try:
repo = Repo.clone_from(url, path, recursive=recursive, depth=depth, progress=progress)
except GitCommandError as e:
if not raise_git_except:
raise errors.GitError("Cannot clone remote Renku project: {}".format(url)) from e
raise e
remote_refs = [Path(ref.abspath).name for ref in repo.remote().refs]
if checkout_rev in remote_refs:
repo.git.checkout(checkout_rev)
elif checkout_rev:
repo.git.checkout(checkout_rev, b=checkout_rev)
if config:
config_writer = repo.config_writer()
for key, value in config.items():
key_path = key.split(".")
key = key_path.pop()
if not key_path or not key:
raise errors.GitError("Cannot write to config. Section path or key is invalid.")
config_writer.set_value(".".join(key_path), key, value)
config_writer.release()
client = LocalClient(path)
if install_githooks:
install(client=client, force=True)
if install_lfs:
command = ["git", "lfs", "install", "--local", "--force"]
if skip_smudge:
command += ["--skip-smudge"]
try:
repo.git.execute(command=command, with_exceptions=True)
except GitCommandError as e:
raise errors.GitError("Cannot install Git LFS") from e
return repo
def repo_sync(repo, message=None, remote=None, paths=None):
"""Commit and push paths."""
origin = None
saved_paths = []
# get branch that's pushed
if repo.active_branch.tracking_branch():
pushed_branch = repo.active_branch.tracking_branch().name
else:
pushed_branch = repo.active_branch.name
if remote:
# get/setup supplied remote for pushing
if repo.remotes:
existing = next((r for r in repo.remotes if r.url == remote), None)
if not existing:
existing = next((r for r in repo.remotes if r.name == remote),
None)
origin = next((r for r in repo.remotes if r.name == "origin"),
None)
if existing:
origin = existing
elif origin:
pushed_branch = uuid4().hex
origin = repo.create_remote(pushed_branch, remote)
if not origin:
origin = repo.create_remote("origin", remote)
elif not repo.active_branch.tracking_branch():
# No remote set on branch, push to available remote if only a single
# one is available
if len(repo.remotes) == 1:
origin = repo.remotes[0]
else:
raise errors.ConfigurationError(
"No remote has been set up for the current branch")
else:
# get remote that's set up to track the local branch
origin = repo.remotes[repo.active_branch.tracking_branch().remote_name]
if paths:
# commit uncommitted changes
try:
repo.git.add(*paths)
saved_paths = [d.b_path for d in repo.index.diff("HEAD")]
if not message:
# Show saved files in message
max_len = 100
message = "Saved changes to: "
paths_with_lens = reduce(
lambda c, x: c + [(x, c[-1][1] + len(x))], saved_paths,
[(None, len(message))])[1:]
# limit first line to max_len characters
message += " ".join(p if l < max_len else "\n\t" + p
for p, l in paths_with_lens)
repo.index.commit(message)
except git.exc.GitCommandError as e:
raise errors.GitError("Cannot commit changes") from e
try:
# push local changes to remote branch
if origin.refs and repo.active_branch in origin.refs:
origin.fetch()
origin.pull(repo.active_branch)
origin.push(repo.active_branch)
except git.exc.GitCommandError as e:
if "protected branches" not in e.stderr:
raise errors.GitError("Cannot push changes") from e
# push to new remote branch if original one is protected
pushed_branch = uuid4().hex
origin = repo.create_remote(pushed_branch, remote)
origin.push(repo.active_branch)
return saved_paths, pushed_branch
def clone(
url,
path=None,
install_githooks=True,
install_lfs=True,
skip_smudge=True,
recursive=True,
depth=None,
progress=None,
config=None,
raise_git_except=False,
):
"""Clone Renku project repo, install Git hooks and LFS."""
from renku.core.management.client import LocalClient
path = path or GitURL.parse(url).name
if isinstance(path, Path):
path = str(path)
# Clone the project
if skip_smudge:
os.environ['GIT_LFS_SKIP_SMUDGE'] = '1'
try:
repo = Repo.clone_from(url,
path,
recursive=recursive,
depth=depth,
progress=progress)
except GitCommandError as e:
if not raise_git_except:
raise errors.GitError(
'Cannot clone remote Renku project: {}'.format(url)) from e
raise e
if config:
config_writer = repo.config_writer()
for key, value in config.items():
key_path = key.split('.')
key = key_path.pop()
if not key_path or not key:
raise errors.GitError(
'Cannot write to config. Section path or key is invalid.')
config_writer.set_value('.'.join(key_path), key, value)
config_writer.release()
client = LocalClient(path)
if install_githooks:
install(client=client, force=True)
if install_lfs:
command = ['git', 'lfs', 'install', '--local', '--force']
if skip_smudge:
command += ['--skip-smudge']
try:
repo.git.execute(command=command, with_exceptions=True)
except GitCommandError as e:
raise errors.GitError('Cannot install Git LFS') from e
return repo