def get_relative_url(self, url):
"""Determine if the repo url should be relative."""
# Check if the default remote of the branch we are on is on
# the same server as the submodule. If so, use a relative path
# instead of an absolute URL.
try:
branch_remote = self.repo.config_reader().get(
'branch "{}"'.format(self.repo.active_branch.name), 'remote')
except NoSectionError:
branch_remote = 'origin'
try:
remote = self.repo.remote(branch_remote)
except ValueError:
warnings.warn(
'Remote {} not found, cannot check for relative URL.'.format(
branch_remote))
return url
remote_url = GitURL.parse(remote.url)
submodule_url = GitURL.parse(url)
if remote_url.hostname == submodule_url.hostname:
# construct the relative path
url = Path('../../{}'.format(submodule_url.owner) if remote_url.
owner == submodule_url.owner else '..')
url = str(url / submodule_url.name)
return url
def detect_registry_url(client, auto_login=True):
"""Return a URL of the Docker registry."""
repo = client.repo
config = repo.config_reader()
# Find registry URL in .git/config
remote_url = None
try:
registry_url = config.get_value('renku', 'registry', None)
except NoSectionError:
registry_url = None
remote_branch = repo.head.reference.tracking_branch()
if remote_branch is not None:
remote_name = remote_branch.remote_name
config_section = 'renku "{remote_name}"'.format(
remote_name=remote_name)
try:
registry_url = config.get_value(config_section, 'registry',
registry_url)
except NoSectionError:
pass
remote_url = repo.remotes[remote_name].url
if registry_url:
# Look in [renku] and [renku "{remote_name}"] for registry_url key.
url = GitURL.parse(registry_url)
elif remote_url:
# Use URL based on remote configuration.
url = GitURL.parse(remote_url)
# Replace gitlab. with registry. unless running on gitlab.com.
hostname_parts = url.hostname.split('.')
if len(hostname_parts) > 2 and hostname_parts[0] == 'gitlab':
hostname_parts = hostname_parts[1:]
hostname = '.'.join(['registry'] + hostname_parts)
url = attr.evolve(url, hostname=hostname)
else:
raise errors.ConfigurationError(
'Configure renku.repository_url or Git remote.')
if auto_login and url.username and url.password:
try:
subprocess.run([
'docker',
'login',
url.hostname,
'-u',
url.username,
'--password-stdin',
],
check=True,
input=url.password.encode('utf-8'))
except subprocess.CalledProcessError:
raise errors.AuthenticationError(
'Check configuration of password or token in the registry URL')
return url
def validate_git_url(self, value):
"""Validates git url."""
try:
GitURL.parse(value)
except ConfigurationError as e:
raise ValidationError(str(e))
return value
def integration_lifecycle(svc_client, mock_redis, authentication_headers):
"""Setup and teardown steps for integration tests."""
from renku.core.models.git import GitURL
url_components = GitURL.parse(IT_REMOTE_REPO_URL)
payload = {'git_url': IT_REMOTE_REPO_URL}
response = svc_client.post(
'/cache.project_clone',
data=json.dumps(payload),
headers=authentication_headers,
)
assert response
assert 'result' in response.json
assert 'error' not in response.json
project_id = response.json['result']['project_id']
assert isinstance(uuid.UUID(project_id), uuid.UUID)
yield svc_client, authentication_headers, project_id, url_components
# Teardown step: Delete all branches except master (if needed).
if integration_repo_path(authentication_headers, url_components).exists():
with integration_repo(authentication_headers, url_components) as repo:
try:
repo.remote().push(
refspec=(':{0}'.format(repo.active_branch.name)))
except git.exc.GitCommandError:
pass
def remote(self, remote_name='origin'):
"""Return host, owner and name of the remote if it exists."""
from renku.core.models.git import GitURL
host = owner = name = None
try:
remote_branch = \
self.repo.head.reference.tracking_branch()
if remote_branch is not None:
remote_name = remote_branch.remote_name
except TypeError:
pass
try:
url = GitURL.parse(self.repo.remotes[remote_name].url)
# Remove gitlab. unless running on gitlab.com.
hostname_parts = url.hostname.split('.')
if len(hostname_parts) > 2 and hostname_parts[0] == 'gitlab':
hostname_parts = hostname_parts[1:]
url = attr.evolve(url, hostname='.'.join(hostname_parts))
except IndexError:
url = None
if url:
host = url.hostname
owner = url.owner
name = url.name
return {'host': host, 'owner': owner, 'name': name}
def _prepare_git_repo(self, url, ref):
def checkout(repo, ref):
try:
repo.git.checkout(ref)
except GitCommandError:
raise errors.ParameterError(
'Cannot find reference "{}" in Git repository: {}'.format(
ref, url))
RENKU_BRANCH = 'renku-default-branch'
ref = ref or RENKU_BRANCH
u = GitURL.parse(url)
path = u.pathname
if u.hostname == 'localhost':
path = str(Path(path).resolve())
url = path
repo_name = os.path.splitext(os.path.basename(path))[0]
path = os.path.dirname(path).lstrip('/')
repo_path = self.renku_path / 'cache' / u.hostname / path / repo_name
if repo_path.exists():
repo = Repo(str(repo_path))
if repo.remotes.origin.url == url:
try:
repo.git.fetch(all=True)
repo.git.checkout(ref)
try:
repo.git.pull()
except GitError:
# When ref is not a branch, an error is thrown
pass
except GitError:
# ignore the error and try re-cloning
pass
else:
return repo, repo_path
try:
shutil.rmtree(str(repo_path))
except PermissionError:
raise errors.InvalidFileOperation(
'Cannot delete files in {}: Permission denied'.format(
repo_path))
repo = clone(url, path=str(repo_path), install_githooks=False)
# Because the name of the default branch is not always 'master', we
# create an alias of the default branch when cloning the repo. It
# is used to refer to the default branch later.
renku_ref = 'refs/heads/' + RENKU_BRANCH
try:
repo.git.execute(
['git', 'symbolic-ref', renku_ref, repo.head.reference.path])
checkout(repo, ref)
except GitCommandError as e:
raise errors.GitError(
'Cannot clone remote Git repo: {}'.format(url)) from e
else:
return repo, repo_path
def set_owner_name(self, data, **kwargs):
"""Set owner and name fields."""
git_url = GitURL.parse(data['git_url'])
data['owner'] = git_url.owner
data['name'] = git_url.name
return data
def set_owner_name(self, data, **kwargs):
"""Set owner and name fields."""
git_url = GitURL.parse(data["git_url"])
data["owner"] = git_url.owner
data["name"] = git_url.name
return data
def integration_lifecycle(svc_client, mock_redis):
"""Setup and teardown steps for integration tests."""
from renku.core.models.git import GitURL
remote_url = 'https://dev.renku.ch/gitlab/contact/integration-test'
url_components = GitURL.parse(remote_url)
headers = {
'Content-Type': 'application/json',
'Renku-User-Id': 'b4b4de0eda0f471ab82702bd5c367fa7',
'Renku-User-FullName': 'Just Sam',
'Renku-User-Email': '*****@*****.**',
'Authorization': 'Bearer {0}'.format(os.getenv('IT_OAUTH_GIT_TOKEN')),
}
payload = {'git_url': remote_url}
response = svc_client.post(
'/cache.project_clone',
data=json.dumps(payload),
headers=headers,
)
assert response
assert 'result' in response.json
assert 'error' not in response.json
project_id = response.json['result']['project_id']
assert isinstance(uuid.UUID(project_id), uuid.UUID)
yield svc_client, headers, project_id, url_components
# Teardown step: Delete all branches except master (if needed).
if integration_repo_path(headers, url_components).exists():
with integration_repo(headers, url_components) as repo:
for repo_branch in repo.references:
if repo_branch.name == 'master':
continue
try:
repo.remote().push(
refspec=(':{0}'.format(repo_branch.name)))
except git.exc.GitCommandError:
continue
def test_clone_projects_list_view_errors(svc_client):
"""Check cache state of cloned projects with no headers."""
headers = {
'Content-Type': 'application/json',
'Renku-User-Id': '{0}'.format(uuid.uuid4().hex),
'Renku-User-FullName': 'Just Sam',
'Renku-User-Email': '*****@*****.**',
'Authorization': 'Bearer {0}'.format(IT_GIT_ACCESS_TOKEN),
}
payload = {
'git_url': REMOTE_URL,
}
response = svc_client.post(
'/cache.project_clone', data=json.dumps(payload), headers=headers
)
assert response
assert {'result'} == set(response.json.keys())
assert isinstance(
uuid.UUID(response.json['result']['project_id']), uuid.UUID
)
response = svc_client.get(
'/cache.project_list',
# no auth headers, expected error
)
assert response
assert {'error'} == set(response.json.keys())
assert INVALID_HEADERS_ERROR_CODE == response.json['error']['code']
response = svc_client.get('/cache.project_list', headers=headers)
assert response
assert {'result'} == set(response.json.keys())
assert 1 == len(response.json['result']['projects'])
project = response.json['result']['projects'][0]
assert isinstance(uuid.UUID(project['project_id']), uuid.UUID)
assert isinstance(GitURL.parse(project['git_url']), GitURL)
def test_clone_projects_list_view_errors(svc_client):
"""Check cache state of cloned projects with no headers."""
headers = {
"Content-Type": "application/json",
"Renku-User-Id": "{0}".format(uuid.uuid4().hex),
"Renku-User-FullName": "Just Sam",
"Renku-User-Email": "*****@*****.**",
"Authorization": "Bearer {0}".format(IT_GIT_ACCESS_TOKEN),
}
payload = {
"git_url": IT_REMOTE_REPO_URL,
}
response = svc_client.post("/cache.project_clone",
data=json.dumps(payload),
headers=headers)
assert response
assert {"result"} == set(response.json.keys())
assert isinstance(uuid.UUID(response.json["result"]["project_id"]),
uuid.UUID)
response = svc_client.get("/cache.project_list",
# no auth headers, expected error
)
assert response
assert {"error"} == set(response.json.keys())
assert INVALID_HEADERS_ERROR_CODE == response.json["error"]["code"]
response = svc_client.get("/cache.project_list", headers=headers)
assert response
assert {"result"} == set(response.json.keys())
assert 1 == len(response.json["result"]["projects"])
project = response.json["result"]["projects"][0]
assert isinstance(uuid.UUID(project["project_id"]), uuid.UUID)
assert isinstance(GitURL.parse(project["git_url"]), GitURL)
def remote(self, remote_name="origin"):
"""Return host, owner and name of the remote if it exists."""
from renku.core.models.git import GitURL
original_remote_name = remote_name
if original_remote_name in self._remote_cache:
return self._remote_cache[original_remote_name]
host = owner = name = None
try:
remote_branch = self.repo.head.reference.tracking_branch()
if remote_branch is not None:
remote_name = remote_branch.remote_name
except TypeError:
pass
try:
url = GitURL.parse(self.repo.remotes[remote_name].url)
# Remove gitlab. unless running on gitlab.com.
hostname_parts = url.hostname.split(".")
if len(hostname_parts) > 2 and hostname_parts[0] == "gitlab":
hostname_parts = hostname_parts[1:]
url = attr.evolve(url, hostname=".".join(hostname_parts))
except IndexError:
url = None
if url:
host = url.hostname
owner = url.owner
name = url.name
remote = {"host": host, "owner": owner, "name": name}
self._remote_cache[original_remote_name] = remote
return remote
def clone(
url,
path=None,
install_githooks=True,
install_lfs=True,
skip_smudge=True,
recursive=True,
depth=None,
progress=None,
config=None,
raise_git_except=False,
checkout_rev=None,
):
"""Clone Renku project repo, install Git hooks and LFS."""
from renku.core.management.client import LocalClient
path = path or GitURL.parse(url).name
if isinstance(path, Path):
path = str(path)
# Clone the project
if skip_smudge:
os.environ["GIT_LFS_SKIP_SMUDGE"] = "1"
try:
repo = Repo.clone_from(url, path, recursive=recursive, depth=depth, progress=progress)
except GitCommandError as e:
if not raise_git_except:
raise errors.GitError("Cannot clone remote Renku project: {}".format(url)) from e
raise e
remote_refs = [Path(ref.abspath).name for ref in repo.remote().refs]
if checkout_rev in remote_refs:
repo.git.checkout(checkout_rev)
elif checkout_rev:
repo.git.checkout(checkout_rev, b=checkout_rev)
if config:
config_writer = repo.config_writer()
for key, value in config.items():
key_path = key.split(".")
key = key_path.pop()
if not key_path or not key:
raise errors.GitError("Cannot write to config. Section path or key is invalid.")
config_writer.set_value(".".join(key_path), key, value)
config_writer.release()
client = LocalClient(path)
if install_githooks:
install(client=client, force=True)
if install_lfs:
command = ["git", "lfs", "install", "--local", "--force"]
if skip_smudge:
command += ["--skip-smudge"]
try:
repo.git.execute(command=command, with_exceptions=True)
except GitCommandError as e:
raise errors.GitError("Cannot install Git LFS") from e
return repo
def test_valid_href(fields):
"""Test the various repo regexes."""
fields.pop("protocols", None)
assert GitURL(**fields) == GitURL.parse(fields["href"])
def clone(
url,
path=None,
install_githooks=True,
install_lfs=True,
skip_smudge=True,
recursive=True,
depth=None,
progress=None,
config=None,
raise_git_except=False,
):
"""Clone Renku project repo, install Git hooks and LFS."""
from renku.core.management.client import LocalClient
path = path or GitURL.parse(url).name
if isinstance(path, Path):
path = str(path)
# Clone the project
if skip_smudge:
os.environ['GIT_LFS_SKIP_SMUDGE'] = '1'
try:
repo = Repo.clone_from(url,
path,
recursive=recursive,
depth=depth,
progress=progress)
except GitCommandError as e:
if not raise_git_except:
raise errors.GitError(
'Cannot clone remote Renku project: {}'.format(url)) from e
raise e
if config:
config_writer = repo.config_writer()
for key, value in config.items():
key_path = key.split('.')
key = key_path.pop()
if not key_path or not key:
raise errors.GitError(
'Cannot write to config. Section path or key is invalid.')
config_writer.set_value('.'.join(key_path), key, value)
config_writer.release()
client = LocalClient(path)
if install_githooks:
install(client=client, force=True)
if install_lfs:
command = ['git', 'lfs', 'install', '--local', '--force']
if skip_smudge:
command += ['--skip-smudge']
try:
repo.git.execute(command=command, with_exceptions=True)
except GitCommandError as e:
raise errors.GitError('Cannot install Git LFS') from e
return repo