示例#1
0
    def get_relative_url(self, url):
        """Determine if the repo url should be relative."""
        # Check if the default remote of the branch we are on is on
        # the same server as the submodule. If so, use a relative path
        # instead of an absolute URL.
        try:
            branch_remote = self.repo.config_reader().get(
                'branch "{}"'.format(self.repo.active_branch.name), 'remote')
        except NoSectionError:
            branch_remote = 'origin'

        try:
            remote = self.repo.remote(branch_remote)
        except ValueError:
            warnings.warn(
                'Remote {} not found, cannot check for relative URL.'.format(
                    branch_remote))
            return url

        remote_url = GitURL.parse(remote.url)
        submodule_url = GitURL.parse(url)

        if remote_url.hostname == submodule_url.hostname:
            # construct the relative path
            url = Path('../../{}'.format(submodule_url.owner) if remote_url.
                       owner == submodule_url.owner else '..')
            url = str(url / submodule_url.name)
        return url
示例#2
0
def detect_registry_url(client, auto_login=True):
    """Return a URL of the Docker registry."""
    repo = client.repo
    config = repo.config_reader()

    # Find registry URL in .git/config
    remote_url = None
    try:
        registry_url = config.get_value('renku', 'registry', None)
    except NoSectionError:
        registry_url = None

    remote_branch = repo.head.reference.tracking_branch()

    if remote_branch is not None:
        remote_name = remote_branch.remote_name
        config_section = 'renku "{remote_name}"'.format(
            remote_name=remote_name)
        try:
            registry_url = config.get_value(config_section, 'registry',
                                            registry_url)
        except NoSectionError:
            pass
        remote_url = repo.remotes[remote_name].url

    if registry_url:
        # Look in [renku] and [renku "{remote_name}"] for registry_url key.
        url = GitURL.parse(registry_url)
    elif remote_url:
        # Use URL based on remote configuration.
        url = GitURL.parse(remote_url)

        # Replace gitlab. with registry. unless running on gitlab.com.
        hostname_parts = url.hostname.split('.')
        if len(hostname_parts) > 2 and hostname_parts[0] == 'gitlab':
            hostname_parts = hostname_parts[1:]
        hostname = '.'.join(['registry'] + hostname_parts)
        url = attr.evolve(url, hostname=hostname)
    else:
        raise errors.ConfigurationError(
            'Configure renku.repository_url or Git remote.')

    if auto_login and url.username and url.password:
        try:
            subprocess.run([
                'docker',
                'login',
                url.hostname,
                '-u',
                url.username,
                '--password-stdin',
            ],
                           check=True,
                           input=url.password.encode('utf-8'))
        except subprocess.CalledProcessError:
            raise errors.AuthenticationError(
                'Check configuration of password or token in the registry URL')

    return url
示例#3
0
    def validate_git_url(self, value):
        """Validates git url."""
        try:
            GitURL.parse(value)
        except ConfigurationError as e:
            raise ValidationError(str(e))

        return value
示例#4
0
def integration_lifecycle(svc_client, mock_redis, authentication_headers):
    """Setup and teardown steps for integration tests."""
    from renku.core.models.git import GitURL
    url_components = GitURL.parse(IT_REMOTE_REPO_URL)

    payload = {'git_url': IT_REMOTE_REPO_URL}

    response = svc_client.post(
        '/cache.project_clone',
        data=json.dumps(payload),
        headers=authentication_headers,
    )

    assert response
    assert 'result' in response.json
    assert 'error' not in response.json

    project_id = response.json['result']['project_id']
    assert isinstance(uuid.UUID(project_id), uuid.UUID)

    yield svc_client, authentication_headers, project_id, url_components

    # Teardown step: Delete all branches except master (if needed).
    if integration_repo_path(authentication_headers, url_components).exists():
        with integration_repo(authentication_headers, url_components) as repo:
            try:
                repo.remote().push(
                    refspec=(':{0}'.format(repo.active_branch.name)))
            except git.exc.GitCommandError:
                pass
示例#5
0
    def remote(self, remote_name='origin'):
        """Return host, owner and name of the remote if it exists."""
        from renku.core.models.git import GitURL

        host = owner = name = None
        try:
            remote_branch = \
                self.repo.head.reference.tracking_branch()
            if remote_branch is not None:
                remote_name = remote_branch.remote_name
        except TypeError:
            pass

        try:
            url = GitURL.parse(self.repo.remotes[remote_name].url)

            # Remove gitlab. unless running on gitlab.com.
            hostname_parts = url.hostname.split('.')
            if len(hostname_parts) > 2 and hostname_parts[0] == 'gitlab':
                hostname_parts = hostname_parts[1:]
            url = attr.evolve(url, hostname='.'.join(hostname_parts))
        except IndexError:
            url = None

        if url:
            host = url.hostname
            owner = url.owner
            name = url.name
        return {'host': host, 'owner': owner, 'name': name}
示例#6
0
    def _prepare_git_repo(self, url, ref):
        def checkout(repo, ref):
            try:
                repo.git.checkout(ref)
            except GitCommandError:
                raise errors.ParameterError(
                    'Cannot find reference "{}" in Git repository: {}'.format(
                        ref, url))

        RENKU_BRANCH = 'renku-default-branch'
        ref = ref or RENKU_BRANCH
        u = GitURL.parse(url)
        path = u.pathname
        if u.hostname == 'localhost':
            path = str(Path(path).resolve())
            url = path
        repo_name = os.path.splitext(os.path.basename(path))[0]
        path = os.path.dirname(path).lstrip('/')
        repo_path = self.renku_path / 'cache' / u.hostname / path / repo_name

        if repo_path.exists():
            repo = Repo(str(repo_path))
            if repo.remotes.origin.url == url:
                try:
                    repo.git.fetch(all=True)
                    repo.git.checkout(ref)
                    try:
                        repo.git.pull()
                    except GitError:
                        # When ref is not a branch, an error is thrown
                        pass
                except GitError:
                    # ignore the error and try re-cloning
                    pass
                else:
                    return repo, repo_path

            try:
                shutil.rmtree(str(repo_path))
            except PermissionError:
                raise errors.InvalidFileOperation(
                    'Cannot delete files in {}: Permission denied'.format(
                        repo_path))

        repo = clone(url, path=str(repo_path), install_githooks=False)

        # Because the name of the default branch is not always 'master', we
        # create an alias of the default branch when cloning the repo. It
        # is used to refer to the default branch later.
        renku_ref = 'refs/heads/' + RENKU_BRANCH
        try:
            repo.git.execute(
                ['git', 'symbolic-ref', renku_ref, repo.head.reference.path])
            checkout(repo, ref)
        except GitCommandError as e:
            raise errors.GitError(
                'Cannot clone remote Git repo: {}'.format(url)) from e
        else:
            return repo, repo_path
示例#7
0
    def set_owner_name(self, data, **kwargs):
        """Set owner and name fields."""
        git_url = GitURL.parse(data['git_url'])

        data['owner'] = git_url.owner
        data['name'] = git_url.name

        return data
示例#8
0
    def set_owner_name(self, data, **kwargs):
        """Set owner and name fields."""
        git_url = GitURL.parse(data["git_url"])

        data["owner"] = git_url.owner
        data["name"] = git_url.name

        return data
示例#9
0
def integration_lifecycle(svc_client, mock_redis):
    """Setup and teardown steps for integration tests."""
    from renku.core.models.git import GitURL
    remote_url = 'https://dev.renku.ch/gitlab/contact/integration-test'
    url_components = GitURL.parse(remote_url)

    headers = {
        'Content-Type': 'application/json',
        'Renku-User-Id': 'b4b4de0eda0f471ab82702bd5c367fa7',
        'Renku-User-FullName': 'Just Sam',
        'Renku-User-Email': '*****@*****.**',
        'Authorization': 'Bearer {0}'.format(os.getenv('IT_OAUTH_GIT_TOKEN')),
    }

    payload = {'git_url': remote_url}

    response = svc_client.post(
        '/cache.project_clone',
        data=json.dumps(payload),
        headers=headers,
    )

    assert response
    assert 'result' in response.json
    assert 'error' not in response.json

    project_id = response.json['result']['project_id']
    assert isinstance(uuid.UUID(project_id), uuid.UUID)

    yield svc_client, headers, project_id, url_components

    # Teardown step: Delete all branches except master (if needed).
    if integration_repo_path(headers, url_components).exists():
        with integration_repo(headers, url_components) as repo:
            for repo_branch in repo.references:
                if repo_branch.name == 'master':
                    continue
                try:
                    repo.remote().push(
                        refspec=(':{0}'.format(repo_branch.name)))
                except git.exc.GitCommandError:
                    continue
def test_clone_projects_list_view_errors(svc_client):
    """Check cache state of cloned projects with no headers."""
    headers = {
        'Content-Type': 'application/json',
        'Renku-User-Id': '{0}'.format(uuid.uuid4().hex),
        'Renku-User-FullName': 'Just Sam',
        'Renku-User-Email': '*****@*****.**',
        'Authorization': 'Bearer {0}'.format(IT_GIT_ACCESS_TOKEN),
    }

    payload = {
        'git_url': REMOTE_URL,
    }

    response = svc_client.post(
        '/cache.project_clone', data=json.dumps(payload), headers=headers
    )
    assert response
    assert {'result'} == set(response.json.keys())

    assert isinstance(
        uuid.UUID(response.json['result']['project_id']), uuid.UUID
    )

    response = svc_client.get(
        '/cache.project_list',
        # no auth headers, expected error
    )
    assert response

    assert {'error'} == set(response.json.keys())
    assert INVALID_HEADERS_ERROR_CODE == response.json['error']['code']

    response = svc_client.get('/cache.project_list', headers=headers)

    assert response
    assert {'result'} == set(response.json.keys())
    assert 1 == len(response.json['result']['projects'])

    project = response.json['result']['projects'][0]
    assert isinstance(uuid.UUID(project['project_id']), uuid.UUID)
    assert isinstance(GitURL.parse(project['git_url']), GitURL)
示例#11
0
def test_clone_projects_list_view_errors(svc_client):
    """Check cache state of cloned projects with no headers."""
    headers = {
        "Content-Type": "application/json",
        "Renku-User-Id": "{0}".format(uuid.uuid4().hex),
        "Renku-User-FullName": "Just Sam",
        "Renku-User-Email": "*****@*****.**",
        "Authorization": "Bearer {0}".format(IT_GIT_ACCESS_TOKEN),
    }

    payload = {
        "git_url": IT_REMOTE_REPO_URL,
    }

    response = svc_client.post("/cache.project_clone",
                               data=json.dumps(payload),
                               headers=headers)
    assert response
    assert {"result"} == set(response.json.keys())

    assert isinstance(uuid.UUID(response.json["result"]["project_id"]),
                      uuid.UUID)

    response = svc_client.get("/cache.project_list",
                              # no auth headers, expected error
                              )
    assert response

    assert {"error"} == set(response.json.keys())
    assert INVALID_HEADERS_ERROR_CODE == response.json["error"]["code"]

    response = svc_client.get("/cache.project_list", headers=headers)

    assert response
    assert {"result"} == set(response.json.keys())
    assert 1 == len(response.json["result"]["projects"])

    project = response.json["result"]["projects"][0]
    assert isinstance(uuid.UUID(project["project_id"]), uuid.UUID)
    assert isinstance(GitURL.parse(project["git_url"]), GitURL)
示例#12
0
    def remote(self, remote_name="origin"):
        """Return host, owner and name of the remote if it exists."""
        from renku.core.models.git import GitURL

        original_remote_name = remote_name

        if original_remote_name in self._remote_cache:
            return self._remote_cache[original_remote_name]

        host = owner = name = None
        try:
            remote_branch = self.repo.head.reference.tracking_branch()
            if remote_branch is not None:
                remote_name = remote_branch.remote_name
        except TypeError:
            pass

        try:
            url = GitURL.parse(self.repo.remotes[remote_name].url)

            # Remove gitlab. unless running on gitlab.com.
            hostname_parts = url.hostname.split(".")
            if len(hostname_parts) > 2 and hostname_parts[0] == "gitlab":
                hostname_parts = hostname_parts[1:]
            url = attr.evolve(url, hostname=".".join(hostname_parts))
        except IndexError:
            url = None

        if url:
            host = url.hostname
            owner = url.owner
            name = url.name

        remote = {"host": host, "owner": owner, "name": name}
        self._remote_cache[original_remote_name] = remote

        return remote
示例#13
0
def clone(
    url,
    path=None,
    install_githooks=True,
    install_lfs=True,
    skip_smudge=True,
    recursive=True,
    depth=None,
    progress=None,
    config=None,
    raise_git_except=False,
    checkout_rev=None,
):
    """Clone Renku project repo, install Git hooks and LFS."""
    from renku.core.management.client import LocalClient

    path = path or GitURL.parse(url).name

    if isinstance(path, Path):
        path = str(path)

    # Clone the project
    if skip_smudge:
        os.environ["GIT_LFS_SKIP_SMUDGE"] = "1"

    try:
        repo = Repo.clone_from(url, path, recursive=recursive, depth=depth, progress=progress)
    except GitCommandError as e:
        if not raise_git_except:
            raise errors.GitError("Cannot clone remote Renku project: {}".format(url)) from e

        raise e

    remote_refs = [Path(ref.abspath).name for ref in repo.remote().refs]

    if checkout_rev in remote_refs:
        repo.git.checkout(checkout_rev)
    elif checkout_rev:
        repo.git.checkout(checkout_rev, b=checkout_rev)

    if config:
        config_writer = repo.config_writer()

        for key, value in config.items():
            key_path = key.split(".")
            key = key_path.pop()

            if not key_path or not key:
                raise errors.GitError("Cannot write to config. Section path or key is invalid.")

            config_writer.set_value(".".join(key_path), key, value)

        config_writer.release()

    client = LocalClient(path)

    if install_githooks:
        install(client=client, force=True)

    if install_lfs:
        command = ["git", "lfs", "install", "--local", "--force"]
        if skip_smudge:
            command += ["--skip-smudge"]
        try:
            repo.git.execute(command=command, with_exceptions=True)
        except GitCommandError as e:
            raise errors.GitError("Cannot install Git LFS") from e

    return repo
示例#14
0
def test_valid_href(fields):
    """Test the various repo regexes."""
    fields.pop("protocols", None)
    assert GitURL(**fields) == GitURL.parse(fields["href"])
示例#15
0
def clone(
    url,
    path=None,
    install_githooks=True,
    install_lfs=True,
    skip_smudge=True,
    recursive=True,
    depth=None,
    progress=None,
    config=None,
    raise_git_except=False,
):
    """Clone Renku project repo, install Git hooks and LFS."""
    from renku.core.management.client import LocalClient

    path = path or GitURL.parse(url).name

    if isinstance(path, Path):
        path = str(path)

    # Clone the project
    if skip_smudge:
        os.environ['GIT_LFS_SKIP_SMUDGE'] = '1'

    try:
        repo = Repo.clone_from(url,
                               path,
                               recursive=recursive,
                               depth=depth,
                               progress=progress)
    except GitCommandError as e:
        if not raise_git_except:
            raise errors.GitError(
                'Cannot clone remote Renku project: {}'.format(url)) from e

        raise e

    if config:
        config_writer = repo.config_writer()

        for key, value in config.items():
            key_path = key.split('.')
            key = key_path.pop()

            if not key_path or not key:
                raise errors.GitError(
                    'Cannot write to config. Section path or key is invalid.')

            config_writer.set_value('.'.join(key_path), key, value)

        config_writer.release()

    client = LocalClient(path)

    if install_githooks:
        install(client=client, force=True)

    if install_lfs:
        command = ['git', 'lfs', 'install', '--local', '--force']
        if skip_smudge:
            command += ['--skip-smudge']
        try:
            repo.git.execute(command=command, with_exceptions=True)
        except GitCommandError as e:
            raise errors.GitError('Cannot install Git LFS') from e

    return repo