async def _metadata_folder(
self, path: GitLabPath) -> typing.List[BaseGitLabMetadata]:
"""Fetch metadata for the contents of the folder at ``path`` and return a `list` of
`GitLabFileMetadata` and `GitLabFolderMetadata` objects.
:param GitLabPath path: `GitLabPath` representing a folder
:rtype: `list`
"""
data = await self._fetch_tree_contents(path)
ret = [] # type: typing.List[BaseGitLabMetadata]
for item in data:
name = item['name']
if item['type'] == 'tree':
folder_path = path.child(name, folder=True)
ret.append(GitLabFolderMetadata(item, folder_path))
else:
file_path = path.child(name, folder=False)
item['mime_type'] = mimetypes.guess_type(name)[0]
ret.append(
GitLabFileMetadata(item,
file_path,
host=self.VIEW_URL,
owner=self.owner,
repo=self.repo))
return ret
async def test_metadata_file_with_branch(self, provider):
path = '/folder1/folder2/file'
gl_path = GitLabPath(path, _ids=([(None, 'my-branch')] * 4))
url = ('http://base.url/api/v4/projects/123/repository/files/'
'folder1%2Ffolder2%2Ffile?ref=my-branch')
aiohttpretty.register_json_uri('GET',
url,
body=fixtures.simple_file_metadata())
history_url = ('http://base.url/api/v4/projects/123/repository/commits'
'?path=folder1/folder2/file&ref_name=my-branch&page=1'
'&per_page={}'.format(provider.MAX_PAGE_SIZE))
aiohttpretty.register_json_uri('GET',
history_url,
body=fixtures.revisions_for_file())
result = await provider.metadata(gl_path)
assert result.json_api_serialized('mst3k')['links'] == {
'move':
('http://localhost:7777/v1/resources/mst3k/providers/gitlab'
'/folder1/folder2/file?branch=my-branch'),
'upload':
None,
'download':
('http://localhost:7777/v1/resources/mst3k/providers/gitlab'
'/folder1/folder2/file?branch=my-branch'),
'delete':
None,
}
async def test_download(self, provider):
path = '/folder1/file.py'
gl_path = GitLabPath(path, _ids=([('a1b2c3d4', 'master')] * 3))
url = ('http://base.url/api/v4/projects/123/repository/files'
'/folder1%2Ffile.py/raw?ref=a1b2c3d4')
aiohttpretty.register_uri('GET', url, body=b'hello', headers={'X-Gitlab-Size': '5'})
result = await provider.download(gl_path, branch='master')
assert await result.read() == b'hello'
async def test_metadata_file_with_default_ref(self, provider):
path = '/folder1/folder2/file'
gl_path = GitLabPath(path, _ids=([('a1b2c3d4', 'master')] * 4))
url = ('http://base.url/api/v4/projects/123/repository/files/'
'folder1%2Ffolder2%2Ffile?ref=a1b2c3d4')
aiohttpretty.register_json_uri('GET',
url,
body=fixtures.simple_file_metadata())
history_url = ('http://base.url/api/v4/projects/123/repository/commits'
'?path=folder1/folder2/file&ref_name=a1b2c3d4&page=1'
'&per_page={}'.format(provider.MAX_PAGE_SIZE))
aiohttpretty.register_json_uri('GET',
history_url,
body=fixtures.revisions_for_file())
etag = hashlib.sha256('{}::{}::{}'.format('gitlab', path, 'a1b2c3d4').encode('utf-8'))\
.hexdigest()
result = await provider.metadata(gl_path)
assert result.serialized() == {
'name': 'file',
'kind': 'file',
'size': 123,
'sizeInt': 123,
'provider': 'gitlab',
'path': path,
'materialized': path,
'modified': '2017-07-24T16:02:17.000-04:00',
'modified_utc': '2017-07-24T20:02:17+00:00',
'created_utc': '2016-11-30T18:30:23+00:00',
'contentType': None,
'etag': etag,
'extra': {
'commitSha':
'a1b2c3d4',
'branch':
'master',
'webView':
'http://base.url/cat/food/blob/master/folder1/folder2/file',
},
}
assert result.json_api_serialized('mst3k')['links'] == {
'move':
('http://localhost:7777/v1/resources/mst3k/providers/gitlab'
'/folder1/folder2/file?commitSha=a1b2c3d4'),
'upload':
None,
'download':
('http://localhost:7777/v1/resources/mst3k/providers/gitlab'
'/folder1/folder2/file?commitSha=a1b2c3d4'),
'delete':
None,
}
async def test_no_such_revision(self, provider):
path = '/folder1/folder2/file'
gl_path = GitLabPath(path, _ids=([('a1b2c3d4', 'master')] * 4))
url = ('http://base.url/api/v4/projects/123/repository/commits'
'?path=folder1/folder2/file&ref_name=a1b2c3d4')
aiohttpretty.register_json_uri('GET', url, body=[])
with pytest.raises(exceptions.RevisionsError) as exc:
await provider.revisions(gl_path)
assert exc.value.code == 404
async def validate_path(self, path: str, **kwargs) -> GitLabPath:
"""Turn the string ``path`` into a `GitLabPath` object. Will infer the branch/commit
information from the query params or from the default branch for the repo if those are
not provided. Does no validation to ensure that the entity described by ``path`` actually
exists.
Valid kwargs are ``commitSha``, ``branch``, and ``revision``. If ``revision`` is given,
its value will be assigned to the commit SHA if it is a valid base-16 number, or branch
otherwise. ``revision`` will override ``commitSha`` or ``branch``. If both a commit SHA
and branch name are given, both will be associated with the new GitLabPath object. No
effort is made to ensure that they point to the same thing. `GitLabPath` objects default
to commit SHAs over branch names when building API calls, as a commit SHA is more specific.
:param str path: The path to a file
:rtype: GitLabPath
"""
commit_sha = kwargs.get('commitSha')
branch_name = kwargs.get('branch')
# revision query param could be commit sha OR branch
# take a guess which one it will be.
revision = kwargs.get('revision', None)
if revision is not None:
try:
int(revision, 16) # is revision valid hex?
except (TypeError, ValueError):
branch_name = revision
else:
commit_sha = revision
if not commit_sha and not branch_name:
branch_name = await self._fetch_default_branch()
if path == '/':
return GitLabPath(path, _ids=[(commit_sha, branch_name)])
gl_path = GitLabPath(path)
for part in gl_path.parts:
part._id = (commit_sha, branch_name)
return gl_path
async def test_metadata_file_no_such_file(self, provider):
path = '/folder1/folder2/file'
gl_path = GitLabPath(path, _ids=([('a1b2c3d4', 'master')] * 4))
url = ('http://base.url/api/v4/projects/123/repository/files/'
'folder1%2Ffolder2%2Ffile?ref=a1b2c3d4')
aiohttpretty.register_json_uri('GET', url, body={}, status=404)
with pytest.raises(exceptions.NotFoundError) as exc:
await provider.metadata(gl_path)
assert exc.value.code == 404
async def test_metadata_folder_no_such_folder_404(self, provider):
path = '/folder1/folder2/folder3/'
gl_path = GitLabPath(path, _ids=([('a1b2c3d4', 'master')] * 4))
url = ('http://base.url/api/v4/projects/123/repository/tree'
'?path=folder1/folder2/folder3/&ref=a1b2c3d4&page=1'
'&per_page={}'.format(provider.MAX_PAGE_SIZE))
aiohttpretty.register_json_uri('GET', url, body={}, status=404)
with pytest.raises(exceptions.NotFoundError) as exc:
await provider.metadata(gl_path)
assert exc.value.code == 404
async def test_download_file_ruby_response(self, provider):
"""See: https://gitlab.com/gitlab-org/gitlab-ce/issues/31790"""
path = '/folder1/folder2/file'
gl_path = GitLabPath(path, _ids=([(None, 'my-branch')] * 4))
url = ('http://base.url/api/v4/projects/123/repository/files/'
'folder1%2Ffolder2%2Ffile?ref=my-branch')
aiohttpretty.register_uri('GET',
url,
body=fixtures.weird_ruby_response())
result = await provider.download(gl_path)
assert await result.read() == b'rolf\n'
async def _metadata_folder(self, path: GitLabPath) -> typing.List[BaseGitLabMetadata]:
"""Fetch metadata for the contents of the folder at ``path`` and return a `list` of
`GitLabFileMetadata` and `GitLabFolderMetadata` objects.
:param GitLabPath path: `GitLabPath` representing a folder
:rtype: `list`
"""
data = await self._fetch_tree_contents(path)
ret = [] # type: typing.List[BaseGitLabMetadata]
for item in data:
name = item['name']
if item['type'] == 'tree':
folder_path = path.child(name, folder=True)
ret.append(GitLabFolderMetadata(item, folder_path))
else:
file_path = path.child(name, folder=False)
item['mime_type'] = mimetypes.guess_type(name)[0]
ret.append(GitLabFileMetadata(item, file_path, host=self.VIEW_URL,
owner=self.owner, repo=self.repo))
return ret
async def test_metadata_folder(self, provider):
path = '/folder1/folder2/folder3/'
gl_path = GitLabPath(path, _ids=([('a1b2c3d4', 'master')] * 4))
url = ('http://base.url/api/v4/projects/123/repository/tree'
'?path=folder1/folder2/folder3/&ref=a1b2c3d4&page=1'
'&per_page={}'.format(provider.MAX_PAGE_SIZE))
aiohttpretty.register_json_uri('GET',
url,
body=[{
'id': '123',
'type': 'tree',
'name': 'my folder'
}, {
'id': '1234',
'type': 'file',
'name': 'my file'
}])
result = await provider.metadata(gl_path)
assert isinstance(result[0], GitLabFolderMetadata)
assert result[0].name == 'my folder'
assert result[0].json_api_serialized('mst3k')['links'] == {
'move':
('http://localhost:7777/v1/resources/mst3k/providers/gitlab'
'/folder1/folder2/folder3/my%20folder/?commitSha=a1b2c3d4'),
'upload':
None,
'delete':
None,
'new_folder':
None,
}
assert result[1].name == 'my file'
assert isinstance(result[1], GitLabFileMetadata)
child_path = provider.path_from_metadata(gl_path, result[1])
child_path.name == 'my file'
child_path.commit_sha == 'a1b2c3d4'
child_path.branch_name == 'master'
async def test_download_file_ruby_response_range(self, provider):
"""See: https://gitlab.com/gitlab-org/gitlab-ce/issues/31790"""
path = '/folder1/folder2/file'
gl_path = GitLabPath(path, _ids=([(None, 'my-branch')] * 4))
url = ('http://base.url/api/v4/projects/123/repository/files/'
'folder1%2Ffolder2%2Ffile?ref=my-branch')
aiohttpretty.register_uri('GET',
url,
body=fixtures.weird_ruby_response())
result = await provider.download(gl_path, range=(0, 1))
assert result.partial
assert await result.read() == b'ro'
assert aiohttpretty.has_call(method='GET',
uri=url,
headers={
'Range': 'bytes=0-1',
'PRIVATE-TOKEN': 'naps'
})
async def test_revisions(self, provider, revisions_for_file):
path = '/folder1/folder2/file'
gl_path = GitLabPath(path, _ids=([('a1b2c3d4', 'master')] * 4))
url = ('http://base.url/api/v4/projects/123/repository/commits'
'?path=folder1/folder2/file&ref_name=a1b2c3d4')
aiohttpretty.register_json_uri('GET', url, body=revisions_for_file)
revisions = await provider.revisions(gl_path)
assert len(revisions) == 3
assert revisions[0].serialized() == {
'version': '931aece9275c0d084dfa7f6e0b3b2bb250e4b089',
'modified': '2017-07-24T16:02:17.000-04:00',
'modified_utc': '2017-07-24T20:02:17+00:00',
'versionIdentifier': 'commitSha',
'extra': {
'user': {
'name': 'Fitz Elliott',
},
},
}
async def test_metadata_file_ruby_response(self, provider):
"""See: https://gitlab.com/gitlab-org/gitlab-ce/issues/31790"""
path = '/folder1/folder2/file'
gl_path = GitLabPath(path, _ids=([(None, 'my-branch')] * 4))
url = ('http://base.url/api/v4/projects/123/repository/files/'
'folder1%2Ffolder2%2Ffile?ref=my-branch')
aiohttpretty.register_uri('GET',
url,
body=fixtures.weird_ruby_response())
history_url = ('http://base.url/api/v4/projects/123/repository/commits'
'?path=folder1/folder2/file&ref_name=my-branch&page=1'
'&per_page={}'.format(provider.MAX_PAGE_SIZE))
aiohttpretty.register_json_uri('GET',
history_url,
body=fixtures.revisions_for_file())
result = await provider.metadata(gl_path)
assert result.name == 'file'
assert result.size == 5
assert result.content_type == None
async def test_download_range(self, provider):
path = '/folder1/file.py'
gl_path = GitLabPath(path, _ids=([('a1b2c3d4', 'master')] * 3))
url = ('http://base.url/api/v4/projects/123/repository/files'
'/folder1%2Ffile.py?ref=a1b2c3d4')
aiohttpretty.register_json_uri('GET',
url,
body={'content': 'aGVsbG8='})
result = await provider.download(gl_path,
branch='master',
range=(0, 1))
assert result.partial
assert await result.read(
) == b'he' # body content after base64 decoding and slice
assert aiohttpretty.has_call(method='GET',
uri=url,
headers={
'Range': 'bytes=0-1',
'PRIVATE-TOKEN': 'naps'
})
def path_from_metadata(
self, # type: ignore
parent_path: GitLabPath,
metadata) -> GitLabPath:
"""Build a GitLabPath for a the child of ``parent_path`` described by ``metadata``."""
return parent_path.child(metadata.name, folder=metadata.is_folder)
def path_from_metadata(self, # type: ignore
parent_path: GitLabPath,
metadata) -> GitLabPath:
"""Build a GitLabPath for a the child of ``parent_path`` described by ``metadata``."""
return parent_path.child(metadata.name, folder=metadata.is_folder)