def test_file_metadata_with_ref(self, file_metadata_tree_endpoint):
try:
metadata = GitHubFileTreeMetadata(file_metadata_tree_endpoint, ref="some-branch")
except Exception as exc:
pytest.fail(str(exc))
assert metadata.name == "README.md"
assert metadata.path == "/README.md"
assert metadata.modified is None
assert metadata.content_type is None
assert metadata.size == 38
assert metadata.etag == "/README.md::d863d70539aa9fcb6b44b057221706f2ab18e341"
assert metadata.extra == {
"fileSha": "d863d70539aa9fcb6b44b057221706f2ab18e341",
"webView": None,
"ref": "some-branch",
}
assert metadata.provider == "github"
assert metadata.commit is None
assert metadata.ref == "some-branch"
assert metadata.web_view is None
json_api = metadata.json_api_serialized("mst3k")
for actions, link in json_api["links"].items():
assert re.search("[?&]ref=some-branch", link)
def test_file_metadata_with_ref(self, metadata_fixtures):
try:
metadata = GitHubFileTreeMetadata(metadata_fixtures['file_metadata_tree_endpoint'], ref="some-branch")
except Exception as exc:
pytest.fail(str(exc))
assert metadata.name == 'README.md'
assert metadata.path == '/README.md'
assert metadata.modified is None
assert metadata.content_type is None
assert metadata.size == 38
assert metadata.size_as_int == 38
assert type(metadata.size_as_int) == int
assert metadata.etag == '/README.md::d863d70539aa9fcb6b44b057221706f2ab18e341'
assert metadata.extra == {
'fileSha': 'd863d70539aa9fcb6b44b057221706f2ab18e341',
'webView': None,
'ref': 'some-branch',
'hashes': {
'git': 'd863d70539aa9fcb6b44b057221706f2ab18e341',
},
}
assert metadata.provider == 'github'
assert metadata.commit is None
assert metadata.ref == 'some-branch'
assert metadata.web_view is None
json_api = metadata.json_api_serialized('mst3k')
for actions, link in json_api['links'].items():
assert re.search('[?&]ref=some-branch', link)
def test_file_metadata_with_ref(self, file_metadata_tree_endpoint):
try:
metadata = GitHubFileTreeMetadata(file_metadata_tree_endpoint,
ref="some-branch")
except Exception as exc:
pytest.fail(str(exc))
assert metadata.name == 'README.md'
assert metadata.path == '/README.md'
assert metadata.modified is None
assert metadata.content_type is None
assert metadata.size == 38
assert metadata.etag == '/README.md::d863d70539aa9fcb6b44b057221706f2ab18e341'
assert metadata.extra == {
'fileSha': 'd863d70539aa9fcb6b44b057221706f2ab18e341',
'webView': None,
'ref': 'some-branch',
}
assert metadata.provider == 'github'
assert metadata.commit is None
assert metadata.ref == 'some-branch'
assert metadata.web_view is None
json_api = metadata.json_api_serialized('mst3k')
for actions, link in json_api['links'].items():
assert re.search('[?&]ref=some-branch', link)
async def _metadata_file(self, path, revision=None, **kwargs):
resp = await self.make_request(
'GET',
self.build_repo_url('commits',
path=path.path,
sha=revision or path.branch_ref),
expects=(200, ),
throws=exceptions.MetadataError,
)
commits = await resp.json()
if not commits:
raise exceptions.NotFoundError(str(path))
latest = commits[0]
tree = await self._fetch_tree(latest['commit']['tree']['sha'],
recursive=True)
try:
data = next(x for x in tree['tree'] if x['path'] == path.path)
except StopIteration:
raise exceptions.NotFoundError(str(path))
if isinstance(data, list):
raise exceptions.MetadataError(
'Could not retrieve file "{0}"'.format(str(path)),
code=404,
)
return GitHubFileTreeMetadata(data,
commit=latest['commit'],
web_view=self._web_view(path),
ref=path.branch_ref)
async def upload(self, stream, path, message=None, branch=None, **kwargs):
assert self.name is not None
assert self.email is not None
try:
exists = await self.exists(path)
except exceptions.ProviderError as e:
if e.data.get('message') == 'Git Repository is empty.':
exists = False
resp = await self.make_request(
'PUT',
self.build_repo_url('contents', '.gitkeep'),
data=json.dumps({
'content': '',
'path': '.gitkeep',
'committer': self.committer,
'branch': path.identifier[0],
'message': 'Initial commit'
}),
expects=(201, ),
throws=exceptions.CreateFolderError)
data = await resp.json()
latest_sha = data['commit']['sha']
else:
latest_sha = await self._get_latest_sha(ref=path.identifier[0])
blob = await self._create_blob(stream)
tree = await self._create_tree({
'base_tree':
latest_sha,
'tree': [{
'path': path.path,
'mode': '100644',
'type': 'blob',
'sha': blob['sha']
}]
})
commit = await self._create_commit({
'tree':
tree['sha'],
'parents': [latest_sha],
'committer':
self.committer,
'message':
message or (settings.UPDATE_FILE_MESSAGE
if exists else settings.UPLOAD_FILE_MESSAGE),
})
# Doesn't return anything useful
await self._update_ref(commit['sha'], ref=path.identifier[0])
# You're hacky
return GitHubFileTreeMetadata(
{
'path': path.path,
'sha': blob['sha'],
'size': stream.size,
},
commit=commit), not exists
def test_build_file_metadata_from_tree(self, metadata_fixtures):
try:
metadata = GitHubFileTreeMetadata(
metadata_fixtures['file_metadata_tree_endpoint'])
except Exception as exc:
pytest.fail(str(exc))
assert metadata.name == 'README.md'
assert metadata.path == '/README.md'
assert metadata.modified is None
assert metadata.content_type is None
assert metadata.size == 38
assert metadata.size_as_int == 38
assert type(metadata.size_as_int) == int
assert metadata.etag == '/README.md::d863d70539aa9fcb6b44b057221706f2ab18e341'
assert metadata.extra == {
'fileSha': 'd863d70539aa9fcb6b44b057221706f2ab18e341',
'webView': None,
'hashes': {
'git': 'd863d70539aa9fcb6b44b057221706f2ab18e341',
},
}
assert metadata.provider == 'github'
assert metadata.commit is None
assert metadata.ref is None
assert metadata.web_view is None
def test_metadata_file(self, provider, repo_metadata, repo_tree_metadata_root):
ref = hashlib.sha1().hexdigest()
path = yield from provider.validate_path('/file.txt')
tree_url = provider.build_repo_url('git', 'trees', ref, recursive=1)
latest_sha_url = provider.build_repo_url('git', 'refs', 'heads', path.identifier[0])
aiohttpretty.register_json_uri('GET', tree_url, body=repo_tree_metadata_root)
aiohttpretty.register_json_uri('GET', latest_sha_url, body={'object': {'sha': ref}})
result = yield from provider.metadata(path)
assert result == GitHubFileTreeMetadata(repo_tree_metadata_root['tree'][0]).serialized()
def test_metadata_file(self, provider, repo_metadata,
repo_tree_metadata_root):
ref = hashlib.sha1().hexdigest()
path = yield from provider.validate_path('/file.txt')
tree_url = provider.build_repo_url('git', 'trees', ref, recursive=1)
commit_url = provider.build_repo_url('commits',
path=path.path.lstrip('/'),
sha=path.identifier[0])
aiohttpretty.register_json_uri('GET',
tree_url,
body=repo_tree_metadata_root)
aiohttpretty.register_json_uri('GET',
commit_url,
body=[{
'commit': {
'tree': {
'sha': ref
},
'author': {
'date':
'this is totally date'
}
},
}])
result = yield from provider.metadata(path)
item = repo_tree_metadata_root['tree'][0]
web_view = provider._web_view(path=path)
assert result == GitHubFileTreeMetadata(item,
web_view=web_view,
commit={
'tree': {
'sha': ref
},
'author': {
'date':
'this is totally date'
}
})
def _metadata_file(self, path, ref=None, **kwargs):
if not GitHubProvider.is_sha(path.identifier[0]):
latest = yield from self._get_latest_sha(ref=path.identifier[0])
else:
latest = path.identifier[0]
tree = yield from self._fetch_tree(latest, recursive=True)
try:
data = next(x for x in tree['tree'] if x['path'] == path.path)
except StopIteration:
raise exceptions.MetadataError(';', code=404)
if isinstance(data, list):
raise exceptions.MetadataError(
'Could not retrieve file "{0}"'.format(str(path)),
code=404,
)
return GitHubFileTreeMetadata(data).serialized()
def upload(self, stream, path, message=None, branch=None, **kwargs):
assert self.name is not None
assert self.email is not None
exists = yield from self.exists(path)
latest_sha = yield from self._get_latest_sha(ref=path.identifier[0])
blob = yield from self._create_blob(stream)
tree = yield from self._create_tree({
'base_tree':
latest_sha,
'tree': [{
'path': path.path,
'mode': '100644',
'type': 'blob',
'sha': blob['sha']
}]
})
commit = yield from self._create_commit({
'tree':
tree['sha'],
'parents': [latest_sha],
'committer':
self.committer,
'message':
message or settings.UPLOAD_FILE_MESSAGE,
})
# Doesn't return anything useful
yield from self._update_ref(commit['sha'], ref=path.identifier[0])
# You're hacky
return GitHubFileTreeMetadata(
{
'path': path.path,
'sha': blob['sha'],
'size': stream.size,
},
commit=commit).serialized(), not exists
async def _do_intra_move_or_copy(self, src_path, dest_path, is_copy):
# ON PATHS:
# WB and GH use slightly different default conventions for their paths, so we often
# have to munge our WB paths before comparison. Here is a quick overview:
# WB (dirs): wb_dir.path == 'foo/bar/' str(wb_dir) == '/foo/bar/'
# WB (file): wb_file.path = 'foo/bar.txt' str(wb_file) == '/foo/bar.txt'
# GH (dir): 'foo/bar'
# GH (file): 'foo/bar.txt'
src_tree, src_head = await self._get_tree_and_head(src_path.branch_ref)
# these are the blobs to copy/move
blobs = [
item for item in src_tree['tree']
if src_path.is_dir and item['path'].startswith(src_path.path)
or src_path.is_file and item['path'] == src_path.path
]
if len(blobs) == 0:
raise exceptions.NotFoundError(str(src_path))
if src_path.is_file:
assert len(blobs) == 1, 'Found multiple targets'
commit_msg = settings.COPY_MESSAGE if is_copy else settings.MOVE_MESSAGE
commit = None
if src_path.branch_ref == dest_path.branch_ref:
exists = self._path_exists_in_tree(src_tree['tree'], dest_path)
# if we're overwriting an existing dir, we must remove its blobs from the tree
if dest_path.is_dir:
src_tree['tree'] = self._remove_path_from_tree(
src_tree['tree'], dest_path)
# if this is a copy, duplicate and append our source blobs. The originals will be updated
# with the new destination path.
if is_copy:
src_tree['tree'].extend(copy.deepcopy(blobs))
# see, I told you they'd be overwritten
self._reparent_blobs(blobs, src_path, dest_path)
src_tree['tree'] = self._prune_subtrees(src_tree['tree'])
commit = await self._commit_tree_and_advance_branch(
src_tree['tree'], {'sha': src_head}, commit_msg,
src_path.branch_ref)
else:
dest_tree, dest_head = await self._get_tree_and_head(
dest_path.branch_ref)
exists = self._path_exists_in_tree(dest_tree['tree'], dest_path)
dest_tree['tree'] = self._remove_path_from_tree(
dest_tree['tree'], dest_path)
new_blobs = copy.deepcopy(blobs)
self._reparent_blobs(new_blobs, src_path, dest_path)
dest_tree['tree'].extend(new_blobs)
dest_tree['tree'] = self._prune_subtrees(dest_tree['tree'])
commit = await self._commit_tree_and_advance_branch(
dest_tree['tree'], {'sha': dest_head}, commit_msg,
dest_path.branch_ref)
if not is_copy:
src_tree['tree'] = self._remove_path_from_tree(
src_tree['tree'], src_path)
src_tree['tree'] = self._prune_subtrees(src_tree['tree'])
await self._commit_tree_and_advance_branch(
src_tree['tree'], {'sha': src_head}, commit_msg,
src_path.branch_ref)
blobs = new_blobs # for the metadata
if dest_path.is_file:
assert len(
blobs
) == 1, 'Destination file should have exactly one candidate'
return GitHubFileTreeMetadata(blobs[0],
commit=commit,
ref=dest_path.branch_ref), not exists
folder = GitHubFolderTreeMetadata({'path': dest_path.path.strip('/')},
commit=commit,
ref=dest_path.branch_ref)
folder.children = []
for item in blobs:
if item['path'] == dest_path.path.rstrip('/'):
continue
if item['type'] == 'tree':
folder.children.append(
GitHubFolderTreeMetadata(item, ref=dest_path.branch_ref))
else:
folder.children.append(
GitHubFileTreeMetadata(item, ref=dest_path.branch_ref))
return folder, not exists
async def _do_intra_move_or_copy(self, src_path, dest_path, is_copy):
# ON PATHS:
# WB and GH use slightly different default conventions for their paths, so we often
# have to munge our WB paths before comparison. Here is a quick overview:
# WB (dirs): wb_dir.path == 'foo/bar/' str(wb_dir) == '/foo/bar/'
# WB (file): wb_file.path = 'foo/bar.txt' str(wb_file) == '/foo/bar.txt'
# GH (dir): 'foo/bar'
# GH (file): 'foo/bar.txt'
branch = src_path.identifier[0]
branch_data = await self._fetch_branch(branch)
old_commit_sha = branch_data['commit']['sha']
old_commit_tree_sha = branch_data['commit']['commit']['tree']['sha']
tree = await self._fetch_tree(old_commit_tree_sha, recursive=True)
exists = any(x['path'] == dest_path.path.rstrip('/')
for x in tree['tree'])
# these are the blobs to copy/move
blobs = [
item for item in tree['tree']
if src_path.is_dir and item['path'].startswith(src_path.path)
or src_path.is_file and item['path'] == src_path.path
]
# if we're overwriting an existing dir, we must remove its blobs from the tree
if dest_path.is_dir:
tree['tree'] = [
item for item in tree['tree']
if not item['path'].startswith(dest_path.path)
]
if len(blobs) == 0:
raise exceptions.NotFoundError(str(src_path))
if src_path.is_file:
assert len(blobs) == 1, 'Found multiple targets'
# if this is a copy, duplicate and append our source blobs. The originals will be updated
# with the new destination path.
if is_copy:
tree['tree'].extend([copy.deepcopy(blob) for blob in blobs])
# see, I told you they'd be overwritten
for blob in blobs:
blob['path'] = blob['path'].replace(src_path.path, dest_path.path,
1)
# github infers tree contents from blob paths
# see: http://www.levibotelho.com/development/commit-a-file-with-the-github-api/
tree['tree'] = [
item for item in tree['tree'] if item['type'] != 'tree'
]
new_tree_data = await self._create_tree({'tree': tree['tree']})
new_tree_sha = new_tree_data['sha']
# Create a new commit which references our top most tree change.
commit_resp = await self.make_request(
'POST',
self.build_repo_url('git', 'commits'),
headers={'Content-Type': 'application/json'},
data=json.dumps({
'tree':
new_tree_sha,
'parents': [old_commit_sha],
'committer':
self.committer,
'message':
settings.COPY_MESSAGE if is_copy else settings.MOVE_MESSAGE
}),
expects=(201, ),
throws=exceptions.DeleteError,
)
commit = await commit_resp.json()
# Update repository reference, point to the newly created commit.
# No need to store data, rely on expects to raise exceptions
resp = await self.make_request(
'PATCH',
self.build_repo_url('git', 'refs', 'heads', branch),
headers={'Content-Type': 'application/json'},
data=json.dumps({'sha': commit['sha']}),
expects=(200, ),
throws=exceptions.DeleteError,
)
await resp.release()
if dest_path.is_file:
assert len(
blobs
) == 1, 'Destination file should have exactly one candidate'
return GitHubFileTreeMetadata(blobs[0], commit=commit), not exists
folder = GitHubFolderTreeMetadata({'path': dest_path.path.strip('/')},
commit=commit)
folder.children = []
for item in blobs:
if item['path'] == src_path.path.rstrip('/'):
continue
if item['type'] == 'tree':
folder.children.append(GitHubFolderTreeMetadata(item))
else:
folder.children.append(GitHubFileTreeMetadata(item))
return folder, not exists
def _do_intra_move_or_copy(self, src_path, dest_path, is_copy):
target, branch = None, src_path.identifier[0]
branch_data = yield from self._fetch_branch(branch)
old_commit_sha = branch_data['commit']['sha']
old_commit_tree_sha = branch_data['commit']['commit']['tree']['sha']
tree = yield from self._fetch_tree(old_commit_tree_sha, recursive=True)
exists = any(x['path'] == dest_path.path for x in tree['tree'])
target, keep = None, []
for item in tree['tree']:
if item['path'] == str(src_path).strip('/'):
assert target is None, 'Found multiple targets'
target = item
elif item['path'].startswith(src_path.path):
keep.append(item)
if target is None or (src_path.is_dir and target['type'] != 'tree'):
raise exceptions.NotFoundError(str(src_path))
if is_copy:
tree['tree'].append(copy.deepcopy(target))
elif src_path.is_dir:
for item in keep:
tree['tree'].remove(item)
target['path'] = target['path'].replace(src_path.path.strip('/'),
dest_path.path.strip('/'), 1)
new_tree_data = yield from self._create_tree({'tree': tree['tree']})
new_tree_sha = new_tree_data['sha']
# Create a new commit which references our top most tree change.
commit_resp = yield from self.make_request(
'POST',
self.build_repo_url('git', 'commits'),
headers={'Content-Type': 'application/json'},
data=json.dumps({
'tree':
new_tree_sha,
'parents': [old_commit_sha],
'committer':
self.committer,
'message':
settings.COPY_MESSAGE if is_copy else settings.MOVE_MESSAGE
}),
expects=(201, ),
throws=exceptions.DeleteError,
)
commit = yield from commit_resp.json()
# Update repository reference, point to the newly created commit.
# No need to store data, rely on expects to raise exceptions
yield from self.make_request(
'PATCH',
self.build_repo_url('git', 'refs', 'heads', branch),
headers={'Content-Type': 'application/json'},
data=json.dumps({'sha': commit['sha']}),
expects=(200, ),
throws=exceptions.DeleteError,
)
if dest_path.is_file:
return GitHubFileTreeMetadata(target, commit=commit), not exists
folder = GitHubFolderTreeMetadata({'path': dest_path.path.strip('/')},
commit=commit)
folder.children = []
for item in keep:
item['path'] = item['path'].replace(src_path.path, dest_path.path,
1)
if item['type'] == 'tree':
folder.children.append(GitHubFolderTreeMetadata(item))
else:
folder.children.append(GitHubFileTreeMetadata(item))
return folder, exists
