def test_build_folder_metadata_from_tree(self,
folder_metadata_tree_endpoint):
try:
metadata = GitHubFolderTreeMetadata(folder_metadata_tree_endpoint)
except Exception as exc:
pytest.fail(str(exc))
assert metadata.name == 'lorch'
assert metadata.path == '/foldera/folderb/lorch/'
assert metadata.extra == {}
assert metadata.provider == 'github'
assert metadata.commit is None
assert metadata.ref is None
async def _do_intra_move_or_copy(self, src_path, dest_path, is_copy):
# ON PATHS:
# WB and GH use slightly different default conventions for their paths, so we often
# have to munge our WB paths before comparison. Here is a quick overview:
# WB (dirs): wb_dir.path == 'foo/bar/' str(wb_dir) == '/foo/bar/'
# WB (file): wb_file.path = 'foo/bar.txt' str(wb_file) == '/foo/bar.txt'
# GH (dir): 'foo/bar'
# GH (file): 'foo/bar.txt'
branch = src_path.identifier[0]
branch_data = await self._fetch_branch(branch)
old_commit_sha = branch_data['commit']['sha']
old_commit_tree_sha = branch_data['commit']['commit']['tree']['sha']
tree = await self._fetch_tree(old_commit_tree_sha, recursive=True)
exists = any(x['path'] == dest_path.path.rstrip('/') for x in tree['tree'])
# these are the blobs to copy/move
blobs = [
item
for item in tree['tree']
if src_path.is_dir and item['path'].startswith(src_path.path) or
src_path.is_file and item['path'] == src_path.path
]
# if we're overwriting an existing dir, we must remove its blobs from the tree
if dest_path.is_dir:
tree['tree'] = [
item
for item in tree['tree']
if not item['path'].startswith(dest_path.path)
]
if len(blobs) == 0:
raise exceptions.NotFoundError(str(src_path))
if src_path.is_file:
assert len(blobs) == 1, 'Found multiple targets'
# if this is a copy, duplicate and append our source blobs. The originals will be updated
# with the new destination path.
if is_copy:
tree['tree'].extend([copy.deepcopy(blob) for blob in blobs])
# see, I told you they'd be overwritten
for blob in blobs:
blob['path'] = blob['path'].replace(src_path.path, dest_path.path, 1)
# github infers tree contents from blob paths
# see: http://www.levibotelho.com/development/commit-a-file-with-the-github-api/
tree['tree'] = [item for item in tree['tree'] if item['type'] != 'tree']
new_tree_data = await self._create_tree({'tree': tree['tree']})
new_tree_sha = new_tree_data['sha']
# Create a new commit which references our top most tree change.
commit_resp = await self.make_request(
'POST',
self.build_repo_url('git', 'commits'),
headers={'Content-Type': 'application/json'},
data=json.dumps({
'tree': new_tree_sha,
'parents': [old_commit_sha],
'committer': self.committer,
'message': settings.COPY_MESSAGE if is_copy else settings.MOVE_MESSAGE
}),
expects=(201, ),
throws=exceptions.DeleteError,
)
commit = await commit_resp.json()
# Update repository reference, point to the newly created commit.
# No need to store data, rely on expects to raise exceptions
resp = await self.make_request(
'PATCH',
self.build_repo_url('git', 'refs', 'heads', branch),
headers={'Content-Type': 'application/json'},
data=json.dumps({'sha': commit['sha']}),
expects=(200, ),
throws=exceptions.DeleteError,
)
await resp.release()
if dest_path.is_file:
assert len(blobs) == 1, 'Destination file should have exactly one candidate'
return GitHubFileTreeMetadata(blobs[0], commit=commit), not exists
folder = GitHubFolderTreeMetadata({
'path': dest_path.path.strip('/')
}, commit=commit)
folder.children = []
for item in blobs:
if item['path'] == src_path.path.rstrip('/'):
continue
if item['type'] == 'tree':
folder.children.append(GitHubFolderTreeMetadata(item))
else:
folder.children.append(GitHubFileTreeMetadata(item))
return folder, not exists
async def _do_intra_move_or_copy(self, src_path, dest_path, is_copy):
# ON PATHS:
# WB and GH use slightly different default conventions for their paths, so we often
# have to munge our WB paths before comparison. Here is a quick overview:
# WB (dirs): wb_dir.path == 'foo/bar/' str(wb_dir) == '/foo/bar/'
# WB (file): wb_file.path = 'foo/bar.txt' str(wb_file) == '/foo/bar.txt'
# GH (dir): 'foo/bar'
# GH (file): 'foo/bar.txt'
src_tree, src_head = await self._get_tree_and_head(src_path.branch_ref)
# these are the blobs to copy/move
blobs = [
item for item in src_tree['tree']
if src_path.is_dir and item['path'].startswith(src_path.path)
or src_path.is_file and item['path'] == src_path.path
]
if len(blobs) == 0:
raise exceptions.NotFoundError(str(src_path))
if src_path.is_file:
assert len(blobs) == 1, 'Found multiple targets'
commit_msg = settings.COPY_MESSAGE if is_copy else settings.MOVE_MESSAGE
commit = None
if src_path.branch_ref == dest_path.branch_ref:
exists = self._path_exists_in_tree(src_tree['tree'], dest_path)
# if we're overwriting an existing dir, we must remove its blobs from the tree
if dest_path.is_dir:
src_tree['tree'] = self._remove_path_from_tree(
src_tree['tree'], dest_path)
# if this is a copy, duplicate and append our source blobs. The originals will be updated
# with the new destination path.
if is_copy:
src_tree['tree'].extend(copy.deepcopy(blobs))
# see, I told you they'd be overwritten
self._reparent_blobs(blobs, src_path, dest_path)
src_tree['tree'] = self._prune_subtrees(src_tree['tree'])
commit = await self._commit_tree_and_advance_branch(
src_tree['tree'], {'sha': src_head}, commit_msg,
src_path.branch_ref)
else:
dest_tree, dest_head = await self._get_tree_and_head(
dest_path.branch_ref)
exists = self._path_exists_in_tree(dest_tree['tree'], dest_path)
dest_tree['tree'] = self._remove_path_from_tree(
dest_tree['tree'], dest_path)
new_blobs = copy.deepcopy(blobs)
self._reparent_blobs(new_blobs, src_path, dest_path)
dest_tree['tree'].extend(new_blobs)
dest_tree['tree'] = self._prune_subtrees(dest_tree['tree'])
commit = await self._commit_tree_and_advance_branch(
dest_tree['tree'], {'sha': dest_head}, commit_msg,
dest_path.branch_ref)
if not is_copy:
src_tree['tree'] = self._remove_path_from_tree(
src_tree['tree'], src_path)
src_tree['tree'] = self._prune_subtrees(src_tree['tree'])
await self._commit_tree_and_advance_branch(
src_tree['tree'], {'sha': src_head}, commit_msg,
src_path.branch_ref)
blobs = new_blobs # for the metadata
if dest_path.is_file:
assert len(
blobs
) == 1, 'Destination file should have exactly one candidate'
return GitHubFileTreeMetadata(blobs[0],
commit=commit,
ref=dest_path.branch_ref), not exists
folder = GitHubFolderTreeMetadata({'path': dest_path.path.strip('/')},
commit=commit,
ref=dest_path.branch_ref)
folder.children = []
for item in blobs:
if item['path'] == dest_path.path.rstrip('/'):
continue
if item['type'] == 'tree':
folder.children.append(
GitHubFolderTreeMetadata(item, ref=dest_path.branch_ref))
else:
folder.children.append(
GitHubFileTreeMetadata(item, ref=dest_path.branch_ref))
return folder, not exists
def _do_intra_move_or_copy(self, src_path, dest_path, is_copy):
target, branch = None, src_path.identifier[0]
branch_data = yield from self._fetch_branch(branch)
old_commit_sha = branch_data['commit']['sha']
old_commit_tree_sha = branch_data['commit']['commit']['tree']['sha']
tree = yield from self._fetch_tree(old_commit_tree_sha, recursive=True)
exists = any(x['path'] == dest_path.path for x in tree['tree'])
target, keep = None, []
for item in tree['tree']:
if item['path'] == str(src_path).strip('/'):
assert target is None, 'Found multiple targets'
target = item
elif item['path'].startswith(src_path.path):
keep.append(item)
if target is None or (src_path.is_dir and target['type'] != 'tree'):
raise exceptions.NotFoundError(str(src_path))
if is_copy:
tree['tree'].append(copy.deepcopy(target))
elif src_path.is_dir:
for item in keep:
tree['tree'].remove(item)
target['path'] = target['path'].replace(src_path.path.strip('/'), dest_path.path.strip('/'), 1)
new_tree_data = yield from self._create_tree({'tree': tree['tree']})
new_tree_sha = new_tree_data['sha']
# Create a new commit which references our top most tree change.
commit_resp = yield from self.make_request(
'POST',
self.build_repo_url('git', 'commits'),
headers={'Content-Type': 'application/json'},
data=json.dumps({
'tree': new_tree_sha,
'parents': [old_commit_sha],
'committer': self.committer,
'message': settings.COPY_MESSAGE if is_copy else settings.MOVE_MESSAGE
}),
expects=(201, ),
throws=exceptions.DeleteError,
)
commit = yield from commit_resp.json()
# Update repository reference, point to the newly created commit.
# No need to store data, rely on expects to raise exceptions
yield from self.make_request(
'PATCH',
self.build_repo_url('git', 'refs', 'heads', branch),
headers={'Content-Type': 'application/json'},
data=json.dumps({'sha': commit['sha']}),
expects=(200, ),
throws=exceptions.DeleteError,
)
if dest_path.is_file:
return GitHubFileTreeMetadata(target, commit=commit), not exists
folder = GitHubFolderTreeMetadata({
'path': dest_path.path.strip('/')
}, commit=commit)
folder.children = []
for item in keep:
item['path'] = item['path'].replace(src_path.path, dest_path.path, 1)
if item['type'] == 'tree':
folder.children.append(GitHubFolderTreeMetadata(item))
else:
folder.children.append(GitHubFileTreeMetadata(item))
return folder, exists
async def _do_intra_move_or_copy(self, src_path, dest_path, is_copy):
# ON PATHS:
# WB and GH use slightly different default conventions for their paths, so we often
# have to munge our WB paths before comparison. Here is a quick overview:
# WB (dirs): wb_dir.path == 'foo/bar/' str(wb_dir) == '/foo/bar/'
# WB (file): wb_file.path = 'foo/bar.txt' str(wb_file) == '/foo/bar.txt'
# GH (dir): 'foo/bar'
# GH (file): 'foo/bar.txt'
src_tree, src_head = await self._get_tree_and_head(src_path.branch_ref)
# these are the blobs to copy/move
blobs = [
item
for item in src_tree['tree']
if src_path.is_dir and item['path'].startswith(src_path.path) or
src_path.is_file and item['path'] == src_path.path
]
if len(blobs) == 0:
raise exceptions.NotFoundError(str(src_path))
if src_path.is_file:
assert len(blobs) == 1, 'Found multiple targets'
commit_msg = pd_settings.COPY_MESSAGE if is_copy else pd_settings.MOVE_MESSAGE
commit = None
if src_path.branch_ref == dest_path.branch_ref:
exists = self._path_exists_in_tree(src_tree['tree'], dest_path)
# if we're overwriting an existing dir, we must remove its blobs from the tree
if dest_path.is_dir:
src_tree['tree'] = self._remove_path_from_tree(src_tree['tree'], dest_path)
# if this is a copy, duplicate and append our source blobs. The originals will be updated
# with the new destination path.
if is_copy:
src_tree['tree'].extend(copy.deepcopy(blobs))
# see, I told you they'd be overwritten
self._reparent_blobs(blobs, src_path, dest_path)
src_tree['tree'] = self._prune_subtrees(src_tree['tree'])
commit = await self._commit_tree_and_advance_branch(src_tree, {'sha': src_head},
commit_msg, src_path.branch_ref)
else:
dest_tree, dest_head = await self._get_tree_and_head(dest_path.branch_ref)
exists = self._path_exists_in_tree(dest_tree['tree'], dest_path)
dest_tree['tree'] = self._remove_path_from_tree(dest_tree['tree'], dest_path)
new_blobs = copy.deepcopy(blobs)
self._reparent_blobs(new_blobs, src_path, dest_path)
dest_tree['tree'].extend(new_blobs)
dest_tree['tree'] = self._prune_subtrees(dest_tree['tree'])
commit = await self._commit_tree_and_advance_branch(dest_tree, {'sha': dest_head},
commit_msg, dest_path.branch_ref)
if not is_copy:
src_tree['tree'] = self._remove_path_from_tree(src_tree['tree'], src_path)
src_tree['tree'] = self._prune_subtrees(src_tree['tree'])
await self._commit_tree_and_advance_branch(src_tree, {'sha': src_head},
commit_msg, src_path.branch_ref)
blobs = new_blobs # for the metadata
if dest_path.is_file:
assert len(blobs) == 1, 'Destination file should have exactly one candidate'
return GitHubFileTreeMetadata(
blobs[0], commit=commit, ref=dest_path.branch_ref
), not exists
folder = GitHubFolderTreeMetadata({
'path': dest_path.path.strip('/')
}, commit=commit, ref=dest_path.branch_ref)
folder.children = []
for item in blobs:
if item['path'] == dest_path.path.rstrip('/'):
continue
if item['type'] == 'tree':
folder.children.append(GitHubFolderTreeMetadata(item, ref=dest_path.branch_ref))
else:
folder.children.append(GitHubFileTreeMetadata(item, ref=dest_path.branch_ref))
return folder, not exists
def _do_intra_move_or_copy(self, src_path, dest_path, is_copy):
target, branch = None, src_path.identifier[0]
branch_data = yield from self._fetch_branch(branch)
old_commit_sha = branch_data["commit"]["sha"]
old_commit_tree_sha = branch_data["commit"]["commit"]["tree"]["sha"]
tree = yield from self._fetch_tree(old_commit_tree_sha, recursive=True)
exists = any(x["path"] == dest_path.path for x in tree["tree"])
target, keep = None, []
for item in tree["tree"]:
if item["path"] == str(src_path).strip("/"):
assert target is None, "Found multiple targets"
target = item
elif item["path"].startswith(src_path.path):
keep.append(item)
if target is None or (src_path.is_dir and target["type"] != "tree"):
raise exceptions.NotFoundError(str(src_path))
if is_copy:
tree["tree"].append(copy.deepcopy(target))
elif src_path.is_dir:
for item in keep:
tree["tree"].remove(item)
target["path"] = target["path"].replace(src_path.path.strip("/"), dest_path.path.strip("/"), 1)
new_tree_data = yield from self._create_tree({"tree": tree["tree"]})
new_tree_sha = new_tree_data["sha"]
# Create a new commit which references our top most tree change.
commit_resp = yield from self.make_request(
"POST",
self.build_repo_url("git", "commits"),
headers={"Content-Type": "application/json"},
data=json.dumps(
{
"tree": new_tree_sha,
"parents": [old_commit_sha],
"committer": self.committer,
"message": settings.COPY_MESSAGE if is_copy else settings.MOVE_MESSAGE,
}
),
expects=(201,),
throws=exceptions.DeleteError,
)
commit = yield from commit_resp.json()
# Update repository reference, point to the newly created commit.
# No need to store data, rely on expects to raise exceptions
yield from self.make_request(
"PATCH",
self.build_repo_url("git", "refs", "heads", branch),
headers={"Content-Type": "application/json"},
data=json.dumps({"sha": commit["sha"]}),
expects=(200,),
throws=exceptions.DeleteError,
)
if dest_path.is_file:
return GitHubFileTreeMetadata(target, commit=commit), not exists
folder = GitHubFolderTreeMetadata({"path": dest_path.path.strip("/")}, commit=commit)
folder.children = []
for item in keep:
item["path"] = item["path"].replace(src_path.path, dest_path.path, 1)
if item["type"] == "tree":
folder.children.append(GitHubFolderTreeMetadata(item))
else:
folder.children.append(GitHubFileTreeMetadata(item))
return folder, exists
async def _do_intra_move_or_copy(self, src_path, dest_path, is_copy):
# ON PATHS:
# WB and GH use slightly different default conventions for their paths, so we often
# have to munge our WB paths before comparison. Here is a quick overview:
# WB (dirs): wb_dir.path == 'foo/bar/' str(wb_dir) == '/foo/bar/'
# WB (file): wb_file.path = 'foo/bar.txt' str(wb_file) == '/foo/bar.txt'
# GH (dir): 'foo/bar'
# GH (file): 'foo/bar.txt'
branch = src_path.identifier[0]
branch_data = await self._fetch_branch(branch)
old_commit_sha = branch_data['commit']['sha']
old_commit_tree_sha = branch_data['commit']['commit']['tree']['sha']
tree = await self._fetch_tree(old_commit_tree_sha, recursive=True)
exists = any(x['path'] == dest_path.path.rstrip('/')
for x in tree['tree'])
# these are the blobs to copy/move
blobs = [
item for item in tree['tree']
if src_path.is_dir and item['path'].startswith(src_path.path)
or src_path.is_file and item['path'] == src_path.path
]
# if we're overwriting an existing dir, we must remove its blobs from the tree
if dest_path.is_dir:
tree['tree'] = [
item for item in tree['tree']
if not item['path'].startswith(dest_path.path)
]
if len(blobs) == 0:
raise exceptions.NotFoundError(str(src_path))
if src_path.is_file:
assert len(blobs) == 1, 'Found multiple targets'
# if this is a copy, duplicate and append our source blobs. The originals will be updated
# with the new destination path.
if is_copy:
tree['tree'].extend([copy.deepcopy(blob) for blob in blobs])
# see, I told you they'd be overwritten
for blob in blobs:
blob['path'] = blob['path'].replace(src_path.path, dest_path.path,
1)
# github infers tree contents from blob paths
# see: http://www.levibotelho.com/development/commit-a-file-with-the-github-api/
tree['tree'] = [
item for item in tree['tree'] if item['type'] != 'tree'
]
new_tree_data = await self._create_tree({'tree': tree['tree']})
new_tree_sha = new_tree_data['sha']
# Create a new commit which references our top most tree change.
commit_resp = await self.make_request(
'POST',
self.build_repo_url('git', 'commits'),
headers={'Content-Type': 'application/json'},
data=json.dumps({
'tree':
new_tree_sha,
'parents': [old_commit_sha],
'committer':
self.committer,
'message':
settings.COPY_MESSAGE if is_copy else settings.MOVE_MESSAGE
}),
expects=(201, ),
throws=exceptions.DeleteError,
)
commit = await commit_resp.json()
# Update repository reference, point to the newly created commit.
# No need to store data, rely on expects to raise exceptions
resp = await self.make_request(
'PATCH',
self.build_repo_url('git', 'refs', 'heads', branch),
headers={'Content-Type': 'application/json'},
data=json.dumps({'sha': commit['sha']}),
expects=(200, ),
throws=exceptions.DeleteError,
)
await resp.release()
if dest_path.is_file:
assert len(
blobs
) == 1, 'Destination file should have exactly one candidate'
return GitHubFileTreeMetadata(blobs[0], commit=commit), not exists
folder = GitHubFolderTreeMetadata({'path': dest_path.path.strip('/')},
commit=commit)
folder.children = []
for item in blobs:
if item['path'] == src_path.path.rstrip('/'):
continue
if item['type'] == 'tree':
folder.children.append(GitHubFolderTreeMetadata(item))
else:
folder.children.append(GitHubFileTreeMetadata(item))
return folder, not exists
def _do_intra_move_or_copy(self, src_path, dest_path, is_copy):
target, branch = None, src_path.identifier[0]
branch_data = yield from self._fetch_branch(branch)
old_commit_sha = branch_data['commit']['sha']
old_commit_tree_sha = branch_data['commit']['commit']['tree']['sha']
tree = yield from self._fetch_tree(old_commit_tree_sha, recursive=True)
exists = any(x['path'] == dest_path.path for x in tree['tree'])
target, keep = None, []
for item in tree['tree']:
if item['path'] == str(src_path).strip('/'):
assert target is None, 'Found multiple targets'
target = item
elif item['path'].startswith(src_path.path):
keep.append(item)
if target is None or (src_path.is_dir and target['type'] != 'tree'):
raise exceptions.NotFoundError(str(src_path))
if is_copy:
tree['tree'].append(copy.deepcopy(target))
elif src_path.is_dir:
for item in keep:
tree['tree'].remove(item)
target['path'] = target['path'].replace(src_path.path.strip('/'),
dest_path.path.strip('/'), 1)
new_tree_data = yield from self._create_tree({'tree': tree['tree']})
new_tree_sha = new_tree_data['sha']
# Create a new commit which references our top most tree change.
commit_resp = yield from self.make_request(
'POST',
self.build_repo_url('git', 'commits'),
headers={'Content-Type': 'application/json'},
data=json.dumps({
'tree':
new_tree_sha,
'parents': [old_commit_sha],
'committer':
self.committer,
'message':
settings.COPY_MESSAGE if is_copy else settings.MOVE_MESSAGE
}),
expects=(201, ),
throws=exceptions.DeleteError,
)
commit = yield from commit_resp.json()
# Update repository reference, point to the newly created commit.
# No need to store data, rely on expects to raise exceptions
yield from self.make_request(
'PATCH',
self.build_repo_url('git', 'refs', 'heads', branch),
headers={'Content-Type': 'application/json'},
data=json.dumps({'sha': commit['sha']}),
expects=(200, ),
throws=exceptions.DeleteError,
)
if dest_path.is_file:
return GitHubFileTreeMetadata(target, commit=commit), not exists
folder = GitHubFolderTreeMetadata({'path': dest_path.path.strip('/')},
commit=commit)
folder.children = []
for item in keep:
item['path'] = item['path'].replace(src_path.path, dest_path.path,
1)
if item['type'] == 'tree':
folder.children.append(GitHubFolderTreeMetadata(item))
else:
folder.children.append(GitHubFileTreeMetadata(item))
return folder, exists