def test_build_folder_metadata_from_tree(self, folder_metadata_tree_endpoint): try: metadata = GitHubFolderTreeMetadata(folder_metadata_tree_endpoint) except Exception as exc: pytest.fail(str(exc)) assert metadata.name == 'lorch' assert metadata.path == '/foldera/folderb/lorch/' assert metadata.extra == {} assert metadata.provider == 'github' assert metadata.commit is None assert metadata.ref is None
async def _do_intra_move_or_copy(self, src_path, dest_path, is_copy): # ON PATHS: # WB and GH use slightly different default conventions for their paths, so we often # have to munge our WB paths before comparison. Here is a quick overview: # WB (dirs): wb_dir.path == 'foo/bar/' str(wb_dir) == '/foo/bar/' # WB (file): wb_file.path = 'foo/bar.txt' str(wb_file) == '/foo/bar.txt' # GH (dir): 'foo/bar' # GH (file): 'foo/bar.txt' branch = src_path.identifier[0] branch_data = await self._fetch_branch(branch) old_commit_sha = branch_data['commit']['sha'] old_commit_tree_sha = branch_data['commit']['commit']['tree']['sha'] tree = await self._fetch_tree(old_commit_tree_sha, recursive=True) exists = any(x['path'] == dest_path.path.rstrip('/') for x in tree['tree']) # these are the blobs to copy/move blobs = [ item for item in tree['tree'] if src_path.is_dir and item['path'].startswith(src_path.path) or src_path.is_file and item['path'] == src_path.path ] # if we're overwriting an existing dir, we must remove its blobs from the tree if dest_path.is_dir: tree['tree'] = [ item for item in tree['tree'] if not item['path'].startswith(dest_path.path) ] if len(blobs) == 0: raise exceptions.NotFoundError(str(src_path)) if src_path.is_file: assert len(blobs) == 1, 'Found multiple targets' # if this is a copy, duplicate and append our source blobs. The originals will be updated # with the new destination path. if is_copy: tree['tree'].extend([copy.deepcopy(blob) for blob in blobs]) # see, I told you they'd be overwritten for blob in blobs: blob['path'] = blob['path'].replace(src_path.path, dest_path.path, 1) # github infers tree contents from blob paths # see: http://www.levibotelho.com/development/commit-a-file-with-the-github-api/ tree['tree'] = [item for item in tree['tree'] if item['type'] != 'tree'] new_tree_data = await self._create_tree({'tree': tree['tree']}) new_tree_sha = new_tree_data['sha'] # Create a new commit which references our top most tree change. commit_resp = await self.make_request( 'POST', self.build_repo_url('git', 'commits'), headers={'Content-Type': 'application/json'}, data=json.dumps({ 'tree': new_tree_sha, 'parents': [old_commit_sha], 'committer': self.committer, 'message': settings.COPY_MESSAGE if is_copy else settings.MOVE_MESSAGE }), expects=(201, ), throws=exceptions.DeleteError, ) commit = await commit_resp.json() # Update repository reference, point to the newly created commit. # No need to store data, rely on expects to raise exceptions resp = await self.make_request( 'PATCH', self.build_repo_url('git', 'refs', 'heads', branch), headers={'Content-Type': 'application/json'}, data=json.dumps({'sha': commit['sha']}), expects=(200, ), throws=exceptions.DeleteError, ) await resp.release() if dest_path.is_file: assert len(blobs) == 1, 'Destination file should have exactly one candidate' return GitHubFileTreeMetadata(blobs[0], commit=commit), not exists folder = GitHubFolderTreeMetadata({ 'path': dest_path.path.strip('/') }, commit=commit) folder.children = [] for item in blobs: if item['path'] == src_path.path.rstrip('/'): continue if item['type'] == 'tree': folder.children.append(GitHubFolderTreeMetadata(item)) else: folder.children.append(GitHubFileTreeMetadata(item)) return folder, not exists
async def _do_intra_move_or_copy(self, src_path, dest_path, is_copy): # ON PATHS: # WB and GH use slightly different default conventions for their paths, so we often # have to munge our WB paths before comparison. Here is a quick overview: # WB (dirs): wb_dir.path == 'foo/bar/' str(wb_dir) == '/foo/bar/' # WB (file): wb_file.path = 'foo/bar.txt' str(wb_file) == '/foo/bar.txt' # GH (dir): 'foo/bar' # GH (file): 'foo/bar.txt' src_tree, src_head = await self._get_tree_and_head(src_path.branch_ref) # these are the blobs to copy/move blobs = [ item for item in src_tree['tree'] if src_path.is_dir and item['path'].startswith(src_path.path) or src_path.is_file and item['path'] == src_path.path ] if len(blobs) == 0: raise exceptions.NotFoundError(str(src_path)) if src_path.is_file: assert len(blobs) == 1, 'Found multiple targets' commit_msg = settings.COPY_MESSAGE if is_copy else settings.MOVE_MESSAGE commit = None if src_path.branch_ref == dest_path.branch_ref: exists = self._path_exists_in_tree(src_tree['tree'], dest_path) # if we're overwriting an existing dir, we must remove its blobs from the tree if dest_path.is_dir: src_tree['tree'] = self._remove_path_from_tree( src_tree['tree'], dest_path) # if this is a copy, duplicate and append our source blobs. The originals will be updated # with the new destination path. if is_copy: src_tree['tree'].extend(copy.deepcopy(blobs)) # see, I told you they'd be overwritten self._reparent_blobs(blobs, src_path, dest_path) src_tree['tree'] = self._prune_subtrees(src_tree['tree']) commit = await self._commit_tree_and_advance_branch( src_tree['tree'], {'sha': src_head}, commit_msg, src_path.branch_ref) else: dest_tree, dest_head = await self._get_tree_and_head( dest_path.branch_ref) exists = self._path_exists_in_tree(dest_tree['tree'], dest_path) dest_tree['tree'] = self._remove_path_from_tree( dest_tree['tree'], dest_path) new_blobs = copy.deepcopy(blobs) self._reparent_blobs(new_blobs, src_path, dest_path) dest_tree['tree'].extend(new_blobs) dest_tree['tree'] = self._prune_subtrees(dest_tree['tree']) commit = await self._commit_tree_and_advance_branch( dest_tree['tree'], {'sha': dest_head}, commit_msg, dest_path.branch_ref) if not is_copy: src_tree['tree'] = self._remove_path_from_tree( src_tree['tree'], src_path) src_tree['tree'] = self._prune_subtrees(src_tree['tree']) await self._commit_tree_and_advance_branch( src_tree['tree'], {'sha': src_head}, commit_msg, src_path.branch_ref) blobs = new_blobs # for the metadata if dest_path.is_file: assert len( blobs ) == 1, 'Destination file should have exactly one candidate' return GitHubFileTreeMetadata(blobs[0], commit=commit, ref=dest_path.branch_ref), not exists folder = GitHubFolderTreeMetadata({'path': dest_path.path.strip('/')}, commit=commit, ref=dest_path.branch_ref) folder.children = [] for item in blobs: if item['path'] == dest_path.path.rstrip('/'): continue if item['type'] == 'tree': folder.children.append( GitHubFolderTreeMetadata(item, ref=dest_path.branch_ref)) else: folder.children.append( GitHubFileTreeMetadata(item, ref=dest_path.branch_ref)) return folder, not exists
def _do_intra_move_or_copy(self, src_path, dest_path, is_copy): target, branch = None, src_path.identifier[0] branch_data = yield from self._fetch_branch(branch) old_commit_sha = branch_data['commit']['sha'] old_commit_tree_sha = branch_data['commit']['commit']['tree']['sha'] tree = yield from self._fetch_tree(old_commit_tree_sha, recursive=True) exists = any(x['path'] == dest_path.path for x in tree['tree']) target, keep = None, [] for item in tree['tree']: if item['path'] == str(src_path).strip('/'): assert target is None, 'Found multiple targets' target = item elif item['path'].startswith(src_path.path): keep.append(item) if target is None or (src_path.is_dir and target['type'] != 'tree'): raise exceptions.NotFoundError(str(src_path)) if is_copy: tree['tree'].append(copy.deepcopy(target)) elif src_path.is_dir: for item in keep: tree['tree'].remove(item) target['path'] = target['path'].replace(src_path.path.strip('/'), dest_path.path.strip('/'), 1) new_tree_data = yield from self._create_tree({'tree': tree['tree']}) new_tree_sha = new_tree_data['sha'] # Create a new commit which references our top most tree change. commit_resp = yield from self.make_request( 'POST', self.build_repo_url('git', 'commits'), headers={'Content-Type': 'application/json'}, data=json.dumps({ 'tree': new_tree_sha, 'parents': [old_commit_sha], 'committer': self.committer, 'message': settings.COPY_MESSAGE if is_copy else settings.MOVE_MESSAGE }), expects=(201, ), throws=exceptions.DeleteError, ) commit = yield from commit_resp.json() # Update repository reference, point to the newly created commit. # No need to store data, rely on expects to raise exceptions yield from self.make_request( 'PATCH', self.build_repo_url('git', 'refs', 'heads', branch), headers={'Content-Type': 'application/json'}, data=json.dumps({'sha': commit['sha']}), expects=(200, ), throws=exceptions.DeleteError, ) if dest_path.is_file: return GitHubFileTreeMetadata(target, commit=commit), not exists folder = GitHubFolderTreeMetadata({ 'path': dest_path.path.strip('/') }, commit=commit) folder.children = [] for item in keep: item['path'] = item['path'].replace(src_path.path, dest_path.path, 1) if item['type'] == 'tree': folder.children.append(GitHubFolderTreeMetadata(item)) else: folder.children.append(GitHubFileTreeMetadata(item)) return folder, exists
async def _do_intra_move_or_copy(self, src_path, dest_path, is_copy): # ON PATHS: # WB and GH use slightly different default conventions for their paths, so we often # have to munge our WB paths before comparison. Here is a quick overview: # WB (dirs): wb_dir.path == 'foo/bar/' str(wb_dir) == '/foo/bar/' # WB (file): wb_file.path = 'foo/bar.txt' str(wb_file) == '/foo/bar.txt' # GH (dir): 'foo/bar' # GH (file): 'foo/bar.txt' src_tree, src_head = await self._get_tree_and_head(src_path.branch_ref) # these are the blobs to copy/move blobs = [ item for item in src_tree['tree'] if src_path.is_dir and item['path'].startswith(src_path.path) or src_path.is_file and item['path'] == src_path.path ] if len(blobs) == 0: raise exceptions.NotFoundError(str(src_path)) if src_path.is_file: assert len(blobs) == 1, 'Found multiple targets' commit_msg = pd_settings.COPY_MESSAGE if is_copy else pd_settings.MOVE_MESSAGE commit = None if src_path.branch_ref == dest_path.branch_ref: exists = self._path_exists_in_tree(src_tree['tree'], dest_path) # if we're overwriting an existing dir, we must remove its blobs from the tree if dest_path.is_dir: src_tree['tree'] = self._remove_path_from_tree(src_tree['tree'], dest_path) # if this is a copy, duplicate and append our source blobs. The originals will be updated # with the new destination path. if is_copy: src_tree['tree'].extend(copy.deepcopy(blobs)) # see, I told you they'd be overwritten self._reparent_blobs(blobs, src_path, dest_path) src_tree['tree'] = self._prune_subtrees(src_tree['tree']) commit = await self._commit_tree_and_advance_branch(src_tree, {'sha': src_head}, commit_msg, src_path.branch_ref) else: dest_tree, dest_head = await self._get_tree_and_head(dest_path.branch_ref) exists = self._path_exists_in_tree(dest_tree['tree'], dest_path) dest_tree['tree'] = self._remove_path_from_tree(dest_tree['tree'], dest_path) new_blobs = copy.deepcopy(blobs) self._reparent_blobs(new_blobs, src_path, dest_path) dest_tree['tree'].extend(new_blobs) dest_tree['tree'] = self._prune_subtrees(dest_tree['tree']) commit = await self._commit_tree_and_advance_branch(dest_tree, {'sha': dest_head}, commit_msg, dest_path.branch_ref) if not is_copy: src_tree['tree'] = self._remove_path_from_tree(src_tree['tree'], src_path) src_tree['tree'] = self._prune_subtrees(src_tree['tree']) await self._commit_tree_and_advance_branch(src_tree, {'sha': src_head}, commit_msg, src_path.branch_ref) blobs = new_blobs # for the metadata if dest_path.is_file: assert len(blobs) == 1, 'Destination file should have exactly one candidate' return GitHubFileTreeMetadata( blobs[0], commit=commit, ref=dest_path.branch_ref ), not exists folder = GitHubFolderTreeMetadata({ 'path': dest_path.path.strip('/') }, commit=commit, ref=dest_path.branch_ref) folder.children = [] for item in blobs: if item['path'] == dest_path.path.rstrip('/'): continue if item['type'] == 'tree': folder.children.append(GitHubFolderTreeMetadata(item, ref=dest_path.branch_ref)) else: folder.children.append(GitHubFileTreeMetadata(item, ref=dest_path.branch_ref)) return folder, not exists
def _do_intra_move_or_copy(self, src_path, dest_path, is_copy): target, branch = None, src_path.identifier[0] branch_data = yield from self._fetch_branch(branch) old_commit_sha = branch_data["commit"]["sha"] old_commit_tree_sha = branch_data["commit"]["commit"]["tree"]["sha"] tree = yield from self._fetch_tree(old_commit_tree_sha, recursive=True) exists = any(x["path"] == dest_path.path for x in tree["tree"]) target, keep = None, [] for item in tree["tree"]: if item["path"] == str(src_path).strip("/"): assert target is None, "Found multiple targets" target = item elif item["path"].startswith(src_path.path): keep.append(item) if target is None or (src_path.is_dir and target["type"] != "tree"): raise exceptions.NotFoundError(str(src_path)) if is_copy: tree["tree"].append(copy.deepcopy(target)) elif src_path.is_dir: for item in keep: tree["tree"].remove(item) target["path"] = target["path"].replace(src_path.path.strip("/"), dest_path.path.strip("/"), 1) new_tree_data = yield from self._create_tree({"tree": tree["tree"]}) new_tree_sha = new_tree_data["sha"] # Create a new commit which references our top most tree change. commit_resp = yield from self.make_request( "POST", self.build_repo_url("git", "commits"), headers={"Content-Type": "application/json"}, data=json.dumps( { "tree": new_tree_sha, "parents": [old_commit_sha], "committer": self.committer, "message": settings.COPY_MESSAGE if is_copy else settings.MOVE_MESSAGE, } ), expects=(201,), throws=exceptions.DeleteError, ) commit = yield from commit_resp.json() # Update repository reference, point to the newly created commit. # No need to store data, rely on expects to raise exceptions yield from self.make_request( "PATCH", self.build_repo_url("git", "refs", "heads", branch), headers={"Content-Type": "application/json"}, data=json.dumps({"sha": commit["sha"]}), expects=(200,), throws=exceptions.DeleteError, ) if dest_path.is_file: return GitHubFileTreeMetadata(target, commit=commit), not exists folder = GitHubFolderTreeMetadata({"path": dest_path.path.strip("/")}, commit=commit) folder.children = [] for item in keep: item["path"] = item["path"].replace(src_path.path, dest_path.path, 1) if item["type"] == "tree": folder.children.append(GitHubFolderTreeMetadata(item)) else: folder.children.append(GitHubFileTreeMetadata(item)) return folder, exists
async def _do_intra_move_or_copy(self, src_path, dest_path, is_copy): # ON PATHS: # WB and GH use slightly different default conventions for their paths, so we often # have to munge our WB paths before comparison. Here is a quick overview: # WB (dirs): wb_dir.path == 'foo/bar/' str(wb_dir) == '/foo/bar/' # WB (file): wb_file.path = 'foo/bar.txt' str(wb_file) == '/foo/bar.txt' # GH (dir): 'foo/bar' # GH (file): 'foo/bar.txt' branch = src_path.identifier[0] branch_data = await self._fetch_branch(branch) old_commit_sha = branch_data['commit']['sha'] old_commit_tree_sha = branch_data['commit']['commit']['tree']['sha'] tree = await self._fetch_tree(old_commit_tree_sha, recursive=True) exists = any(x['path'] == dest_path.path.rstrip('/') for x in tree['tree']) # these are the blobs to copy/move blobs = [ item for item in tree['tree'] if src_path.is_dir and item['path'].startswith(src_path.path) or src_path.is_file and item['path'] == src_path.path ] # if we're overwriting an existing dir, we must remove its blobs from the tree if dest_path.is_dir: tree['tree'] = [ item for item in tree['tree'] if not item['path'].startswith(dest_path.path) ] if len(blobs) == 0: raise exceptions.NotFoundError(str(src_path)) if src_path.is_file: assert len(blobs) == 1, 'Found multiple targets' # if this is a copy, duplicate and append our source blobs. The originals will be updated # with the new destination path. if is_copy: tree['tree'].extend([copy.deepcopy(blob) for blob in blobs]) # see, I told you they'd be overwritten for blob in blobs: blob['path'] = blob['path'].replace(src_path.path, dest_path.path, 1) # github infers tree contents from blob paths # see: http://www.levibotelho.com/development/commit-a-file-with-the-github-api/ tree['tree'] = [ item for item in tree['tree'] if item['type'] != 'tree' ] new_tree_data = await self._create_tree({'tree': tree['tree']}) new_tree_sha = new_tree_data['sha'] # Create a new commit which references our top most tree change. commit_resp = await self.make_request( 'POST', self.build_repo_url('git', 'commits'), headers={'Content-Type': 'application/json'}, data=json.dumps({ 'tree': new_tree_sha, 'parents': [old_commit_sha], 'committer': self.committer, 'message': settings.COPY_MESSAGE if is_copy else settings.MOVE_MESSAGE }), expects=(201, ), throws=exceptions.DeleteError, ) commit = await commit_resp.json() # Update repository reference, point to the newly created commit. # No need to store data, rely on expects to raise exceptions resp = await self.make_request( 'PATCH', self.build_repo_url('git', 'refs', 'heads', branch), headers={'Content-Type': 'application/json'}, data=json.dumps({'sha': commit['sha']}), expects=(200, ), throws=exceptions.DeleteError, ) await resp.release() if dest_path.is_file: assert len( blobs ) == 1, 'Destination file should have exactly one candidate' return GitHubFileTreeMetadata(blobs[0], commit=commit), not exists folder = GitHubFolderTreeMetadata({'path': dest_path.path.strip('/')}, commit=commit) folder.children = [] for item in blobs: if item['path'] == src_path.path.rstrip('/'): continue if item['type'] == 'tree': folder.children.append(GitHubFolderTreeMetadata(item)) else: folder.children.append(GitHubFileTreeMetadata(item)) return folder, not exists
def _do_intra_move_or_copy(self, src_path, dest_path, is_copy): target, branch = None, src_path.identifier[0] branch_data = yield from self._fetch_branch(branch) old_commit_sha = branch_data['commit']['sha'] old_commit_tree_sha = branch_data['commit']['commit']['tree']['sha'] tree = yield from self._fetch_tree(old_commit_tree_sha, recursive=True) exists = any(x['path'] == dest_path.path for x in tree['tree']) target, keep = None, [] for item in tree['tree']: if item['path'] == str(src_path).strip('/'): assert target is None, 'Found multiple targets' target = item elif item['path'].startswith(src_path.path): keep.append(item) if target is None or (src_path.is_dir and target['type'] != 'tree'): raise exceptions.NotFoundError(str(src_path)) if is_copy: tree['tree'].append(copy.deepcopy(target)) elif src_path.is_dir: for item in keep: tree['tree'].remove(item) target['path'] = target['path'].replace(src_path.path.strip('/'), dest_path.path.strip('/'), 1) new_tree_data = yield from self._create_tree({'tree': tree['tree']}) new_tree_sha = new_tree_data['sha'] # Create a new commit which references our top most tree change. commit_resp = yield from self.make_request( 'POST', self.build_repo_url('git', 'commits'), headers={'Content-Type': 'application/json'}, data=json.dumps({ 'tree': new_tree_sha, 'parents': [old_commit_sha], 'committer': self.committer, 'message': settings.COPY_MESSAGE if is_copy else settings.MOVE_MESSAGE }), expects=(201, ), throws=exceptions.DeleteError, ) commit = yield from commit_resp.json() # Update repository reference, point to the newly created commit. # No need to store data, rely on expects to raise exceptions yield from self.make_request( 'PATCH', self.build_repo_url('git', 'refs', 'heads', branch), headers={'Content-Type': 'application/json'}, data=json.dumps({'sha': commit['sha']}), expects=(200, ), throws=exceptions.DeleteError, ) if dest_path.is_file: return GitHubFileTreeMetadata(target, commit=commit), not exists folder = GitHubFolderTreeMetadata({'path': dest_path.path.strip('/')}, commit=commit) folder.children = [] for item in keep: item['path'] = item['path'].replace(src_path.path, dest_path.path, 1) if item['type'] == 'tree': folder.children.append(GitHubFolderTreeMetadata(item)) else: folder.children.append(GitHubFileTreeMetadata(item)) return folder, exists