def test_file_metadata_with_ref(self, file_metadata_tree_endpoint): try: metadata = GitHubFileTreeMetadata(file_metadata_tree_endpoint, ref="some-branch") except Exception as exc: pytest.fail(str(exc)) assert metadata.name == "README.md" assert metadata.path == "/README.md" assert metadata.modified is None assert metadata.content_type is None assert metadata.size == 38 assert metadata.etag == "/README.md::d863d70539aa9fcb6b44b057221706f2ab18e341" assert metadata.extra == { "fileSha": "d863d70539aa9fcb6b44b057221706f2ab18e341", "webView": None, "ref": "some-branch", } assert metadata.provider == "github" assert metadata.commit is None assert metadata.ref == "some-branch" assert metadata.web_view is None json_api = metadata.json_api_serialized("mst3k") for actions, link in json_api["links"].items(): assert re.search("[?&]ref=some-branch", link)
def test_file_metadata_with_ref(self, metadata_fixtures): try: metadata = GitHubFileTreeMetadata(metadata_fixtures['file_metadata_tree_endpoint'], ref="some-branch") except Exception as exc: pytest.fail(str(exc)) assert metadata.name == 'README.md' assert metadata.path == '/README.md' assert metadata.modified is None assert metadata.content_type is None assert metadata.size == 38 assert metadata.size_as_int == 38 assert type(metadata.size_as_int) == int assert metadata.etag == '/README.md::d863d70539aa9fcb6b44b057221706f2ab18e341' assert metadata.extra == { 'fileSha': 'd863d70539aa9fcb6b44b057221706f2ab18e341', 'webView': None, 'ref': 'some-branch', 'hashes': { 'git': 'd863d70539aa9fcb6b44b057221706f2ab18e341', }, } assert metadata.provider == 'github' assert metadata.commit is None assert metadata.ref == 'some-branch' assert metadata.web_view is None json_api = metadata.json_api_serialized('mst3k') for actions, link in json_api['links'].items(): assert re.search('[?&]ref=some-branch', link)
def test_file_metadata_with_ref(self, file_metadata_tree_endpoint): try: metadata = GitHubFileTreeMetadata(file_metadata_tree_endpoint, ref="some-branch") except Exception as exc: pytest.fail(str(exc)) assert metadata.name == 'README.md' assert metadata.path == '/README.md' assert metadata.modified is None assert metadata.content_type is None assert metadata.size == 38 assert metadata.etag == '/README.md::d863d70539aa9fcb6b44b057221706f2ab18e341' assert metadata.extra == { 'fileSha': 'd863d70539aa9fcb6b44b057221706f2ab18e341', 'webView': None, 'ref': 'some-branch', } assert metadata.provider == 'github' assert metadata.commit is None assert metadata.ref == 'some-branch' assert metadata.web_view is None json_api = metadata.json_api_serialized('mst3k') for actions, link in json_api['links'].items(): assert re.search('[?&]ref=some-branch', link)
async def _metadata_file(self, path, revision=None, **kwargs): resp = await self.make_request( 'GET', self.build_repo_url('commits', path=path.path, sha=revision or path.branch_ref), expects=(200, ), throws=exceptions.MetadataError, ) commits = await resp.json() if not commits: raise exceptions.NotFoundError(str(path)) latest = commits[0] tree = await self._fetch_tree(latest['commit']['tree']['sha'], recursive=True) try: data = next(x for x in tree['tree'] if x['path'] == path.path) except StopIteration: raise exceptions.NotFoundError(str(path)) if isinstance(data, list): raise exceptions.MetadataError( 'Could not retrieve file "{0}"'.format(str(path)), code=404, ) return GitHubFileTreeMetadata(data, commit=latest['commit'], web_view=self._web_view(path), ref=path.branch_ref)
async def upload(self, stream, path, message=None, branch=None, **kwargs): assert self.name is not None assert self.email is not None try: exists = await self.exists(path) except exceptions.ProviderError as e: if e.data.get('message') == 'Git Repository is empty.': exists = False resp = await self.make_request( 'PUT', self.build_repo_url('contents', '.gitkeep'), data=json.dumps({ 'content': '', 'path': '.gitkeep', 'committer': self.committer, 'branch': path.identifier[0], 'message': 'Initial commit' }), expects=(201, ), throws=exceptions.CreateFolderError) data = await resp.json() latest_sha = data['commit']['sha'] else: latest_sha = await self._get_latest_sha(ref=path.identifier[0]) blob = await self._create_blob(stream) tree = await self._create_tree({ 'base_tree': latest_sha, 'tree': [{ 'path': path.path, 'mode': '100644', 'type': 'blob', 'sha': blob['sha'] }] }) commit = await self._create_commit({ 'tree': tree['sha'], 'parents': [latest_sha], 'committer': self.committer, 'message': message or (settings.UPDATE_FILE_MESSAGE if exists else settings.UPLOAD_FILE_MESSAGE), }) # Doesn't return anything useful await self._update_ref(commit['sha'], ref=path.identifier[0]) # You're hacky return GitHubFileTreeMetadata( { 'path': path.path, 'sha': blob['sha'], 'size': stream.size, }, commit=commit), not exists
def test_build_file_metadata_from_tree(self, metadata_fixtures): try: metadata = GitHubFileTreeMetadata( metadata_fixtures['file_metadata_tree_endpoint']) except Exception as exc: pytest.fail(str(exc)) assert metadata.name == 'README.md' assert metadata.path == '/README.md' assert metadata.modified is None assert metadata.content_type is None assert metadata.size == 38 assert metadata.size_as_int == 38 assert type(metadata.size_as_int) == int assert metadata.etag == '/README.md::d863d70539aa9fcb6b44b057221706f2ab18e341' assert metadata.extra == { 'fileSha': 'd863d70539aa9fcb6b44b057221706f2ab18e341', 'webView': None, 'hashes': { 'git': 'd863d70539aa9fcb6b44b057221706f2ab18e341', }, } assert metadata.provider == 'github' assert metadata.commit is None assert metadata.ref is None assert metadata.web_view is None
def test_metadata_file(self, provider, repo_metadata, repo_tree_metadata_root): ref = hashlib.sha1().hexdigest() path = yield from provider.validate_path('/file.txt') tree_url = provider.build_repo_url('git', 'trees', ref, recursive=1) latest_sha_url = provider.build_repo_url('git', 'refs', 'heads', path.identifier[0]) aiohttpretty.register_json_uri('GET', tree_url, body=repo_tree_metadata_root) aiohttpretty.register_json_uri('GET', latest_sha_url, body={'object': {'sha': ref}}) result = yield from provider.metadata(path) assert result == GitHubFileTreeMetadata(repo_tree_metadata_root['tree'][0]).serialized()
def test_metadata_file(self, provider, repo_metadata, repo_tree_metadata_root): ref = hashlib.sha1().hexdigest() path = yield from provider.validate_path('/file.txt') tree_url = provider.build_repo_url('git', 'trees', ref, recursive=1) commit_url = provider.build_repo_url('commits', path=path.path.lstrip('/'), sha=path.identifier[0]) aiohttpretty.register_json_uri('GET', tree_url, body=repo_tree_metadata_root) aiohttpretty.register_json_uri('GET', commit_url, body=[{ 'commit': { 'tree': { 'sha': ref }, 'author': { 'date': 'this is totally date' } }, }]) result = yield from provider.metadata(path) item = repo_tree_metadata_root['tree'][0] web_view = provider._web_view(path=path) assert result == GitHubFileTreeMetadata(item, web_view=web_view, commit={ 'tree': { 'sha': ref }, 'author': { 'date': 'this is totally date' } })
def _metadata_file(self, path, ref=None, **kwargs): if not GitHubProvider.is_sha(path.identifier[0]): latest = yield from self._get_latest_sha(ref=path.identifier[0]) else: latest = path.identifier[0] tree = yield from self._fetch_tree(latest, recursive=True) try: data = next(x for x in tree['tree'] if x['path'] == path.path) except StopIteration: raise exceptions.MetadataError(';', code=404) if isinstance(data, list): raise exceptions.MetadataError( 'Could not retrieve file "{0}"'.format(str(path)), code=404, ) return GitHubFileTreeMetadata(data).serialized()
def upload(self, stream, path, message=None, branch=None, **kwargs): assert self.name is not None assert self.email is not None exists = yield from self.exists(path) latest_sha = yield from self._get_latest_sha(ref=path.identifier[0]) blob = yield from self._create_blob(stream) tree = yield from self._create_tree({ 'base_tree': latest_sha, 'tree': [{ 'path': path.path, 'mode': '100644', 'type': 'blob', 'sha': blob['sha'] }] }) commit = yield from self._create_commit({ 'tree': tree['sha'], 'parents': [latest_sha], 'committer': self.committer, 'message': message or settings.UPLOAD_FILE_MESSAGE, }) # Doesn't return anything useful yield from self._update_ref(commit['sha'], ref=path.identifier[0]) # You're hacky return GitHubFileTreeMetadata( { 'path': path.path, 'sha': blob['sha'], 'size': stream.size, }, commit=commit).serialized(), not exists
async def _do_intra_move_or_copy(self, src_path, dest_path, is_copy): # ON PATHS: # WB and GH use slightly different default conventions for their paths, so we often # have to munge our WB paths before comparison. Here is a quick overview: # WB (dirs): wb_dir.path == 'foo/bar/' str(wb_dir) == '/foo/bar/' # WB (file): wb_file.path = 'foo/bar.txt' str(wb_file) == '/foo/bar.txt' # GH (dir): 'foo/bar' # GH (file): 'foo/bar.txt' src_tree, src_head = await self._get_tree_and_head(src_path.branch_ref) # these are the blobs to copy/move blobs = [ item for item in src_tree['tree'] if src_path.is_dir and item['path'].startswith(src_path.path) or src_path.is_file and item['path'] == src_path.path ] if len(blobs) == 0: raise exceptions.NotFoundError(str(src_path)) if src_path.is_file: assert len(blobs) == 1, 'Found multiple targets' commit_msg = settings.COPY_MESSAGE if is_copy else settings.MOVE_MESSAGE commit = None if src_path.branch_ref == dest_path.branch_ref: exists = self._path_exists_in_tree(src_tree['tree'], dest_path) # if we're overwriting an existing dir, we must remove its blobs from the tree if dest_path.is_dir: src_tree['tree'] = self._remove_path_from_tree( src_tree['tree'], dest_path) # if this is a copy, duplicate and append our source blobs. The originals will be updated # with the new destination path. if is_copy: src_tree['tree'].extend(copy.deepcopy(blobs)) # see, I told you they'd be overwritten self._reparent_blobs(blobs, src_path, dest_path) src_tree['tree'] = self._prune_subtrees(src_tree['tree']) commit = await self._commit_tree_and_advance_branch( src_tree['tree'], {'sha': src_head}, commit_msg, src_path.branch_ref) else: dest_tree, dest_head = await self._get_tree_and_head( dest_path.branch_ref) exists = self._path_exists_in_tree(dest_tree['tree'], dest_path) dest_tree['tree'] = self._remove_path_from_tree( dest_tree['tree'], dest_path) new_blobs = copy.deepcopy(blobs) self._reparent_blobs(new_blobs, src_path, dest_path) dest_tree['tree'].extend(new_blobs) dest_tree['tree'] = self._prune_subtrees(dest_tree['tree']) commit = await self._commit_tree_and_advance_branch( dest_tree['tree'], {'sha': dest_head}, commit_msg, dest_path.branch_ref) if not is_copy: src_tree['tree'] = self._remove_path_from_tree( src_tree['tree'], src_path) src_tree['tree'] = self._prune_subtrees(src_tree['tree']) await self._commit_tree_and_advance_branch( src_tree['tree'], {'sha': src_head}, commit_msg, src_path.branch_ref) blobs = new_blobs # for the metadata if dest_path.is_file: assert len( blobs ) == 1, 'Destination file should have exactly one candidate' return GitHubFileTreeMetadata(blobs[0], commit=commit, ref=dest_path.branch_ref), not exists folder = GitHubFolderTreeMetadata({'path': dest_path.path.strip('/')}, commit=commit, ref=dest_path.branch_ref) folder.children = [] for item in blobs: if item['path'] == dest_path.path.rstrip('/'): continue if item['type'] == 'tree': folder.children.append( GitHubFolderTreeMetadata(item, ref=dest_path.branch_ref)) else: folder.children.append( GitHubFileTreeMetadata(item, ref=dest_path.branch_ref)) return folder, not exists
async def _do_intra_move_or_copy(self, src_path, dest_path, is_copy): # ON PATHS: # WB and GH use slightly different default conventions for their paths, so we often # have to munge our WB paths before comparison. Here is a quick overview: # WB (dirs): wb_dir.path == 'foo/bar/' str(wb_dir) == '/foo/bar/' # WB (file): wb_file.path = 'foo/bar.txt' str(wb_file) == '/foo/bar.txt' # GH (dir): 'foo/bar' # GH (file): 'foo/bar.txt' branch = src_path.identifier[0] branch_data = await self._fetch_branch(branch) old_commit_sha = branch_data['commit']['sha'] old_commit_tree_sha = branch_data['commit']['commit']['tree']['sha'] tree = await self._fetch_tree(old_commit_tree_sha, recursive=True) exists = any(x['path'] == dest_path.path.rstrip('/') for x in tree['tree']) # these are the blobs to copy/move blobs = [ item for item in tree['tree'] if src_path.is_dir and item['path'].startswith(src_path.path) or src_path.is_file and item['path'] == src_path.path ] # if we're overwriting an existing dir, we must remove its blobs from the tree if dest_path.is_dir: tree['tree'] = [ item for item in tree['tree'] if not item['path'].startswith(dest_path.path) ] if len(blobs) == 0: raise exceptions.NotFoundError(str(src_path)) if src_path.is_file: assert len(blobs) == 1, 'Found multiple targets' # if this is a copy, duplicate and append our source blobs. The originals will be updated # with the new destination path. if is_copy: tree['tree'].extend([copy.deepcopy(blob) for blob in blobs]) # see, I told you they'd be overwritten for blob in blobs: blob['path'] = blob['path'].replace(src_path.path, dest_path.path, 1) # github infers tree contents from blob paths # see: http://www.levibotelho.com/development/commit-a-file-with-the-github-api/ tree['tree'] = [ item for item in tree['tree'] if item['type'] != 'tree' ] new_tree_data = await self._create_tree({'tree': tree['tree']}) new_tree_sha = new_tree_data['sha'] # Create a new commit which references our top most tree change. commit_resp = await self.make_request( 'POST', self.build_repo_url('git', 'commits'), headers={'Content-Type': 'application/json'}, data=json.dumps({ 'tree': new_tree_sha, 'parents': [old_commit_sha], 'committer': self.committer, 'message': settings.COPY_MESSAGE if is_copy else settings.MOVE_MESSAGE }), expects=(201, ), throws=exceptions.DeleteError, ) commit = await commit_resp.json() # Update repository reference, point to the newly created commit. # No need to store data, rely on expects to raise exceptions resp = await self.make_request( 'PATCH', self.build_repo_url('git', 'refs', 'heads', branch), headers={'Content-Type': 'application/json'}, data=json.dumps({'sha': commit['sha']}), expects=(200, ), throws=exceptions.DeleteError, ) await resp.release() if dest_path.is_file: assert len( blobs ) == 1, 'Destination file should have exactly one candidate' return GitHubFileTreeMetadata(blobs[0], commit=commit), not exists folder = GitHubFolderTreeMetadata({'path': dest_path.path.strip('/')}, commit=commit) folder.children = [] for item in blobs: if item['path'] == src_path.path.rstrip('/'): continue if item['type'] == 'tree': folder.children.append(GitHubFolderTreeMetadata(item)) else: folder.children.append(GitHubFileTreeMetadata(item)) return folder, not exists
def _do_intra_move_or_copy(self, src_path, dest_path, is_copy): target, branch = None, src_path.identifier[0] branch_data = yield from self._fetch_branch(branch) old_commit_sha = branch_data['commit']['sha'] old_commit_tree_sha = branch_data['commit']['commit']['tree']['sha'] tree = yield from self._fetch_tree(old_commit_tree_sha, recursive=True) exists = any(x['path'] == dest_path.path for x in tree['tree']) target, keep = None, [] for item in tree['tree']: if item['path'] == str(src_path).strip('/'): assert target is None, 'Found multiple targets' target = item elif item['path'].startswith(src_path.path): keep.append(item) if target is None or (src_path.is_dir and target['type'] != 'tree'): raise exceptions.NotFoundError(str(src_path)) if is_copy: tree['tree'].append(copy.deepcopy(target)) elif src_path.is_dir: for item in keep: tree['tree'].remove(item) target['path'] = target['path'].replace(src_path.path.strip('/'), dest_path.path.strip('/'), 1) new_tree_data = yield from self._create_tree({'tree': tree['tree']}) new_tree_sha = new_tree_data['sha'] # Create a new commit which references our top most tree change. commit_resp = yield from self.make_request( 'POST', self.build_repo_url('git', 'commits'), headers={'Content-Type': 'application/json'}, data=json.dumps({ 'tree': new_tree_sha, 'parents': [old_commit_sha], 'committer': self.committer, 'message': settings.COPY_MESSAGE if is_copy else settings.MOVE_MESSAGE }), expects=(201, ), throws=exceptions.DeleteError, ) commit = yield from commit_resp.json() # Update repository reference, point to the newly created commit. # No need to store data, rely on expects to raise exceptions yield from self.make_request( 'PATCH', self.build_repo_url('git', 'refs', 'heads', branch), headers={'Content-Type': 'application/json'}, data=json.dumps({'sha': commit['sha']}), expects=(200, ), throws=exceptions.DeleteError, ) if dest_path.is_file: return GitHubFileTreeMetadata(target, commit=commit), not exists folder = GitHubFolderTreeMetadata({'path': dest_path.path.strip('/')}, commit=commit) folder.children = [] for item in keep: item['path'] = item['path'].replace(src_path.path, dest_path.path, 1) if item['type'] == 'tree': folder.children.append(GitHubFolderTreeMetadata(item)) else: folder.children.append(GitHubFileTreeMetadata(item)) return folder, exists