def _api_get_src(self, repository, path, revision, base_commit_id): # If a base commit ID is provided, use it. It may not be provided, # though, and in this case, we need to use the provided revision, # which will work for Mercurial but not for Git. # # If not provided, and using Git, we'll give the user a File Not # Found error with some info on what they need to do to correct # this. if base_commit_id: revision = base_commit_id elif repository.tool.name == 'Git': raise FileNotFoundError( path, revision, detail='The necessary revision information needed to find ' 'this file was not provided. Use RBTools 0.5.2 or ' 'newer.') # NOTE: As of this writing, the 2.0 API does not support fetching # the raw contents of files. We have to use the 1.0 API for # this instead. url = self._build_repository_api_url(repository, 'raw/%s/%s' % (quote(revision), quote(path)), version='1.0') try: return self._api_get(url, raw_content=True) except FileNotFoundError: raise FileNotFoundError(path, revision=revision, base_commit_id=base_commit_id)
def cat_file(self, extended_path, revision): import tempfile # Use tempfile to generate temporary filename temp = tempfile.NamedTemporaryFile() # Remove the file, so cleartool can write to it temp.close() cmdline = ["cleartool", "get", "-to", temp.name, extended_path] p = subprocess.Popen( cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=_popen_shell) (res, error) = p.communicate() failure = p.poll() if failure: raise FileNotFoundError(extended_path, revision) try: with open(temp.name, 'rb') as f: return f.read() except: raise FileNotFoundError(extended_path, revision)
def _api_get_src(self, repository, path, revision, base_commit_id): # If a base commit ID is provided, use it. It may not be provided, # though, and in this case, we need to use the provided revision, # which will work for Mercurial but not for Git. # # If not provided, and using Git, we'll give the user a File Not # Found error with some info on what they need to do to correct # this. if base_commit_id: revision = base_commit_id elif repository.tool.name == 'Git': raise FileNotFoundError( path, revision, detail='The necessary revision information needed to find ' 'this file was not provided. Use RBTools 0.5.2 or ' 'newer.') url = self._build_api_url( 'repositories/%s/%s/raw/%s/%s' % (quote(self._get_repository_owner(repository)), quote(self._get_repository_name(repository)), quote(revision), quote(path))) try: return self._api_get(url, raw_content=True) except FileNotFoundError: raise FileNotFoundError(path, revision=revision, base_commit_id=base_commit_id)
def _do_on_path(self, cb, path, revision=HEAD): if not path: raise FileNotFoundError(path, revision) try: normpath = self.__normalize_path(path) # SVN expects to have URLs escaped. Take care to only # escape the path part of the URL. if self.client.is_url(normpath): pathtuple = urlparse.urlsplit(normpath) path = pathtuple[2] if isinstance(path, unicode): path = path.encode('utf-8', 'ignore') normpath = urlparse.urlunsplit( (pathtuple[0], pathtuple[1], urllib.quote(path), '', '')) normrev = self.__normalize_revision(revision) return cb(normpath, normrev) except ClientError, e: stre = str(e) if 'File not found' in stre or 'path not found' in stre: raise FileNotFoundError(path, revision, detail=str(e)) elif 'callback_ssl_server_trust_prompt required' in stre: raise SCMError( 'HTTPS certificate not accepted. Please ensure that ' 'the proper certificate exists in %s ' 'for the user that reviewboard is running as.' % os.path.join(self.config_dir, 'auth')) elif 'callback_get_login required' in stre: raise AuthenticationError( msg='Login to the SCM server failed.') else: raise SCMError(e)
def get_file(self, extended_path, revision=HEAD, **kwargs): """Return content of file or list content of directory""" if not extended_path: raise FileNotFoundError(extended_path, revision) if revision == PRE_CREATION: return '' if self.viewtype == self.VIEW_SNAPSHOT: # Get the path to (presumably) file element (remove version) # The '@@' at the end of file_path is required. file_path = extended_path.rsplit('@@', 1)[0] + '@@' okind = self._get_object_kind(file_path) if okind == 'directory element': raise SCMError('Directory elements are unsupported.') elif okind == 'file element': output = self.client.cat_file(extended_path, revision) else: raise FileNotFoundError(extended_path, revision) else: if cpath.isdir(extended_path): output = self.client.list_dir(extended_path, revision) elif cpath.exists(extended_path): output = self.client.cat_file(extended_path, revision) else: raise FileNotFoundError(extended_path, revision) return output
def _do_on_path(self, cb, path, revision=HEAD): if not path: raise FileNotFoundError(path, revision) try: normpath = self.normalize_path(path) # SVN expects to have URLs escaped. Take care to only # escape the path part of the URL. if self.client.is_url(normpath): pathtuple = urlsplit(normpath) path = pathtuple[2] if isinstance(path, six.text_type): path = path.encode('utf-8', 'ignore') normpath = urlunsplit((pathtuple[0], pathtuple[1], quote(path), '', '')) normrev = self._normalize_revision(revision) return cb(normpath, normrev) except ClientError as e: exc = force_text(e) if 'File not found' in exc or 'path not found' in exc: raise FileNotFoundError(path, revision, detail=exc) elif 'callback_ssl_server_trust_prompt required' in exc: raise SCMError( _('HTTPS certificate not accepted. Please ensure that ' 'the proper certificate exists in %s ' 'for the user that reviewboard is running as.') % os.path.join(self.config_dir, 'auth')) else: raise SVNTool.normalize_error(e)
def get_file(self, extended_path, revision=HEAD, **kwargs): """Return content of file or list content of directory. Args: extended_path (unicode): The path of the element, including revision information. revision (reviewboard.scmtools.core.Revision, optional): Revision information. This will be either :py:data:`~reviewboard.scmtools.core.PRE_CREATION` (new file), or :py:data:`~reviewboard.scmtools.core.HEAD` (signifying to use the revision information included in ``extended_path``). **kwargs (dict, optional): Additional unused keyword arguments. Returns: bytes: The contents of the element. Raises: reviewboard.scmtools.errors.FileNotFoundError: The given ``extended_path`` did not match a valid element. reviewboard.scmtools.errors.SCMError: Another error occurred. """ if not extended_path: raise FileNotFoundError(extended_path, revision) if revision == PRE_CREATION: return '' if self.viewtype == self.VIEW_SNAPSHOT: # Get the path to (presumably) file element (remove version) # The '@@' at the end of file_path is required. file_path = extended_path.rsplit('@@', 1)[0] + '@@' okind = self._get_element_kind(file_path) if okind == 'directory element': raise SCMError('Directory elements are unsupported.') elif okind == 'file element': output = self.client.cat_file(extended_path) else: raise FileNotFoundError(extended_path) else: if cpath.isdir(extended_path): output = self.client.list_dir(extended_path) elif cpath.exists(extended_path): output = self.client.cat_file(extended_path) else: raise FileNotFoundError(extended_path) return output
def _api_get_file_meta(self, repository, path, revision, base_commit_id): """Return metadata on a file. Args: repository (reviewboard.scmtools.models.Repository): The repository containing the file. path (unicode): The path to the file. revision (unicode): The revision of the file. base_commit_id (unicode): The SHA1 of the commit to fetch the file at. If provided, this will take precedence over ``revision``. This is needed by Git. Returns: dict: The metadata on the file. Raises: reviewboard.scmtools.errors.FileNotFoundError: The file could not be found. """ # If a base commit ID is provided, use it. It may not be provided, # though, and in this case, we need to use the provided revision, # which will work for Mercurial but not for Git. # # If not provided, and using Git, we'll give the user a File Not # Found error with some info on what they need to do to correct # this. if base_commit_id: revision = base_commit_id elif repository.tool.name == 'Git': raise FileNotFoundError( path, revision, detail='The necessary revision information needed to find ' 'this file was not provided. Use RBTools 0.5.2 or ' 'newer.') url = ('%s?format=meta' % self._build_repository_api_url( repository, 'src/%s/%s' % (quote(revision), quote(path)))) try: return self.api_get(url) except FileNotFoundError: raise FileNotFoundError(path, revision=revision, base_commit_id=base_commit_id)
def api_get_blob(self, repo_api_url, path, sha): """Return the contents of a file using the GitHub API. Args: repo_api_url (unicode): The absolute URL for the base repository API. path (unicode): The path of the file within the repository. sha (unicode): The SHA1 of the file within the repository. Returns: bytes: The contents of the file. Raises: reviewboard.scmtools.errors.FileNotFoundError: The file could not be found or the API could not be accessed. """ try: return self.http_get(url='%s/git/blobs/%s' % (repo_api_url, sha), headers={ 'Accept': self.RAW_MIMETYPE, }).data except HostingServiceError: raise FileNotFoundError(path, sha)
def get_file(self, repository, path, revision, base_commit_id=None, *args, **kwargs): """Fetches a file from Unfuddle. This will perform an API request to fetch the contents of a file. If using Git, this will expect a base commit ID to be provided. """ try: commit_id = self._get_commit_id(repository, path, revision, base_commit_id) url = self._build_api_url( self._get_repository_account_domain(repository), 'repositories/%s/download/?path=%s&commit=%s' % (self._get_repository_id(repository), quote(path), quote(commit_id))) return self._api_get(url, raw_content=True) except (HTTPError, URLError): raise FileNotFoundError(path, revision)
def get_file(self, path, revision=HEAD): """Returns the contents of a given file at the given revision.""" if not path: raise FileNotFoundError(path, revision) revnum = self._normalize_revision(revision) path = B(self.normalize_path(path)) data = six.StringIO() try: self.client.cat(path, data, revnum) except SubversionException as e: raise FileNotFoundError(e) contents = data.getvalue() keywords = self.get_keywords(path, revision) if keywords: contents = self.collapse_keywords(contents, keywords) return contents
def get_file(self, repository, path, revision, base_commit_id=None, *args, **kwargs): """Fetches a file from Beanstalk. This will perform an API request to fetch the contents of a file. If using Git, this will expect a base commit ID to be provided. """ try: contents = self._api_get_node(repository, path, revision, base_commit_id, contents=True) except URLError: raise FileNotFoundError(path, revision) # On Subversion repositories, we may need to expand properties within # the file, like ``$Id$``. We only want to do this if we see keywords. if repository.tool.name == 'Subversion': contents = self._normalize_svn_file_content( repository, contents, path, revision) return contents
def api_get_blob(self, repo_api_url, path, sha): """Return the contents of a file using the GitHub API. Args: repo_api_url (unicode): The absolute URL for the base repository API. path (unicode): The path of the file within the repository. sha (unicode): The SHA1 of the file within the repository. Returns: bytes: The contents of the file. Raises: reviewboard.scmtools.errors.FileNotFoundError: The file could not be found or the API could not be accessed. """ credentials = self.get_http_credentials(self.account) url = '%s/git/blobs/%s' % (repo_api_url, sha) try: return self.http_get(url, headers={ 'Accept': self.RAW_MIMETYPE, }, **credentials)[0] except (URLError, HTTPError): raise FileNotFoundError(path, sha)
def get_file(self, repository, path, revision, *args, **kwargs): """Return the specified file from the repository. If the given file path is ``/invalid-path``, the file will be assumed to not exist and :py:exc:`reviewboard.scmtools.errors.FileNotFoundError` will be raised. Args: repository (reviewboard.scmtools.models.Repository): The repository the file belongs to. path (unicode): The file path. revision (unicode): The file revision. *args (tuple): Additional positional arguments. **kwargs (dict): Additional keyword arguments. Returns: unicode: The file data. Raises: reviewboard.scmtools.errors.FileNotFoundError: Raised if the file does not exist. """ if path == '/invalid-path': raise FileNotFoundError(path, revision) return super(TestService, self).get_file(repository, path, revision, *args, **kwargs)
def get_file(self, extended_path, revision=HEAD): """Return content of file or list content of directory""" if not extended_path: raise FileNotFoundError(extended_path, revision) if revision == PRE_CREATION: return '' if cpath.isdir(extended_path): output = self.client.list_dir(extended_path, revision) elif cpath.exists(extended_path): output = self.client.cat_file(extended_path, revision) else: raise FileNotFoundError(extended_path, revision) return output
def _normalize_revision(self, revision): """Normalize a revision to an integer or byte string. Args: revision (object): The revision to normalize. This can be an integer, byte string, Unicode string, :py:class:`~reviewboard.scmtools.core.Revision` object, or ``None``. Returns: object: The resulting revision. This may be an integer (if providing a revision number) or a Unicode string (if using an identifier like "HEAD"). Raises: reviewboard.scmtools.errors.FileNotFoundError: The revision indicates that the file does not yet exist. """ if revision is None: return None elif revision == HEAD: return 'HEAD' elif revision == PRE_CREATION: raise FileNotFoundError('', revision) elif isinstance(revision, Revision): revision = int(revision.name) elif isinstance(revision, (six.text_type, six.binary_type)): revision = int(revision) return revision
def _check_api_error(self, e): data = e.read() try: rsp = json.loads(data) except: rsp = None message = data if rsp and 'error' in rsp: error = rsp['error'] if 'message' in error: message = error['message'] if message: message = six.text_type(message) if e.code == 401: self._raise_auth_error(message) elif e.code == 404: if message.startswith('Repository'): raise HostingServiceError(message, http_code=e.code) # We don't have a path here, but it will be filled in inside # _api_get_src. raise FileNotFoundError('') else: raise HostingServiceAPIError( message or (ugettext('Unexpected HTTP %s error when talking to ' 'Bitbucket') % e.code), http_code=e.code, rsp=e)
def _cat_file(self, path, revision, option): """ Call git-cat-file(1) to get content or type information for a repository object. If called with just "commit", gets the content of a blob (or raises an exception if the commit is not a blob). Otherwise, "option" can be used to pass a switch to git-cat-file, e.g. to test or existence or get the type of "commit". """ commit = self._resolve_head(revision, path) p = self._run_git(['--git-dir=%s' % self.git_dir, 'cat-file', option, commit]) contents = p.stdout.read() errmsg = p.stderr.read() failure = p.wait() if failure: if errmsg.startswith("fatal: Not a valid object name"): raise FileNotFoundError(commit) else: raise SCMError(errmsg) return contents
def get_file_http(self, url, path, revision): logging.info('Fetching file from %s' % url) try: request = URLRequest(url) if self.username: auth_string = base64.b64encode('%s:%s' % (self.username, self.password)) request.add_header('Authorization', 'Basic %s' % auth_string) return urlopen(request).read() except HTTPError as e: if e.code == 404: logging.error('404') raise FileNotFoundError(path, revision) else: msg = "HTTP error code %d when fetching file from %s: %s" % \ (e.code, url, e) logging.error(msg) raise SCMError(msg) except Exception as e: msg = "Unexpected error fetching file from %s: %s" % (url, e) logging.error(msg) raise SCMError(msg)
def _get_file_http(client, url, path, revision, mime_type, *args, **kwargs): if url.startswith('http://hg.example.com/raw-rev/'): self.assertEqual( url, 'http://hg.example.com/raw-rev/' '1ca5879492b8fd606df1964ea3c1e2f4520f076f') self.assertEqual(path, '') self.assertEqual(revision, '') self.assertIsNone(mime_type) return b'diff payload' elif url.startswith('http://hg.example.com/json-rev/'): self.assertEqual( url, 'http://hg.example.com/json-rev/' '1ca5879492b8fd606df1964ea3c1e2f4520f076f') self.assertEqual(mime_type, 'application/json') self.assertEqual(path, '') self.assertEqual(revision, '') return self._dump_json({ 'node': '1ca5879492b8fd606df1964ea3c1e2f4520f076f', 'desc': 'This is the change description', 'user': '******', 'date': [1583149219, 28800], 'parents': ['b9af6489f6f2004ad11b82c6057f7007e3c35372'], }) else: raise FileNotFoundError(path=path, revision=revision)
def _check_api_error(self, e): data = e.read() try: rsp = json.loads(data) except: rsp = None message = data if rsp and 'error' in rsp: error = rsp['error'] if 'message' in error: message = error['message'] if message: message = six.text_type(message) if e.code == 401: raise AuthorizationError( message or ugettext('Invalid Bitbucket username or password')) elif e.code == 404: # We don't have a path here, but it will be filled in inside # _api_get_src. raise FileNotFoundError('') else: raise HostingServiceError( message or ugettext('Unknown error when talking to Bitbucket'))
def get_file(self, path, revspec): """Return the contents of a file. This expects a path within the repository and a Bazaar revision specifier. Args: path (unicode): The path to the file within the repository. revspec (unicode): The Bazaar revision specifier used to look up the file. Returns: bytes: The contents of the file. Raises: reviewboard.scmtools.errors.FileNotFoundError: The file could not be found. """ path = self._build_repo_path(path) p = self._run_bzr(['cat', '-r', revspec, path]) contents = p.stdout.read() errmsg = force_text(p.stderr.read()) failure = p.wait() self._check_error(errmsg) if failure: raise FileNotFoundError(path=path, revision=revspec, detail=errmsg) return contents
def get_file(self, path, revision=HEAD): if not path: raise FileNotFoundError(path, revision) try: normpath = self.__normalize_path(path) # SVN expects to have URLs escaped. Take care to only # escape the path part of the URL. if self.client.is_url(normpath): pathtuple = urlparse.urlsplit(normpath) path = pathtuple[2] if isinstance(path, unicode): path = path.encode('utf-8', 'ignore') normpath = urlparse.urlunsplit((pathtuple[0], pathtuple[1], urllib.quote(path), '','')) normrev = self.__normalize_revision(revision) data = self.client.cat(normpath, normrev) # Find out if this file has any keyword expansion set. # If it does, collapse these keywords. This is because SVN # will return the file expanded to us, which would break patching. keywords = self.client.propget("svn:keywords", normpath, normrev, recurse=True) if normpath in keywords: data = self.collapse_keywords(data, keywords[normpath]) return data except ClientError, e: stre = str(e) if 'File not found' in stre or 'path not found' in stre: raise FileNotFoundError(path, revision, str(e)) elif 'callback_ssl_server_trust_prompt required' in stre: raise SCMError( 'HTTPS certificate not accepted. Please ensure that ' 'the proper certificate exists in %s ' 'for the user that reviewboard is running as.' % os.path.join(self.config_dir, 'auth')) elif 'callback_get_login required' in stre: raise SCMError('Login to the SCM server failed.') else: raise SCMError(e)
def get_file(self, repository, path, revision, *args, **kwargs): """Return the contents of a file at the specified revision. Args: repository (reviewboard.scmtools.models.Repository): The repository configured to use Gerrit. path (unicode): The file path (ignored). revision (unicode): The file's Git object ID. *args (tuple): Ignored positional arguments. **kwargs (dict): Ignored keyword arguments. Returns: bytes: The contents of the file. Raises: reviewboard.hostingsvcs.errors.FileNotFoundError: The file does not exist in the remote repository. reviewboard.hostingsvcs.errors.HostingServiceAPIError: An error occurred communicating with the Gerrit API. """ url = self._build_project_api_url(repository, 'blobs', revision, 'content') try: rsp = self.client.http_get(url).data except HostingServiceAPIError as e: if e.http_code == 404: raise FileNotFoundError(path, revision=revision) raise HostingServiceAPIError( ugettext('Could not get file "%(file)s" at revision ' '"%(rev)s": %(error)s') % { 'file': path, 'rev': revision, 'error': e.read(), }, http_code=e.http_code) try: return base64.b64decode(rsp) except Exception as e: raise HostingServiceAPIError( ugettext('An error occurred while retrieving "%(file)s" at ' 'revision "%(rev)s" from Gerrit: the response could ' 'not be decoded: %(error)s') % { 'file': path, 'rev': revision, 'error': e, })
def __normalize_revision(self, revision): if revision == HEAD: r = Revision(opt_revision_kind.head) elif revision == PRE_CREATION: raise FileNotFoundError('', revision) else: r = Revision(opt_revision_kind.number, str(revision)) return r
def api_get_blob(self, repo_api_url, path, sha): url = self._build_api_url(repo_api_url, 'git/blobs/%s' % sha) try: return self.http_get(url, headers={ 'Accept': self.RAW_MIMETYPE, })[0] except (URLError, HTTPError): raise FileNotFoundError(path, sha)
def get_file(self, repository, path, revision, *args, **kwargs): url = self._build_api_url(repository, 'git/blobs/%s' % revision) try: return self._http_get(url, headers={ 'Accept': self.RAW_MIMETYPE, })[0] except (urllib2.URLError, urllib2.HTTPError): raise FileNotFoundError(path, revision)
def _normalize_revision(self, revision, negatives_allowed=True): if revision == HEAD: return B('HEAD') elif revision == PRE_CREATION: raise FileNotFoundError('', revision) elif isinstance(revision, Revision): revnum = int(revision.name) elif isinstance(revision, (B, ) + six.string_types): revnum = int(revision) return revnum
def get_file_http(self, url, path, revision): """Return the contents of a file from an HTTP(S) URL. This is a convenience for looking up the contents of files that are referenced in diffs through an HTTP(S) request. Authentication is performed using the username and password provided (if any). Args: url (unicode): The URL to fetch the file contents from. path (unicode): The path of the file, as referenced in the diff. revision (Revision): The revision of the file, as referenced in the diff. Returns: bytes: The contents of the file. Raises: reviewboard.scmtools.errors.FileNotFoundError: The file could not be found. reviewboard.scmtools.errors.SCMError: Unexpected error in fetching the file. This may be an unexpected HTTP status code. """ logging.info('Fetching file from %s' % url) try: request = URLRequest(url) if self.username: auth_string = base64.b64encode('%s:%s' % (self.username, self.password)) request.add_header('Authorization', 'Basic %s' % auth_string) return urlopen(request).read() except HTTPError as e: if e.code == 404: logging.error('404') raise FileNotFoundError(path, revision) else: msg = "HTTP error code %d when fetching file from %s: %s" % \ (e.code, url, e) logging.error(msg) raise SCMError(msg) except Exception as e: msg = "Unexpected error fetching file from %s: %s" % (url, e) logging.error(msg) raise SCMError(msg)
def get_file(self, repository, path, revision, base_commit_id=None, *args, **kwargs): """Fetches a file from GitLab. This will perform an API request to fetch the contents of a file. """ try: return self._api_get( self._get_blob_url(repository, path, revision, base_commit_id), raw_content=True) except (HTTPError, URLError): raise FileNotFoundError(path, revision)