def _api_get_src(self, repository, path, revision, base_commit_id):
# If a base commit ID is provided, use it. It may not be provided,
# though, and in this case, we need to use the provided revision,
# which will work for Mercurial but not for Git.
#
# If not provided, and using Git, we'll give the user a File Not
# Found error with some info on what they need to do to correct
# this.
if base_commit_id:
revision = base_commit_id
elif repository.tool.name == 'Git':
raise FileNotFoundError(
path,
revision,
detail='The necessary revision information needed to find '
'this file was not provided. Use RBTools 0.5.2 or '
'newer.')
# NOTE: As of this writing, the 2.0 API does not support fetching
# the raw contents of files. We have to use the 1.0 API for
# this instead.
url = self._build_repository_api_url(repository,
'raw/%s/%s' %
(quote(revision), quote(path)),
version='1.0')
try:
return self._api_get(url, raw_content=True)
except FileNotFoundError:
raise FileNotFoundError(path,
revision=revision,
base_commit_id=base_commit_id)
def cat_file(self, extended_path, revision):
import tempfile
# Use tempfile to generate temporary filename
temp = tempfile.NamedTemporaryFile()
# Remove the file, so cleartool can write to it
temp.close()
cmdline = ["cleartool", "get", "-to", temp.name, extended_path]
p = subprocess.Popen(
cmdline,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=_popen_shell)
(res, error) = p.communicate()
failure = p.poll()
if failure:
raise FileNotFoundError(extended_path, revision)
try:
with open(temp.name, 'rb') as f:
return f.read()
except:
raise FileNotFoundError(extended_path, revision)
def _api_get_src(self, repository, path, revision, base_commit_id):
# If a base commit ID is provided, use it. It may not be provided,
# though, and in this case, we need to use the provided revision,
# which will work for Mercurial but not for Git.
#
# If not provided, and using Git, we'll give the user a File Not
# Found error with some info on what they need to do to correct
# this.
if base_commit_id:
revision = base_commit_id
elif repository.tool.name == 'Git':
raise FileNotFoundError(
path,
revision,
detail='The necessary revision information needed to find '
'this file was not provided. Use RBTools 0.5.2 or '
'newer.')
url = self._build_api_url(
'repositories/%s/%s/raw/%s/%s' %
(quote(self._get_repository_owner(repository)),
quote(self._get_repository_name(repository)), quote(revision),
quote(path)))
try:
return self._api_get(url, raw_content=True)
except FileNotFoundError:
raise FileNotFoundError(path,
revision=revision,
base_commit_id=base_commit_id)
def _do_on_path(self, cb, path, revision=HEAD):
if not path:
raise FileNotFoundError(path, revision)
try:
normpath = self.__normalize_path(path)
# SVN expects to have URLs escaped. Take care to only
# escape the path part of the URL.
if self.client.is_url(normpath):
pathtuple = urlparse.urlsplit(normpath)
path = pathtuple[2]
if isinstance(path, unicode):
path = path.encode('utf-8', 'ignore')
normpath = urlparse.urlunsplit(
(pathtuple[0], pathtuple[1], urllib.quote(path), '', ''))
normrev = self.__normalize_revision(revision)
return cb(normpath, normrev)
except ClientError, e:
stre = str(e)
if 'File not found' in stre or 'path not found' in stre:
raise FileNotFoundError(path, revision, detail=str(e))
elif 'callback_ssl_server_trust_prompt required' in stre:
raise SCMError(
'HTTPS certificate not accepted. Please ensure that '
'the proper certificate exists in %s '
'for the user that reviewboard is running as.' %
os.path.join(self.config_dir, 'auth'))
elif 'callback_get_login required' in stre:
raise AuthenticationError(
msg='Login to the SCM server failed.')
else:
raise SCMError(e)
def get_file(self, extended_path, revision=HEAD, **kwargs):
"""Return content of file or list content of directory"""
if not extended_path:
raise FileNotFoundError(extended_path, revision)
if revision == PRE_CREATION:
return ''
if self.viewtype == self.VIEW_SNAPSHOT:
# Get the path to (presumably) file element (remove version)
# The '@@' at the end of file_path is required.
file_path = extended_path.rsplit('@@', 1)[0] + '@@'
okind = self._get_object_kind(file_path)
if okind == 'directory element':
raise SCMError('Directory elements are unsupported.')
elif okind == 'file element':
output = self.client.cat_file(extended_path, revision)
else:
raise FileNotFoundError(extended_path, revision)
else:
if cpath.isdir(extended_path):
output = self.client.list_dir(extended_path, revision)
elif cpath.exists(extended_path):
output = self.client.cat_file(extended_path, revision)
else:
raise FileNotFoundError(extended_path, revision)
return output
def _do_on_path(self, cb, path, revision=HEAD):
if not path:
raise FileNotFoundError(path, revision)
try:
normpath = self.normalize_path(path)
# SVN expects to have URLs escaped. Take care to only
# escape the path part of the URL.
if self.client.is_url(normpath):
pathtuple = urlsplit(normpath)
path = pathtuple[2]
if isinstance(path, six.text_type):
path = path.encode('utf-8', 'ignore')
normpath = urlunsplit((pathtuple[0],
pathtuple[1],
quote(path),
'', ''))
normrev = self._normalize_revision(revision)
return cb(normpath, normrev)
except ClientError as e:
exc = force_text(e)
if 'File not found' in exc or 'path not found' in exc:
raise FileNotFoundError(path, revision, detail=exc)
elif 'callback_ssl_server_trust_prompt required' in exc:
raise SCMError(
_('HTTPS certificate not accepted. Please ensure that '
'the proper certificate exists in %s '
'for the user that reviewboard is running as.')
% os.path.join(self.config_dir, 'auth'))
else:
raise SVNTool.normalize_error(e)
def get_file(self, extended_path, revision=HEAD, **kwargs):
"""Return content of file or list content of directory.
Args:
extended_path (unicode):
The path of the element, including revision information.
revision (reviewboard.scmtools.core.Revision, optional):
Revision information. This will be either
:py:data:`~reviewboard.scmtools.core.PRE_CREATION` (new file),
or :py:data:`~reviewboard.scmtools.core.HEAD` (signifying to
use the revision information included in ``extended_path``).
**kwargs (dict, optional):
Additional unused keyword arguments.
Returns:
bytes:
The contents of the element.
Raises:
reviewboard.scmtools.errors.FileNotFoundError:
The given ``extended_path`` did not match a valid element.
reviewboard.scmtools.errors.SCMError:
Another error occurred.
"""
if not extended_path:
raise FileNotFoundError(extended_path, revision)
if revision == PRE_CREATION:
return ''
if self.viewtype == self.VIEW_SNAPSHOT:
# Get the path to (presumably) file element (remove version)
# The '@@' at the end of file_path is required.
file_path = extended_path.rsplit('@@', 1)[0] + '@@'
okind = self._get_element_kind(file_path)
if okind == 'directory element':
raise SCMError('Directory elements are unsupported.')
elif okind == 'file element':
output = self.client.cat_file(extended_path)
else:
raise FileNotFoundError(extended_path)
else:
if cpath.isdir(extended_path):
output = self.client.list_dir(extended_path)
elif cpath.exists(extended_path):
output = self.client.cat_file(extended_path)
else:
raise FileNotFoundError(extended_path)
return output
def _api_get_file_meta(self, repository, path, revision, base_commit_id):
"""Return metadata on a file.
Args:
repository (reviewboard.scmtools.models.Repository):
The repository containing the file.
path (unicode):
The path to the file.
revision (unicode):
The revision of the file.
base_commit_id (unicode):
The SHA1 of the commit to fetch the file at. If provided,
this will take precedence over ``revision``.
This is needed by Git.
Returns:
dict:
The metadata on the file.
Raises:
reviewboard.scmtools.errors.FileNotFoundError:
The file could not be found.
"""
# If a base commit ID is provided, use it. It may not be provided,
# though, and in this case, we need to use the provided revision,
# which will work for Mercurial but not for Git.
#
# If not provided, and using Git, we'll give the user a File Not
# Found error with some info on what they need to do to correct
# this.
if base_commit_id:
revision = base_commit_id
elif repository.tool.name == 'Git':
raise FileNotFoundError(
path,
revision,
detail='The necessary revision information needed to find '
'this file was not provided. Use RBTools 0.5.2 or '
'newer.')
url = ('%s?format=meta' % self._build_repository_api_url(
repository, 'src/%s/%s' % (quote(revision), quote(path))))
try:
return self.api_get(url)
except FileNotFoundError:
raise FileNotFoundError(path,
revision=revision,
base_commit_id=base_commit_id)
def api_get_blob(self, repo_api_url, path, sha):
"""Return the contents of a file using the GitHub API.
Args:
repo_api_url (unicode):
The absolute URL for the base repository API.
path (unicode):
The path of the file within the repository.
sha (unicode):
The SHA1 of the file within the repository.
Returns:
bytes:
The contents of the file.
Raises:
reviewboard.scmtools.errors.FileNotFoundError:
The file could not be found or the API could not be accessed.
"""
try:
return self.http_get(url='%s/git/blobs/%s' % (repo_api_url, sha),
headers={
'Accept': self.RAW_MIMETYPE,
}).data
except HostingServiceError:
raise FileNotFoundError(path, sha)
def get_file(self,
repository,
path,
revision,
base_commit_id=None,
*args,
**kwargs):
"""Fetches a file from Unfuddle.
This will perform an API request to fetch the contents of a file.
If using Git, this will expect a base commit ID to be provided.
"""
try:
commit_id = self._get_commit_id(repository, path, revision,
base_commit_id)
url = self._build_api_url(
self._get_repository_account_domain(repository),
'repositories/%s/download/?path=%s&commit=%s' %
(self._get_repository_id(repository), quote(path),
quote(commit_id)))
return self._api_get(url, raw_content=True)
except (HTTPError, URLError):
raise FileNotFoundError(path, revision)
def get_file(self, path, revision=HEAD):
"""Returns the contents of a given file at the given revision."""
if not path:
raise FileNotFoundError(path, revision)
revnum = self._normalize_revision(revision)
path = B(self.normalize_path(path))
data = six.StringIO()
try:
self.client.cat(path, data, revnum)
except SubversionException as e:
raise FileNotFoundError(e)
contents = data.getvalue()
keywords = self.get_keywords(path, revision)
if keywords:
contents = self.collapse_keywords(contents, keywords)
return contents
def get_file(self,
repository,
path,
revision,
base_commit_id=None,
*args,
**kwargs):
"""Fetches a file from Beanstalk.
This will perform an API request to fetch the contents of a file.
If using Git, this will expect a base commit ID to be provided.
"""
try:
contents = self._api_get_node(repository,
path,
revision,
base_commit_id,
contents=True)
except URLError:
raise FileNotFoundError(path, revision)
# On Subversion repositories, we may need to expand properties within
# the file, like ``$Id$``. We only want to do this if we see keywords.
if repository.tool.name == 'Subversion':
contents = self._normalize_svn_file_content(
repository, contents, path, revision)
return contents
def api_get_blob(self, repo_api_url, path, sha):
"""Return the contents of a file using the GitHub API.
Args:
repo_api_url (unicode):
The absolute URL for the base repository API.
path (unicode):
The path of the file within the repository.
sha (unicode):
The SHA1 of the file within the repository.
Returns:
bytes:
The contents of the file.
Raises:
reviewboard.scmtools.errors.FileNotFoundError:
The file could not be found or the API could not be accessed.
"""
credentials = self.get_http_credentials(self.account)
url = '%s/git/blobs/%s' % (repo_api_url, sha)
try:
return self.http_get(url,
headers={
'Accept': self.RAW_MIMETYPE,
},
**credentials)[0]
except (URLError, HTTPError):
raise FileNotFoundError(path, sha)
def get_file(self, repository, path, revision, *args, **kwargs):
"""Return the specified file from the repository.
If the given file path is ``/invalid-path``, the file will be assumed
to not exist and
:py:exc:`reviewboard.scmtools.errors.FileNotFoundError` will be raised.
Args:
repository (reviewboard.scmtools.models.Repository):
The repository the file belongs to.
path (unicode):
The file path.
revision (unicode):
The file revision.
*args (tuple):
Additional positional arguments.
**kwargs (dict):
Additional keyword arguments.
Returns:
unicode: The file data.
Raises:
reviewboard.scmtools.errors.FileNotFoundError:
Raised if the file does not exist.
"""
if path == '/invalid-path':
raise FileNotFoundError(path, revision)
return super(TestService, self).get_file(repository, path, revision,
*args, **kwargs)
def get_file(self, extended_path, revision=HEAD):
"""Return content of file or list content of directory"""
if not extended_path:
raise FileNotFoundError(extended_path, revision)
if revision == PRE_CREATION:
return ''
if cpath.isdir(extended_path):
output = self.client.list_dir(extended_path, revision)
elif cpath.exists(extended_path):
output = self.client.cat_file(extended_path, revision)
else:
raise FileNotFoundError(extended_path, revision)
return output
def _normalize_revision(self, revision):
"""Normalize a revision to an integer or byte string.
Args:
revision (object):
The revision to normalize. This can be an integer, byte string,
Unicode string,
:py:class:`~reviewboard.scmtools.core.Revision` object, or
``None``.
Returns:
object:
The resulting revision. This may be an integer (if providing
a revision number) or a Unicode string (if using an identifier
like "HEAD").
Raises:
reviewboard.scmtools.errors.FileNotFoundError:
The revision indicates that the file does not yet exist.
"""
if revision is None:
return None
elif revision == HEAD:
return 'HEAD'
elif revision == PRE_CREATION:
raise FileNotFoundError('', revision)
elif isinstance(revision, Revision):
revision = int(revision.name)
elif isinstance(revision, (six.text_type, six.binary_type)):
revision = int(revision)
return revision
def _check_api_error(self, e):
data = e.read()
try:
rsp = json.loads(data)
except:
rsp = None
message = data
if rsp and 'error' in rsp:
error = rsp['error']
if 'message' in error:
message = error['message']
if message:
message = six.text_type(message)
if e.code == 401:
self._raise_auth_error(message)
elif e.code == 404:
if message.startswith('Repository'):
raise HostingServiceError(message, http_code=e.code)
# We don't have a path here, but it will be filled in inside
# _api_get_src.
raise FileNotFoundError('')
else:
raise HostingServiceAPIError(
message
or (ugettext('Unexpected HTTP %s error when talking to '
'Bitbucket') % e.code),
http_code=e.code,
rsp=e)
def _cat_file(self, path, revision, option):
"""
Call git-cat-file(1) to get content or type information for a
repository object.
If called with just "commit", gets the content of a blob (or
raises an exception if the commit is not a blob).
Otherwise, "option" can be used to pass a switch to git-cat-file,
e.g. to test or existence or get the type of "commit".
"""
commit = self._resolve_head(revision, path)
p = self._run_git(['--git-dir=%s' % self.git_dir, 'cat-file',
option, commit])
contents = p.stdout.read()
errmsg = p.stderr.read()
failure = p.wait()
if failure:
if errmsg.startswith("fatal: Not a valid object name"):
raise FileNotFoundError(commit)
else:
raise SCMError(errmsg)
return contents
def get_file_http(self, url, path, revision):
logging.info('Fetching file from %s' % url)
try:
request = URLRequest(url)
if self.username:
auth_string = base64.b64encode('%s:%s' %
(self.username, self.password))
request.add_header('Authorization', 'Basic %s' % auth_string)
return urlopen(request).read()
except HTTPError as e:
if e.code == 404:
logging.error('404')
raise FileNotFoundError(path, revision)
else:
msg = "HTTP error code %d when fetching file from %s: %s" % \
(e.code, url, e)
logging.error(msg)
raise SCMError(msg)
except Exception as e:
msg = "Unexpected error fetching file from %s: %s" % (url, e)
logging.error(msg)
raise SCMError(msg)
def _get_file_http(client, url, path, revision, mime_type, *args,
**kwargs):
if url.startswith('http://hg.example.com/raw-rev/'):
self.assertEqual(
url, 'http://hg.example.com/raw-rev/'
'1ca5879492b8fd606df1964ea3c1e2f4520f076f')
self.assertEqual(path, '')
self.assertEqual(revision, '')
self.assertIsNone(mime_type)
return b'diff payload'
elif url.startswith('http://hg.example.com/json-rev/'):
self.assertEqual(
url, 'http://hg.example.com/json-rev/'
'1ca5879492b8fd606df1964ea3c1e2f4520f076f')
self.assertEqual(mime_type, 'application/json')
self.assertEqual(path, '')
self.assertEqual(revision, '')
return self._dump_json({
'node':
'1ca5879492b8fd606df1964ea3c1e2f4520f076f',
'desc':
'This is the change description',
'user':
'******',
'date': [1583149219, 28800],
'parents': ['b9af6489f6f2004ad11b82c6057f7007e3c35372'],
})
else:
raise FileNotFoundError(path=path, revision=revision)
def _check_api_error(self, e):
data = e.read()
try:
rsp = json.loads(data)
except:
rsp = None
message = data
if rsp and 'error' in rsp:
error = rsp['error']
if 'message' in error:
message = error['message']
if message:
message = six.text_type(message)
if e.code == 401:
raise AuthorizationError(
message or ugettext('Invalid Bitbucket username or password'))
elif e.code == 404:
# We don't have a path here, but it will be filled in inside
# _api_get_src.
raise FileNotFoundError('')
else:
raise HostingServiceError(
message or ugettext('Unknown error when talking to Bitbucket'))
def get_file(self, path, revspec):
"""Return the contents of a file.
This expects a path within the repository and a Bazaar revision
specifier.
Args:
path (unicode):
The path to the file within the repository.
revspec (unicode):
The Bazaar revision specifier used to look up the file.
Returns:
bytes:
The contents of the file.
Raises:
reviewboard.scmtools.errors.FileNotFoundError:
The file could not be found.
"""
path = self._build_repo_path(path)
p = self._run_bzr(['cat', '-r', revspec, path])
contents = p.stdout.read()
errmsg = force_text(p.stderr.read())
failure = p.wait()
self._check_error(errmsg)
if failure:
raise FileNotFoundError(path=path, revision=revspec, detail=errmsg)
return contents
def get_file(self, path, revision=HEAD):
if not path:
raise FileNotFoundError(path, revision)
try:
normpath = self.__normalize_path(path)
# SVN expects to have URLs escaped. Take care to only
# escape the path part of the URL.
if self.client.is_url(normpath):
pathtuple = urlparse.urlsplit(normpath)
path = pathtuple[2]
if isinstance(path, unicode):
path = path.encode('utf-8', 'ignore')
normpath = urlparse.urlunsplit((pathtuple[0],
pathtuple[1],
urllib.quote(path),
'',''))
normrev = self.__normalize_revision(revision)
data = self.client.cat(normpath, normrev)
# Find out if this file has any keyword expansion set.
# If it does, collapse these keywords. This is because SVN
# will return the file expanded to us, which would break patching.
keywords = self.client.propget("svn:keywords", normpath, normrev,
recurse=True)
if normpath in keywords:
data = self.collapse_keywords(data, keywords[normpath])
return data
except ClientError, e:
stre = str(e)
if 'File not found' in stre or 'path not found' in stre:
raise FileNotFoundError(path, revision, str(e))
elif 'callback_ssl_server_trust_prompt required' in stre:
raise SCMError(
'HTTPS certificate not accepted. Please ensure that '
'the proper certificate exists in %s '
'for the user that reviewboard is running as.'
% os.path.join(self.config_dir, 'auth'))
elif 'callback_get_login required' in stre:
raise SCMError('Login to the SCM server failed.')
else:
raise SCMError(e)
def get_file(self, repository, path, revision, *args, **kwargs):
"""Return the contents of a file at the specified revision.
Args:
repository (reviewboard.scmtools.models.Repository):
The repository configured to use Gerrit.
path (unicode):
The file path (ignored).
revision (unicode):
The file's Git object ID.
*args (tuple):
Ignored positional arguments.
**kwargs (dict):
Ignored keyword arguments.
Returns:
bytes:
The contents of the file.
Raises:
reviewboard.hostingsvcs.errors.FileNotFoundError:
The file does not exist in the remote repository.
reviewboard.hostingsvcs.errors.HostingServiceAPIError:
An error occurred communicating with the Gerrit API.
"""
url = self._build_project_api_url(repository, 'blobs', revision,
'content')
try:
rsp = self.client.http_get(url).data
except HostingServiceAPIError as e:
if e.http_code == 404:
raise FileNotFoundError(path, revision=revision)
raise HostingServiceAPIError(
ugettext('Could not get file "%(file)s" at revision '
'"%(rev)s": %(error)s') % {
'file': path,
'rev': revision,
'error': e.read(),
},
http_code=e.http_code)
try:
return base64.b64decode(rsp)
except Exception as e:
raise HostingServiceAPIError(
ugettext('An error occurred while retrieving "%(file)s" at '
'revision "%(rev)s" from Gerrit: the response could '
'not be decoded: %(error)s') % {
'file': path,
'rev': revision,
'error': e,
})
def __normalize_revision(self, revision):
if revision == HEAD:
r = Revision(opt_revision_kind.head)
elif revision == PRE_CREATION:
raise FileNotFoundError('', revision)
else:
r = Revision(opt_revision_kind.number, str(revision))
return r
def api_get_blob(self, repo_api_url, path, sha):
url = self._build_api_url(repo_api_url, 'git/blobs/%s' % sha)
try:
return self.http_get(url, headers={
'Accept': self.RAW_MIMETYPE,
})[0]
except (URLError, HTTPError):
raise FileNotFoundError(path, sha)
def get_file(self, repository, path, revision, *args, **kwargs):
url = self._build_api_url(repository, 'git/blobs/%s' % revision)
try:
return self._http_get(url, headers={
'Accept': self.RAW_MIMETYPE,
})[0]
except (urllib2.URLError, urllib2.HTTPError):
raise FileNotFoundError(path, revision)
def _normalize_revision(self, revision, negatives_allowed=True):
if revision == HEAD:
return B('HEAD')
elif revision == PRE_CREATION:
raise FileNotFoundError('', revision)
elif isinstance(revision, Revision):
revnum = int(revision.name)
elif isinstance(revision, (B, ) + six.string_types):
revnum = int(revision)
return revnum
def get_file_http(self, url, path, revision):
"""Return the contents of a file from an HTTP(S) URL.
This is a convenience for looking up the contents of files that are
referenced in diffs through an HTTP(S) request.
Authentication is performed using the username and password provided
(if any).
Args:
url (unicode):
The URL to fetch the file contents from.
path (unicode):
The path of the file, as referenced in the diff.
revision (Revision):
The revision of the file, as referenced in the diff.
Returns:
bytes:
The contents of the file.
Raises:
reviewboard.scmtools.errors.FileNotFoundError:
The file could not be found.
reviewboard.scmtools.errors.SCMError:
Unexpected error in fetching the file. This may be an
unexpected HTTP status code.
"""
logging.info('Fetching file from %s' % url)
try:
request = URLRequest(url)
if self.username:
auth_string = base64.b64encode('%s:%s' %
(self.username, self.password))
request.add_header('Authorization', 'Basic %s' % auth_string)
return urlopen(request).read()
except HTTPError as e:
if e.code == 404:
logging.error('404')
raise FileNotFoundError(path, revision)
else:
msg = "HTTP error code %d when fetching file from %s: %s" % \
(e.code, url, e)
logging.error(msg)
raise SCMError(msg)
except Exception as e:
msg = "Unexpected error fetching file from %s: %s" % (url, e)
logging.error(msg)
raise SCMError(msg)
def get_file(self, repository, path, revision, base_commit_id=None,
*args, **kwargs):
"""Fetches a file from GitLab.
This will perform an API request to fetch the contents of a file.
"""
try:
return self._api_get(
self._get_blob_url(repository, path, revision, base_commit_id),
raw_content=True)
except (HTTPError, URLError):
raise FileNotFoundError(path, revision)
