def get_change(self, revision):
"""Return detailed information about a changeset.
This method retrieves the patch in JSON format from hgweb.
Args:
revision (unicode):
An identifier of changeset
Returns:
reviewboard.scmtools.core.Commit:
The commit object.
"""
try:
url = '%s/raw-rev/%s' % (self.path_stripped, revision)
contents = self.get_file_http(url, '', '')
except Exception as e:
logging.exception('Cannot load patch from hgweb: %s', e)
raise SCMError('Cannot load patch from hgweb')
if contents:
changeset = self._get_commit(revision)
if changeset:
changeset.diff = contents
return changeset
logging.error('Cannot load changeset %s from hgweb', revision)
raise SCMError('Cannot load changeset %s from hgweb' % revision)
def parse_diff_revision(self, file_str, revision_str):
# The "(revision )" is generated by IntelliJ and has the same
# meaning as "(working copy)". See bug 1937.
if revision_str in ("(working copy)", "(revision )"):
return file_str, HEAD
# Binary diffs don't provide revision information, so we set a fake
# "(unknown)" in the SVNDiffParser. This will never actually appear
# in SVN diffs.
if revision_str == "(unknown)":
return file_str, UNKNOWN
m = self.revision_re.match(revision_str)
if not m:
raise SCMError("Unable to parse diff revision header '%s'" %
revision_str)
relocated_file = m.group(2)
revision = m.group(3)
if revision == "0":
revision = PRE_CREATION
if relocated_file:
if not relocated_file.startswith("..."):
raise SCMError("Unable to parse SVN relocated path '%s'" %
relocated_file)
file_str = "%s/%s" % (relocated_file[4:], file_str)
return file_str, revision
def get_change(self, revision):
"""Return detailed information about a changeset.
Receive changeset data and patch from repository in JSON.
Args:
revision (unicode):
An identifier of changeset.
Returns:
reviewboard.scmtools.core.Commit:
The commit object.
"""
revset = ['-r', revision]
changesets = self._get_commits(revset)
if changesets:
commit = changesets[0]
cmd = ['diff', '-c', revision]
p = self._run_hg(cmd)
if p.wait() != 0:
e = p.stderr.read()
raise SCMError('Cannot load patch %s: %s' % (revision, e))
commit.diff = p.stdout.read()
return commit
raise SCMError('Cannot load changeset %s' % revision)
def _do_on_path(self, cb, path, revision=HEAD):
if not path:
raise FileNotFoundError(path, revision)
try:
normpath = self.__normalize_path(path)
# SVN expects to have URLs escaped. Take care to only
# escape the path part of the URL.
if self.client.is_url(normpath):
pathtuple = urlparse.urlsplit(normpath)
path = pathtuple[2]
if isinstance(path, unicode):
path = path.encode('utf-8', 'ignore')
normpath = urlparse.urlunsplit(
(pathtuple[0], pathtuple[1], urllib.quote(path), '', ''))
normrev = self.__normalize_revision(revision)
return cb(normpath, normrev)
except ClientError, e:
stre = str(e)
if 'File not found' in stre or 'path not found' in stre:
raise FileNotFoundError(path, revision, detail=str(e))
elif 'callback_ssl_server_trust_prompt required' in stre:
raise SCMError(
'HTTPS certificate not accepted. Please ensure that '
'the proper certificate exists in %s '
'for the user that reviewboard is running as.' %
os.path.join(self.config_dir, 'auth'))
elif 'callback_get_login required' in stre:
raise AuthenticationError(
msg='Login to the SCM server failed.')
else:
raise SCMError(e)
def get_file_http(self, url, path, revision):
logging.info('Fetching file from %s' % url)
try:
request = URLRequest(url)
if self.username:
auth_string = base64.b64encode('%s:%s' %
(self.username, self.password))
request.add_header('Authorization', 'Basic %s' % auth_string)
return urlopen(request).read()
except HTTPError as e:
if e.code == 404:
logging.error('404')
raise FileNotFoundError(path, revision)
else:
msg = "HTTP error code %d when fetching file from %s: %s" % \
(e.code, url, e)
logging.error(msg)
raise SCMError(msg)
except Exception as e:
msg = "Unexpected error fetching file from %s: %s" % (url, e)
logging.error(msg)
raise SCMError(msg)
def __init__(self, path, raw_file_url=None):
if not is_exe_in_path('git'):
# This is technically not the right kind of error, but it's the
# pattern we use with all the other tools.
raise ImportError
self.path = self._normalize_git_url(path)
self.raw_file_url = raw_file_url
self.git_dir = None
url_parts = urlparse.urlparse(self.path)
if url_parts[0] == 'file':
self.git_dir = url_parts[2]
p = self._run_git(['--git-dir=%s' % self.git_dir, 'config',
'core.repositoryformatversion'])
failure = p.wait()
if failure:
# See if we have a permissions error
if not os.access(self.git_dir, os.R_OK):
raise SCMError(_("Permission denied accessing the local "
"Git repository '%s'") % self.git_dir)
else:
raise SCMError(_('Unable to retrieve information from '
'local Git repository'))
def get_file_http(self, url, path, revision):
"""Return the contents of a file from an HTTP(S) URL.
This is a convenience for looking up the contents of files that are
referenced in diffs through an HTTP(S) request.
Authentication is performed using the username and password provided
(if any).
Args:
url (unicode):
The URL to fetch the file contents from.
path (unicode):
The path of the file, as referenced in the diff.
revision (Revision):
The revision of the file, as referenced in the diff.
Returns:
bytes:
The contents of the file.
Raises:
reviewboard.scmtools.errors.FileNotFoundError:
The file could not be found.
reviewboard.scmtools.errors.SCMError:
Unexpected error in fetching the file. This may be an
unexpected HTTP status code.
"""
logging.info('Fetching file from %s' % url)
try:
request = URLRequest(url)
if self.username:
auth_string = base64.b64encode('%s:%s' %
(self.username, self.password))
request.add_header('Authorization', 'Basic %s' % auth_string)
return urlopen(request).read()
except HTTPError as e:
if e.code == 404:
logging.error('404')
raise FileNotFoundError(path, revision)
else:
msg = "HTTP error code %d when fetching file from %s: %s" % \
(e.code, url, e)
logging.error(msg)
raise SCMError(msg)
except Exception as e:
msg = "Unexpected error fetching file from %s: %s" % (url, e)
logging.error(msg)
raise SCMError(msg)
def get_file(self, path, revision):
if revision == BZRTool.PRE_CREATION_TIMESTAMP:
return ''
revspec = self._revspec_from_revision(revision)
filepath = self._get_full_path(path)
branch = None
try:
try:
branch, relpath = bzrdir.BzrDir.open_containing_tree_or_branch(
filepath)[1:]
branch.lock_read()
revtree = revisionspec.RevisionSpec.from_string(
revspec).as_tree(branch)
fileid = revtree.path2id(relpath)
if fileid:
contents = revtree.get_file_text(fileid)
else:
contents = ""
except BzrError, e:
raise SCMError(e)
finally:
if branch:
branch.unlock()
return contents
def get_commits(self, repository, branch=None, start=None):
if start is not None:
url = ('%s/repos/%s/branches/%s/commits?start=%s'
% (self.account.hosting_url,
repository.extra_data['rbgateway_repo_name'],
branch,
start))
else:
url = ('%s/repos/%s/branches/%s/commits'
% (self.account.hosting_url,
repository.extra_data['rbgateway_repo_name'],
branch))
try:
data, headers = self._api_get(url)
commits = json.loads(data)
results = []
for commit in commits:
results.append(Commit(commit['author'],
commit['id'],
commit['date'],
commit['message'],
commit['parent_id']))
return results
except Exception as e:
logger.exception('Failed to fetch commits from %s: %s', url, e)
raise SCMError(six.text_type(e))
def check_repository(cls,
path,
username=None,
password=None,
local_site_name=None):
"""
Performs checks on a repository to test its validity.
This should check if a repository exists and can be connected to.
This will also check if the repository requires an HTTPS certificate.
The result is returned as an exception. The exception may contain
extra information, such as a human-readable description of the problem.
If the repository is valid and can be connected to, no exception
will be thrown.
"""
super(BZRTool, cls).check_repository(path, username, password,
local_site_name)
if local_site_name and sshutils.is_ssh_uri(path):
path += '?rb-local-site-name=%s' % local_site_name
try:
tree, branch, repository, relpath = \
bzrdir.BzrDir.open_containing_tree_branch_or_repository(
path.encode('ascii'))
except AttributeError:
raise RepositoryNotFoundError()
except NotBranchError:
raise RepositoryNotFoundError()
except Exception as e:
raise SCMError(e)
def get_file(self, path, revision, **kwargs):
if revision == BZRTool.PRE_CREATION_TIMESTAMP:
return ''
revspec = self._revspec_from_revision(revision)
filepath = self._get_full_path(path)
branch = None
try:
try:
branch, relpath = bzrdir.BzrDir.open_containing_tree_or_branch(
filepath.encode('ascii'))[1:]
branch.lock_read()
revtree = revisionspec.RevisionSpec.from_string(
revspec.encode('ascii')).as_tree(branch)
fileid = revtree.path2id(relpath)
if fileid:
# XXX: get_file_text returns str, which isn't Python 3
# safe. According to the internet they have no immediate
# plans to port to 3, so we may find it hard to support
# that combination.
contents = bytes(revtree.get_file_text(fileid))
else:
contents = b''
except BzrError as e:
raise SCMError(e)
finally:
if branch:
branch.unlock()
return contents
def _resolve_head(self, revision, path):
if revision == HEAD:
if path == "":
raise SCMError("path must be supplied if revision is %s" % HEAD)
return "HEAD:%s" % path
else:
return str(revision)
def get_change(self, revision):
"""Get an individual change.
This returns a Commit object containing the details of the commit.
"""
revision = int(revision)
commits = self.client.get_log('/', start=revision, limit=2)
commit = commits[0]
message = commit['message'].decode('utf-8', 'replace')
author_name = commit['author'].decode('utf-8', 'replace')
date = commit['date'].isoformat()
if len(commits) > 1:
base_revision = commits[1]['revision']
else:
base_revision = 0
try:
diff = self.client.diff(base_revision, revision)
except Exception as e:
raise SCMError(e)
commit = Commit(author_name, six.text_type(revision), date, message,
six.text_type(base_revision))
commit.diff = diff
return commit
def _cat_file(self, path, revision, option):
"""
Call git-cat-file(1) to get content or type information for a
repository object.
If called with just "commit", gets the content of a blob (or
raises an exception if the commit is not a blob).
Otherwise, "option" can be used to pass a switch to git-cat-file,
e.g. to test or existence or get the type of "commit".
"""
commit = self._resolve_head(revision, path)
p = self._run_git(['--git-dir=%s' % self.git_dir, 'cat-file',
option, commit])
contents = p.stdout.read()
errmsg = p.stderr.read()
failure = p.wait()
if failure:
if errmsg.startswith("fatal: Not a valid object name"):
raise FileNotFoundError(commit)
else:
raise SCMError(errmsg)
return contents
def _get_commits(self, revset):
"""Return a list of commit objects.
This method calls the given revset and parses the returned
JSON data to retrieve detailed information about changesets.
Args:
revset (list of unicode):
Hg command line that will be executed with JSON
template as log command.
Returns:
list of reviewboard.scmtools.core.Commit:
The list of commit objects.
"""
cmd = ['log'] + revset + ['--template', 'json']
p = self._run_hg(cmd)
if p.wait() != 0:
raise SCMError('Cannot load commits: %s' % p.stderr.read())
results = []
for data in json.load(p.stdout):
p = data['parents'][0]
results.append(
Commit(id=data['node'],
message=data['desc'],
author_name=data['user'],
date=HgTool.date_tuple_to_iso8601(data['date']),
parent=p,
base_commit_id=p))
return results
def get_file(self, path, revision):
logging.debug('Plastic: get_file %s rev %s' % (path, revision))
repo = "rep:%s@repserver:%s:%s" % (self.reponame, self.hostname,
self.port)
# Work around a plastic bug, where 'cm cat --file=blah' gets an
# extra newline, but plain 'cm cat' doesn't
fd, tmpfile = mkstemp()
os.close(fd)
p = subprocess.Popen(
['cm', 'cat', revision + '@' + repo, '--file=' + tmpfile],
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
close_fds=(os.name != 'nt'))
errmsg = six.text_type(p.stderr.read())
failure = p.wait()
if failure:
if not errmsg:
errmsg = p.stdout.read()
raise SCMError(errmsg)
with open(tmpfile, 'rb') as readtmp:
contents = readtmp.read()
os.unlink(tmpfile)
return contents
def diff(self, revision1, revision2, path=None):
"""Returns a diff between two revisions.
The diff will contain the differences between the two revisions,
and may optionally be limited to a specific path.
The returned diff will be returned as a Unicode object.
"""
if path:
path = self.normalize_path(path)
else:
path = self.repopath
tmpdir = mkdtemp(prefix='reviewboard-svn.')
try:
diff = force_bytes(self.client.diff(
tmpdir,
path,
revision1=self._normalize_revision(revision1),
revision2=self._normalize_revision(revision2),
header_encoding='UTF-8',
diff_options=['-u']))
except Exception as e:
logging.error('Failed to generate diff using pysvn for revisions '
'%s:%s for path %s: %s',
revision1, revision2, path, e, exc_info=1)
raise SCMError(
_('Unable to get diff revisions %s through %s: %s')
% (revision1, revision2, e))
finally:
rmtree(tmpdir)
return diff
def _check_error(self, errmsg):
"""Check an error message from bzr and raise an exception if needed.
If the error is an internal error, it will be raised, without the
exception. If it's a known error that we can report better information
on, then that information will be raised.
Args:
errmsg (unicode):
The error message.
Raises:
reviewboard.scmtools.errors.SCMError:
A suitable error message, if an internal error was hit.
"""
if 'Bazaar has encountered an internal error' in errmsg:
if 'prefetch() takes exactly 2 arguments (1 given)' in errmsg:
errmsg = ('Installed bzr and paramiko modules are '
'incompatible. See '
'https://bugs.launchpad.net/bzr/+bug/1524066')
else:
errmsg = errmsg.split(
'Traceback (most recent call last):')[0].strip()
raise SCMError(errmsg)
def normalize_error(cls, e):
if 'callback_get_login required' in six.text_type(e):
return AuthenticationError(
msg='Authentication failed when talking to the Subversion '
'repository')
else:
return SCMError(e)
def get_file(self, extended_path, revision=HEAD, **kwargs):
"""Return content of file or list content of directory"""
if not extended_path:
raise FileNotFoundError(extended_path, revision)
if revision == PRE_CREATION:
return ''
if self.viewtype == self.VIEW_SNAPSHOT:
# Get the path to (presumably) file element (remove version)
# The '@@' at the end of file_path is required.
file_path = extended_path.rsplit('@@', 1)[0] + '@@'
okind = self._get_object_kind(file_path)
if okind == 'directory element':
raise SCMError('Directory elements are unsupported.')
elif okind == 'file element':
output = self.client.cat_file(extended_path, revision)
else:
raise FileNotFoundError(extended_path, revision)
else:
if cpath.isdir(extended_path):
output = self.client.list_dir(extended_path, revision)
elif cpath.exists(extended_path):
output = self.client.cat_file(extended_path, revision)
else:
raise FileNotFoundError(extended_path, revision)
return output
def get_file(self, path, revision=HEAD):
if revision == PRE_CREATION:
return ''
if revision == HEAD:
file = path
else:
file = '%s#%s' % (path, revision)
cmdline = ['p4', '-p', self.p4.port]
if self.p4.user:
cmdline.extend(['-u', self.p4.user])
if self.p4.password:
cmdline.extend(['-P', self.p4.password])
cmdline.extend(['print', '-q', file])
p = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(res, errdata) = p.communicate()
failure = p.poll()
if failure:
error = errdata.splitlines()
# The command-line output is the same as the contents of a P4Error
# except they're prefixed with a line that says "Perforce client
# error:", and the lines of the error are indented with tabs.
if error[0].startswith("Perforce client error:"):
error = error[1:]
raise SCMError('\n'.join(line.lstrip("\t") for line in error))
else:
return res
def _do_on_path(self, cb, path, revision=HEAD):
if not path:
raise FileNotFoundError(path, revision)
try:
normpath = self.normalize_path(path)
# SVN expects to have URLs escaped. Take care to only
# escape the path part of the URL.
if self.client.is_url(normpath):
pathtuple = urlsplit(normpath)
path = pathtuple[2]
if isinstance(path, six.text_type):
path = path.encode('utf-8', 'ignore')
normpath = urlunsplit((pathtuple[0],
pathtuple[1],
quote(path),
'', ''))
normrev = self._normalize_revision(revision)
return cb(normpath, normrev)
except ClientError as e:
exc = force_text(e)
if 'File not found' in exc or 'path not found' in exc:
raise FileNotFoundError(path, revision, detail=exc)
elif 'callback_ssl_server_trust_prompt required' in exc:
raise SCMError(
_('HTTPS certificate not accepted. Please ensure that '
'the proper certificate exists in %s '
'for the user that reviewboard is running as.')
% os.path.join(self.config_dir, 'auth'))
else:
raise SVNTool.normalize_error(e)
def get_file(self, path, revision=HEAD):
if not path:
raise FileNotFoundError(path, revision)
try:
normpath = self.__normalize_path(path)
# SVN expects to have URLs escaped. Take care to only
# escape the path part of the URL.
if self.client.is_url(normpath):
pathtuple = urlparse.urlsplit(normpath)
path = pathtuple[2]
if isinstance(path, unicode):
path = path.encode('utf-8', 'ignore')
normpath = urlparse.urlunsplit((pathtuple[0],
pathtuple[1],
urllib.quote(path),
'',''))
normrev = self.__normalize_revision(revision)
data = self.client.cat(normpath, normrev)
# Find out if this file has any keyword expansion set.
# If it does, collapse these keywords. This is because SVN
# will return the file expanded to us, which would break patching.
keywords = self.client.propget("svn:keywords", normpath, normrev,
recurse=True)
if normpath in keywords:
data = self.collapse_keywords(data, keywords[normpath])
return data
except ClientError, e:
stre = str(e)
if 'File not found' in stre or 'path not found' in stre:
raise FileNotFoundError(path, revision, str(e))
elif 'callback_ssl_server_trust_prompt required' in stre:
raise SCMError(
'HTTPS certificate not accepted. Please ensure that '
'the proper certificate exists in %s '
'for the user that reviewboard is running as.'
% os.path.join(self.config_dir, 'auth'))
elif 'callback_get_login required' in stre:
raise SCMError('Login to the SCM server failed.')
else:
raise SCMError(e)
def _convert_p4exception_to_scmexception(e):
error = str(e)
if 'Perforce password' in error or 'Password must be set' in error:
raise AuthenticationError(msg=error)
elif 'check $P4PORT' in error:
raise RepositoryNotFoundError
else:
raise SCMError(error)
def get_branches(self, repository):
"""Return the branches for the repository.
Args:
repository (reviewboard.scmtools.models.Repository):
The repository to fetch branches for.
Returns:
list of reviewboard.scmtools.core.Branch:
The branches returned from the Review Board Gateway API.
Raises:
SCMError:
The branches could not be retrieved from Review Board Gateway.
"""
url = ('%s/repos/%s/branches' %
(self.account.hosting_url,
repository.extra_data['rbgateway_repo_name']))
tool_name = repository.scmtool_class.name
if tool_name == 'Git':
default_branch = 'master'
elif tool_name == 'Mercurial':
default_branch = 'default'
else:
raise SCMError('Review Board Gateway does not support %s',
tool_name)
try:
data, headers = self._api_get(url)
branches = json.loads(data)
results = []
for branch in branches:
results.append(Branch(
id=branch['name'],
commit=branch['id'],
default=(branch['name'] == default_branch)
))
return results
except Exception as e:
logger.exception('Failed to get branches from %s: %s', url, e)
raise SCMError(six.text_type(e))
def __init__(self,
path,
raw_file_url=None,
username=None,
password=None,
encoding='',
local_site_name=None):
super(GitClient, self).__init__(self._normalize_git_url(path),
username=username,
password=password)
if not is_exe_in_path('git'):
# This is technically not the right kind of error, but it's the
# pattern we use with all the other tools.
raise ImportError
self.raw_file_url = raw_file_url
self.encoding = encoding
self.local_site_name = local_site_name
self.git_dir = None
url_parts = urllib_urlparse(self.path)
if url_parts[0] == 'file':
if platform.system() == "Windows":
# Windows requires drive letter (e.g. C:/)
self.git_dir = url_parts[1] + url_parts[2]
else:
self.git_dir = url_parts[2]
p = self._run_git([
'--git-dir=%s' % self.git_dir, 'config',
'core.repositoryformatversion'
])
failure = p.wait()
if failure:
# See if we have a permissions error
if not os.access(self.git_dir, os.R_OK):
raise SCMError(
_("Permission denied accessing the local "
"Git repository '%s'") % self.git_dir)
else:
raise SCMError(
_('Unable to retrieve information from '
'local Git repository'))
def get_changeset(self, changesetid):
logging.debug('Plastic: get_changeset %s' % (changesetid))
changesetdata = self.client.get_changeset(changesetid)
logging.debug('Plastic: changesetdata %s' % (changesetdata))
# Changeset data is in the form of multiple lines of:
# <changesetid> <user> <revid> <file spec>
#
# We assume the user and comment will be the same for each item, so
# read it out of the first.
#
changeset = ChangeSet()
changeset.changenum = changesetid
split = changesetdata.split('\n')
m = self.CS_RE.match(split[0])
revid = m.group("revid")
changeset.username = m.group("user")
changeset.summary = self.client.get_changeset_comment(
changesetid, revid)
logging.debug('Plastic: changeset user %s summary %s' %
(changeset.username, changeset.summary))
for line in split:
if line:
m = self.CS_RE.match(line)
if not m:
logging.debug('Plastic: bad re %s failed to match %s' %
(self.CS_RE, line))
raise SCMError("Error looking up changeset")
if m.group("csid") != str(changesetid):
logging.debug('Plastic: csid %s != %s' %
(m.group("csid"), changesetid))
raise SCMError(
"The server returned a changeset ID that was not requested"
)
logging.debug('Plastic: adding file %s' % (m.group("file")))
changeset.files += m.group("file")
return changeset
def parse_diff_revision(self, file_str, revision_str):
if revision_str == "PRE-CREATION":
return file_str, PRE_CREATION
m = self.rev_re.match(revision_str)
if not m:
raise SCMError("Unable to parse diff revision header '%s'" %
revision_str)
return file_str, m.group(1)
def api_get_issue(self, repo_api_url, issue_id):
url = self._build_api_url(repo_api_url, 'issues/%s' % issue_id)
try:
return self.api_get(url)
except Exception as e:
logging.warning('GitHub: Failed to fetch issue from %s: %s',
url, e, exc_info=1)
raise SCMError(six.text_type(e))
def _get_vobs_uuid(self, vobstag):
cmdline = ["cleartool", "lsvob", "-long", vobstag]
p = subprocess.Popen(cmdline,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=self.repopath)
(res, error) = p.communicate()
failure = p.poll()
if failure:
raise SCMError(error)
for line in res.splitlines(True):
if line.startswith('Vob family uuid:'):
return line.split(' ')[-1].rstrip()
raise SCMError("Can't find familly uuid for vob: %s" % vobstag)
