def get_file(self, path, revision, **kwargs):
if revision == BZRTool.PRE_CREATION_TIMESTAMP:
return ''
revspec = self._revspec_from_revision(revision)
filepath = self._get_full_path(path)
branch = None
try:
try:
branch, relpath = bzrdir.BzrDir.open_containing_tree_or_branch(
filepath.encode('ascii'))[1:]
branch.lock_read()
revtree = revisionspec.RevisionSpec.from_string(
revspec.encode('ascii')).as_tree(branch)
fileid = revtree.path2id(relpath)
if fileid:
# XXX: get_file_text returns str, which isn't Python 3
# safe. According to the internet they have no immediate
# plans to port to 3, so we may find it hard to support
# that combination.
contents = bytes(revtree.get_file_text(fileid))
else:
contents = b''
except BzrError as e:
raise SCMError(e)
finally:
if branch:
branch.unlock()
return contents
def check_repository(cls,
path,
username=None,
password=None,
local_site_name=None):
"""
Performs checks on a repository to test its validity.
This should check if a repository exists and can be connected to.
This will also check if the repository requires an HTTPS certificate.
The result is returned as an exception. The exception may contain
extra information, such as a human-readable description of the problem.
If the repository is valid and can be connected to, no exception
will be thrown.
"""
super(BZRTool, cls).check_repository(path, username, password,
local_site_name)
if local_site_name and sshutils.is_ssh_uri(path):
path += '?rb-local-site-name=%s' % local_site_name
try:
tree, branch, repository, relpath = \
bzrdir.BzrDir.open_containing_tree_branch_or_repository(
path.encode('ascii'))
except AttributeError:
raise RepositoryNotFoundError()
except NotBranchError:
raise RepositoryNotFoundError()
except Exception as e:
raise SCMError(e)
def get_file(self, extended_path, revision=HEAD, **kwargs):
"""Return content of file or list content of directory"""
if not extended_path:
raise FileNotFoundError(extended_path, revision)
if revision == PRE_CREATION:
return ''
if self.viewtype == self.VIEW_SNAPSHOT:
# Get the path to (presumably) file element (remove version)
# The '@@' at the end of file_path is required.
file_path = extended_path.rsplit('@@', 1)[0] + '@@'
okind = self._get_object_kind(file_path)
if okind == 'directory element':
raise SCMError('Directory elements are unsupported.')
elif okind == 'file element':
output = self.client.cat_file(extended_path, revision)
else:
raise FileNotFoundError(extended_path, revision)
else:
if cpath.isdir(extended_path):
output = self.client.list_dir(extended_path, revision)
elif cpath.exists(extended_path):
output = self.client.cat_file(extended_path, revision)
else:
raise FileNotFoundError(extended_path, revision)
return output
def _cat_file(self, path, revision, option):
"""
Call git-cat-file(1) to get content or type information for a
repository object.
If called with just "commit", gets the content of a blob (or
raises an exception if the commit is not a blob).
Otherwise, "option" can be used to pass a switch to git-cat-file,
e.g. to test or existence or get the type of "commit".
"""
commit = self._resolve_head(revision, path)
p = self._run_git(
['--git-dir=%s' % self.git_dir, 'cat-file', option, commit])
contents = p.stdout.read()
errmsg = six.text_type(p.stderr.read())
failure = p.wait()
if failure:
if errmsg.startswith("fatal: Not a valid object name"):
raise FileNotFoundError(commit)
else:
raise SCMError(errmsg)
return contents
def _check_error(self, errmsg):
"""Check an error message from bzr and raise an exception if needed.
If the error is an internal error, it will be raised, without the
exception. If it's a known error that we can report better information
on, then that information will be raised.
Args:
errmsg (unicode):
The error message.
Raises:
reviewboard.scmtools.errors.SCMError:
A suitable error message, if an internal error was hit.
"""
if 'Bazaar has encountered an internal error' in errmsg:
if 'prefetch() takes exactly 2 arguments (1 given)' in errmsg:
errmsg = ('Installed bzr and paramiko modules are '
'incompatible. See '
'https://bugs.launchpad.net/bzr/+bug/1524066')
else:
errmsg = errmsg.split(
'Traceback (most recent call last):')[0].strip()
raise SCMError(errmsg)
def get_file(self, path, revision):
if revision == BZRTool.PRE_CREATION_TIMESTAMP:
return ''
revspec = self._revspec_from_revision(revision)
filepath = self._get_full_path(path)
branch = None
try:
try:
branch, relpath = bzrdir.BzrDir.open_containing_tree_or_branch(
filepath)[1:]
branch.lock_read()
revtree = revisionspec.RevisionSpec.from_string(
revspec).as_tree(branch)
fileid = revtree.path2id(relpath)
if fileid:
contents = revtree.get_file_text(fileid)
else:
contents = ""
except BzrError, e:
raise SCMError(e)
finally:
if branch:
branch.unlock()
return contents
def _resolve_head(self, revision, path):
if revision == HEAD:
if path == "":
raise SCMError("path must be supplied if revision is %s" % HEAD)
return "HEAD:%s" % path
else:
return str(revision)
def diff(self, revision1, revision2, path=None):
"""Returns a diff between two revisions.
The diff will contain the differences between the two revisions,
and may optionally be limited to a specific path.
The returned diff will be returned as a Unicode object.
"""
if path:
path = self.normalize_path(path)
else:
path = self.repopath
tmpdir = mkdtemp(prefix='reviewboard-svn.')
try:
diff = force_bytes(self.client.diff(
tmpdir,
path,
revision1=self._normalize_revision(revision1),
revision2=self._normalize_revision(revision2),
header_encoding='UTF-8',
diff_options=['-u']))
except Exception as e:
logging.error('Failed to generate diff using pysvn for revisions '
'%s:%s for path %s: %s',
revision1, revision2, path, e, exc_info=1)
raise SCMError(
_('Unable to get diff revisions %s through %s: %s')
% (revision1, revision2, e))
finally:
rmtree(tmpdir)
return diff
def get_file(self, path, revision):
logging.debug('Plastic: get_file %s rev %s' % (path, revision))
repo = "rep:%s@repserver:%s:%s" % (self.reponame, self.hostname,
self.port)
# Work around a plastic bug, where 'cm cat --file=blah' gets an
# extra newline, but plain 'cm cat' doesn't
fd, tmpfile = mkstemp()
os.close(fd)
p = subprocess.Popen(['cm', 'cat', revision + '@' + repo,
'--file=' + tmpfile],
stderr=subprocess.PIPE, stdout=subprocess.PIPE,
close_fds=(os.name != 'nt'))
errmsg = p.stderr.read()
failure = p.wait()
if failure:
if not errmsg:
errmsg = p.stdout.read()
raise SCMError(errmsg)
readtmp = open(tmpfile)
contents = readtmp.read()
readtmp.close()
os.unlink(tmpfile)
return contents
def normalize_error(cls, e):
if 'callback_get_login required' in six.text_type(e):
return AuthenticationError(
msg='Authentication failed when talking to the Subversion '
'repository')
else:
return SCMError(e)
def _do_on_path(self, cb, path, revision=HEAD):
if not path:
raise FileNotFoundError(path, revision)
try:
normpath = self.normalize_path(path)
# SVN expects to have URLs escaped. Take care to only
# escape the path part of the URL.
if self.client.is_url(normpath):
pathtuple = urlsplit(normpath)
path = pathtuple[2]
if isinstance(path, six.text_type):
path = path.encode('utf-8', 'ignore')
normpath = urlunsplit(
(pathtuple[0], pathtuple[1], quote(path), '', ''))
normrev = self._normalize_revision(revision)
return cb(normpath, normrev)
except ClientError as e:
exc = bytes(e).decode('utf-8')
if 'File not found' in exc or 'path not found' in exc:
raise FileNotFoundError(path, revision, detail=exc)
elif 'callback_ssl_server_trust_prompt required' in exc:
raise SCMError(
_('HTTPS certificate not accepted. Please ensure that '
'the proper certificate exists in %s '
'for the user that reviewboard is running as.') %
os.path.join(self.config_dir, 'auth'))
else:
raise SVNTool.normalize_error(e)
def get_file(self, path, revision=HEAD):
if revision == PRE_CREATION:
return ''
if revision == HEAD:
file = path
else:
file = '%s#%s' % (path, revision)
cmdline = ['p4', '-p', self.p4.port]
if self.p4.user:
cmdline.extend(['-u', self.p4.user])
if self.p4.password:
cmdline.extend(['-P', self.p4.password])
cmdline.extend(['print', '-q', file])
p = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(res, errdata) = p.communicate()
failure = p.poll()
if failure:
error = errdata.splitlines()
# The command-line output is the same as the contents of a P4Error
# except they're prefixed with a line that says "Perforce client
# error:", and the lines of the error are indented with tabs.
if error[0].startswith("Perforce client error:"):
error = error[1:]
raise SCMError('\n'.join(line.lstrip("\t") for line in error))
else:
return res
def get_change(self, revision):
"""Get an individual change.
This returns a Commit object containing the details of the commit.
"""
revision = int(revision)
commits = self.client.get_log('/', start=revision, limit=2)
commit = commits[0]
message = commit['message'].decode('utf-8', 'replace')
author_name = commit['author'].decode('utf-8', 'replace')
date = commit['date'].isoformat()
if len(commits) > 1:
base_revision = commits[1]['revision']
else:
base_revision = 0
try:
diff = self.client.diff(base_revision, revision)
except Exception as e:
raise SCMError(e)
commit = Commit(author_name, six.text_type(revision), date, message,
six.text_type(base_revision))
commit.diff = diff
return commit
def get_commits(self, repository, branch=None, start=None):
if start is not None:
url = (
'%s/repos/%s/branches/%s/commits?start=%s' %
(self.account.hosting_url,
repository.extra_data['rbgateway_repo_name'], branch, start))
else:
url = ('%s/repos/%s/branches/%s/commits' %
(self.account.hosting_url,
repository.extra_data['rbgateway_repo_name'], branch))
try:
data, headers = self._api_get(url)
commits = json.loads(data)
results = []
for commit in commits:
results.append(
Commit(commit['author'], commit['id'], commit['date'],
commit['message'], commit['parent_id']))
return results
except Exception as e:
logging.warning('Failed to fetch commits from %s: %s',
url,
e,
exc_info=1)
raise SCMError(six.text_type(e))
def get_file(self, path, revision=HEAD):
if not path:
raise FileNotFoundError(path, revision)
try:
normpath = self.__normalize_path(path)
# SVN expects to have URLs escaped. Take care to only
# escape the path part of the URL.
if self.client.is_url(normpath):
pathtuple = urlparse.urlsplit(normpath)
path = pathtuple[2]
if isinstance(path, unicode):
path = path.encode('utf-8', 'ignore')
normpath = urlparse.urlunsplit((pathtuple[0],
pathtuple[1],
urllib.quote(path),
'',''))
normrev = self.__normalize_revision(revision)
data = self.client.cat(normpath, normrev)
# Find out if this file has any keyword expansion set.
# If it does, collapse these keywords. This is because SVN
# will return the file expanded to us, which would break patching.
keywords = self.client.propget("svn:keywords", normpath, normrev,
recurse=True)
if normpath in keywords:
data = self.collapse_keywords(data, keywords[normpath])
return data
except ClientError, e:
stre = str(e)
if 'File not found' in stre or 'path not found' in stre:
raise FileNotFoundError(path, revision, str(e))
elif 'callback_ssl_server_trust_prompt required' in stre:
raise SCMError(
'HTTPS certificate not accepted. Please ensure that '
'the proper certificate exists in %s '
'for the user that reviewboard is running as.'
% os.path.join(self.config_dir, 'auth'))
elif 'callback_get_login required' in stre:
raise SCMError('Login to the SCM server failed.')
else:
raise SCMError(e)
def _convert_p4exception_to_scmexception(e):
error = str(e)
if 'Perforce password' in error or 'Password must be set' in error:
raise AuthenticationError(msg=error)
elif 'check $P4PORT' in error:
raise RepositoryNotFoundError
else:
raise SCMError(error)
def __init__(self,
path,
raw_file_url=None,
username=None,
password=None,
encoding='',
local_site_name=None):
super(GitClient, self).__init__(self._normalize_git_url(path),
username=username,
password=password)
if not is_exe_in_path('git'):
# This is technically not the right kind of error, but it's the
# pattern we use with all the other tools.
raise ImportError
self.raw_file_url = raw_file_url
self.encoding = encoding
self.local_site_name = local_site_name
self.git_dir = None
url_parts = urllib_urlparse(self.path)
if url_parts[0] == 'file':
if platform.system() == "Windows":
# Windows requires drive letter (e.g. C:/)
self.git_dir = url_parts[1] + url_parts[2]
else:
self.git_dir = url_parts[2]
p = self._run_git([
'--git-dir=%s' % self.git_dir, 'config',
'core.repositoryformatversion'
])
failure = p.wait()
if failure:
# See if we have a permissions error
if not os.access(self.git_dir, os.R_OK):
raise SCMError(
_("Permission denied accessing the local "
"Git repository '%s'") % self.git_dir)
else:
raise SCMError(
_('Unable to retrieve information from '
'local Git repository'))
def get_branches(self, repository):
"""Return the branches for the repository.
Args:
repository (reviewboard.scmtools.models.Repository):
The repository to fetch branches for.
Returns:
list of reviewboard.scmtools.core.Branch:
The branches returned from the Review Board Gateway API.
Raises:
SCMError:
The branches could not be retrieved from Review Board Gateway.
"""
url = ('%s/repos/%s/branches' %
(self.account.hosting_url,
repository.extra_data['rbgateway_repo_name']))
tool_name = repository.scmtool_class.name
if tool_name == 'Git':
default_branch = 'master'
elif tool_name == 'Mercurial':
default_branch = 'default'
else:
raise SCMError('Review Board Gateway does not support %s',
tool_name)
try:
data, headers = self._api_get(url)
branches = json.loads(data)
results = []
for branch in branches:
results.append(Branch(
id=branch['name'],
commit=branch['id'],
default=(branch['name'] == default_branch)
))
return results
except Exception as e:
logger.exception('Failed to get branches from %s: %s', url, e)
raise SCMError(six.text_type(e))
def get_changeset(self, changesetid):
logging.debug('Plastic: get_changeset %s' % (changesetid))
changesetdata = self.client.get_changeset(changesetid)
logging.debug('Plastic: changesetdata %s' % (changesetdata))
# Changeset data is in the form of multiple lines of:
# <changesetid> <user> <revid> <file spec>
#
# We assume the user and comment will be the same for each item, so
# read it out of the first.
#
changeset = ChangeSet()
changeset.changenum = changesetid
split = changesetdata.split('\n')
m = self.CS_RE.match(split[0])
revid = m.group("revid")
changeset.username = m.group("user")
changeset.summary = self.client.get_changeset_comment(
changesetid, revid)
logging.debug('Plastic: changeset user %s summary %s' %
(changeset.username, changeset.summary))
for line in split:
if line:
m = self.CS_RE.match(line)
if not m:
logging.debug('Plastic: bad re %s failed to match %s' %
(self.CS_RE, line))
raise SCMError("Error looking up changeset")
if m.group("csid") != str(changesetid):
logging.debug('Plastic: csid %s != %s' %
(m.group("csid"), changesetid))
raise SCMError(
"The server returned a changeset ID that was not requested"
)
logging.debug('Plastic: adding file %s' % (m.group("file")))
changeset.files += m.group("file")
return changeset
def parse_diff_revision(self, file_str, revision_str):
if revision_str == "PRE-CREATION":
return file_str, PRE_CREATION
m = self.rev_re.match(revision_str)
if not m:
raise SCMError("Unable to parse diff revision header '%s'" %
revision_str)
return file_str, m.group(1)
def api_get_issue(self, repo_api_url, issue_id):
url = self._build_api_url(repo_api_url, 'issues/%s' % issue_id)
try:
return self.api_get(url)
except Exception as e:
logging.warning('GitHub: Failed to fetch issue from %s: %s',
url, e, exc_info=1)
raise SCMError(six.text_type(e))
def api_get_issue(self, repo_api_url, issue_id):
url = '%s/issues/%s' % (repo_api_url, issue_id)
try:
return self.http_get(url).json
except Exception as e:
logger.warning('GitHub: Failed to fetch issue from %s: %s',
url, e, exc_info=1)
raise SCMError(six.text_type(e))
def _get_vobs_uuid(self, vobstag):
cmdline = ["cleartool", "lsvob", "-long", vobstag]
p = subprocess.Popen(cmdline,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=self.repopath)
(res, error) = p.communicate()
failure = p.poll()
if failure:
raise SCMError(error)
for line in res.splitlines(True):
if line.startswith('Vob family uuid:'):
return line.split(' ')[-1].rstrip()
raise SCMError("Can't find familly uuid for vob: %s" % vobstag)
def _convert_p4exception_to_scmexception(e):
error = str(e)
if 'Perforce password' in error or 'Password must be set' in error:
raise AuthenticationError(msg=error)
elif 'SSL library must be at least version' in error:
raise SCMError('The specified Perforce port includes ssl:, but '
'the p4python library was built without SSL '
'support or the system library path is incorrect. ')
elif ('check $P4PORT' in error
or (error.startswith('[P4.connect()] TCP connect to')
and 'failed.' in error)):
raise RepositoryNotFoundError
elif "To allow connection use the 'p4 trust' command" in error:
fingerprint = error.split('\\n')[3]
raise UnverifiedCertificateError(
Certificate(fingerprint=fingerprint))
else:
raise SCMError(error)
def api_get_heads(self, repo_api_url):
url = self._build_api_url(repo_api_url, 'git/refs/heads')
try:
rsp = self.api_get(url)
return [ref for ref in rsp if ref['ref'].startswith('refs/heads/')]
except Exception as e:
logging.warning('Failed to fetch commits from %s: %s',
url, e, exc_info=1)
raise SCMError(six.text_type(e))
def api_get_heads(self, repo_api_url):
url = '%s/git/refs/heads' % repo_api_url
try:
rsp = self.http_get(url).json
return [ref for ref in rsp if ref['ref'].startswith('refs/heads/')]
except Exception as e:
logger.warning('Failed to fetch commits from %s: %s',
url, e, exc_info=1)
raise SCMError(six.text_type(e))
def __init__(self, path):
if not is_exe_in_path('mtn'):
# This is technically not the right kind of error, but it's the
# pattern we use with all the other tools.
raise ImportError
self.path = path
if not os.path.isfile(self.path):
raise SCMError("Repository %s does not exist" % path)
def check_repository(self):
# Running 'cvs version' and specifying a CVSROOT will bail out if said
# CVSROOT is invalid, which is perfect for us. This used to use
# 'cvs rls' which is maybe slightly more correct, but rls is only
# available in CVS 1.12+
p = SCMTool.popen(['cvs', '-f', '-d', self.cvsroot, 'version'],
self.local_site_name)
errmsg = six.text_type(p.stderr.read())
if p.wait() != 0:
raise SCMError(errmsg)
def parse_diff_revision(self, file_str, revision_str, *args, **kwargs):
# Some diffs have additional tabs between the parts of the file
# revisions
revision_str = revision_str.strip()
if self.working_copy_re.match(revision_str):
return file_str, HEAD
# "(revision )" is generated by a few weird tools (like IntelliJ). If
# in the +++ line of the diff, it means HEAD, and in the --- line, it
# means PRE_CREATION. Since the more important use case is parsing the
# source revision, we treat it as a new file. See bugs 1937 and 2632.
if revision_str == "(revision )":
return file_str, PRE_CREATION
# Binary diffs don't provide revision information, so we set a fake
# "(unknown)" in the SVNDiffParser. This will never actually appear
# in SVN diffs.
if revision_str == "(unknown)":
return file_str, UNKNOWN
m = self.revision_re.match(revision_str)
if not m:
raise SCMError("Unable to parse diff revision header '%s'" %
revision_str)
relocated_file = m.group(2)
revision = m.group(4)
# group(3) holds the revision string in braces, like '(revision 4)'
# group(4) only matches the revision number, which might by None when
# 'nonexistent' is given as the revision string
if revision in (None, '0'):
revision = PRE_CREATION
if relocated_file:
if not relocated_file.startswith("..."):
raise SCMError("Unable to parse SVN relocated path '%s'" %
relocated_file)
file_str = "%s/%s" % (relocated_file[4:], file_str)
return file_str, revision
def get_branches(self):
"""Returns a list of branches.
This assumes the standard layout in the repository.
"""
results = []
try:
root_dirents = self.client.list_dir('/')
except Exception as e:
raise SCMError(e)
if 'trunk' in root_dirents:
# Looks like the standard layout. Adds trunk and any branches.
trunk = root_dirents['trunk']
results.append(
self._create_branch_from_dirent('trunk', trunk, default=True))
if 'branches' in root_dirents:
try:
dirents = self.client.list_dir('branches')
results += [
self._create_branch_from_dirent(name, dirents[name])
for name in sorted(six.iterkeys(dirents))
]
except Exception as e:
raise SCMError(e)
else:
# If the repository doesn't use the standard layout, just use a
# listing of the root directory as the "branches". This probably
# corresponds to a list of projects instead of branches, but it
# will at least give people a useful result.
default = True
for name in sorted(six.iterkeys(root_dirents)):
results.append(
self._create_branch_from_dirent(name, root_dirents[name],
default))
default = False
return results
