def setup_basic_post_test(self, user, with_local_site, local_site_name,
post_valid_data):
if post_valid_data:
self.create_review_group(name='group1',
with_local_site=with_local_site)
self.create_review_group(name='group2',
with_local_site=with_local_site)
repo1 = self.create_repository(name='Test Repo 1',
with_local_site=with_local_site,
path='test-repo-1')
repo2 = self.create_repository(name='Test Repo 2',
with_local_site=with_local_site,
path='test-repo-2')
if with_local_site:
site = LocalSite.objects.get(name=local_site_name)
site.users.add(User.objects.get(username='******'))
site.users.add(User.objects.get(username='******'))
post_data = {
'name': 'my-default',
'file_regex': '.*',
'users': 'doc,dopey',
'groups': 'group1,group2',
'repositories': ','.join([six.text_type(repo1.pk),
six.text_type(repo2.pk)]),
}
else:
post_data = {}
return (get_default_reviewer_list_url(local_site_name),
default_reviewer_item_mimetype,
post_data,
[local_site_name])
def get_branches(self):
"""Returns a list of branches.
This assumes the standard layout in the repository."""
results = []
trunk, unused = self.client.list(self.__normalize_path('trunk'),
dirent_fields=SVN_DIRENT_CREATED_REV,
recurse=False)[0]
results.append(
Branch('trunk', six.text_type(trunk['created_rev'].number), True))
try:
branches = self.client.list(
self.__normalize_path('branches'),
dirent_fields=SVN_DIRENT_CREATED_REV)[1:]
for branch, unused in branches:
results.append(Branch(
branch['path'].split('/')[-1],
six.text_type(branch['created_rev'].number)))
except ClientError:
# It's possible there aren't any branches. Ignore errors for this
# part.
pass
return results
def make_cache_key(self):
"""Creates and returns a cache key representing the diff to render."""
filediff = self.diff_file['filediff']
key = '%s-%s-%s-' % (self.template_name, self.diff_file['index'],
filediff.diffset.revision)
if self.diff_file['force_interdiff']:
interfilediff = self.diff_file['interfilediff']
key += 'interdiff-%s-' % filediff.pk
if interfilediff:
key += six.text_type(interfilediff.pk)
else:
key += 'none'
else:
key += six.text_type(filediff.pk)
if self.chunk_index is not None:
key += '-chunk-%s' % self.chunk_index
if self.collapse_all:
key += '-collapsed'
if self.highlighting:
key += '-highlighting'
key += '-%s-%s' % (get_language(), settings.AJAX_SERIAL)
return key
def check_repository(cls,
path,
username=None,
password=None,
local_site_name=None):
"""
Performs checks on a repository to test its validity.
This should check if a repository exists and can be connected to.
The result is returned as an exception. The exception may contain extra
information, such as a human-readable description of the problem. If the
repository is valid and can be connected to, no exception will be
thrown.
"""
super(PerforceTool, cls).check_repository(path, username, password,
local_site_name)
# 'p4 info' will succeed even if the server requires ticket auth and we
# don't run 'p4 login' first. We therefore don't go through all the
# trouble of handling tickets here.
client = cls._create_client(six.text_type(path),
six.text_type(username),
six.text_type(password))
client.get_info()
def make_cache_key(self):
"""Creates and returns a cache key representing the diff to render."""
filediff = self.diff_file["filediff"]
key = "%s-%s-%s-" % (self.template_name, self.diff_file["index"], filediff.diffset.revision)
if self.diff_file["force_interdiff"]:
interfilediff = self.diff_file["interfilediff"]
key += "interdiff-%s-" % filediff.pk
if interfilediff:
key += six.text_type(interfilediff.pk)
else:
key += "none"
else:
key += six.text_type(filediff.pk)
if self.chunk_index is not None:
key += "-chunk-%s" % self.chunk_index
if self.collapse_all:
key += "-collapsed"
if self.highlighting:
key += "-highlighting"
key += "-%s-%s" % (get_language(), settings.AJAX_SERIAL)
return key
def index_review_request(self, writer, request):
if lucene_is_2x:
lucene_tokenized = lucene.Field.Index.TOKENIZED
lucene_un_tokenized = lucene.Field.Index.UN_TOKENIZED
elif lucene_is_3x:
lucene_tokenized = lucene.Field.Index.ANALYZED
lucene_un_tokenized = lucene.Field.Index.NOT_ANALYZED
else:
assert False
# There are several fields we want to make available to users.
# We index them individually, but also create a big hunk of text
# to use for the default field, so people can just type in a
# string and get results.
doc = lucene.Document()
doc.add(lucene.Field("id", six.text_type(request.id), lucene.Field.Store.YES, lucene.Field.Index.NO))
doc.add(lucene.Field("summary", request.summary, lucene.Field.Store.NO, lucene_tokenized))
if request.changenum:
doc.add(
lucene.Field("changenum", six.text_type(request.changenum), lucene.Field.Store.NO, lucene_tokenized)
)
# Remove commas, since lucene won't tokenize it right with them
bugs = " ".join(request.bugs_closed.split(","))
doc.add(lucene.Field("bug", bugs, lucene.Field.Store.NO, lucene_tokenized))
name = " ".join([request.submitter.username, request.submitter.get_full_name()])
doc.add(lucene.Field("author", name, lucene.Field.Store.NO, lucene_tokenized))
doc.add(lucene.Field("username", request.submitter.username, lucene.Field.Store.NO, lucene_un_tokenized))
# FIXME: index reviews
# FIXME: index dates
files = []
if request.diffset_history:
for diffset in request.diffset_history.diffsets.all():
for filediff in diffset.files.all():
if filediff.source_file:
files.append(filediff.source_file)
if filediff.dest_file:
files.append(filediff.dest_file)
aggregate_files = "\n".join(set(files))
# FIXME: this tokenization doesn't let people search for files
# in a really natural way. It'll split on '/' which handles the
# majority case, but it'd be nice to be able to drill down
# (main.cc, vmuiLinux/main.cc, and player/linux/main.cc)
doc.add(lucene.Field("file", aggregate_files, lucene.Field.Store.NO, lucene_tokenized))
text = "\n".join(
[
request.summary,
request.description,
six.text_type(request.changenum),
request.testing_done,
bugs,
name,
aggregate_files,
]
)
doc.add(lucene.Field("text", text, lucene.Field.Store.NO, lucene_tokenized))
writer.addDocument(doc)
def get_commits(self, start):
return [
Commit('user%d' % i, six.text_type(i),
'2013-01-01T%02d:00:00.0000000' % i,
'Commit %d' % i,
six.text_type(i - 1))
for i in range(int(start), 0, -1)
]
def __init__(self, repository):
SCMTool.__init__(self, repository)
self.client = self._create_client(
six.text_type(repository.mirror_path or repository.path),
six.text_type(repository.username),
six.text_type(repository.password),
six.text_type(repository.encoding),
repository.extra_data.get('use_ticket_auth', False))
def __init__(self, repository):
SCMTool.__init__(self, repository)
self.client = self._create_client(
six.text_type(repository.mirror_path or repository.path),
six.text_type(repository.username),
six.text_type(repository.password),
six.text_type(repository.encoding),
repository.extra_data.get('use_ticket_auth', False))
def __init__(self, repository):
SCMTool.__init__(self, repository)
credentials = repository.get_credentials()
self.client = self._create_client(
six.text_type(repository.mirror_path or repository.path),
six.text_type(credentials['username']),
six.text_type(credentials['password'] or ''),
six.text_type(repository.encoding),
repository.extra_data.get('use_ticket_auth', False))
def _cat_specific_file(self, filename, revision):
# Somehow CVS sometimes seems to write .cvsignore files to current
# working directory even though we force stdout with -p.
self.tempdir = tempfile.mkdtemp()
os.chdir(self.tempdir)
p = SCMTool.popen([
'cvs', '-f', '-d', self.cvsroot, 'checkout', '-r',
six.text_type(revision), '-p', filename
], self.local_site_name)
contents = p.stdout.read()
errmsg = six.text_type(p.stderr.read())
failure = p.wait()
# Unfortunately, CVS is not consistent about exiting non-zero on
# errors. If the file is not found at all, then CVS will print an
# error message on stderr, but it doesn't set an exit code with
# pservers. If the file is found but an invalid revision is requested,
# then cvs exits zero and nothing is printed at all. (!)
#
# But, when it is successful, cvs will print a header on stderr like
# so:
#
# ===================================================================
# Checking out foobar
# RCS: /path/to/repo/foobar,v
# VERS: 1.1
# ***************
# So, if nothing is in errmsg, or errmsg has a specific recognized
# message, call it FileNotFound.
if (not errmsg or errmsg.startswith('cvs checkout: cannot find module')
or errmsg.startswith('cvs checkout: could not read RCS file')):
self.cleanup()
raise FileNotFoundError(filename, revision)
# Otherwise, if there's an exit code, or errmsg doesn't look like
# successful header, then call it a generic SCMError.
#
# If the .cvspass file doesn't exist, CVS will return an error message
# stating this. This is safe to ignore.
if ((failure and not errmsg.startswith('==========')) and
not '.cvspass does not exist - creating new file' in errmsg):
self.cleanup()
raise SCMError(errmsg)
self.cleanup()
return contents
def _cat_specific_file(self, filename, revision):
# Somehow CVS sometimes seems to write .cvsignore files to current
# working directory even though we force stdout with -p.
self.tempdir = tempfile.mkdtemp()
os.chdir(self.tempdir)
p = SCMTool.popen(['cvs', '-f', '-d', self.cvsroot, 'checkout',
'-r', six.text_type(revision), '-p', filename],
self.local_site_name)
contents = p.stdout.read()
errmsg = six.text_type(p.stderr.read())
failure = p.wait()
# Unfortunately, CVS is not consistent about exiting non-zero on
# errors. If the file is not found at all, then CVS will print an
# error message on stderr, but it doesn't set an exit code with
# pservers. If the file is found but an invalid revision is requested,
# then cvs exits zero and nothing is printed at all. (!)
#
# But, when it is successful, cvs will print a header on stderr like
# so:
#
# ===================================================================
# Checking out foobar
# RCS: /path/to/repo/foobar,v
# VERS: 1.1
# ***************
# So, if nothing is in errmsg, or errmsg has a specific recognized
# message, call it FileNotFound.
if (not errmsg or
errmsg.startswith('cvs checkout: cannot find module') or
errmsg.startswith('cvs checkout: could not read RCS file')):
self.cleanup()
raise FileNotFoundError(filename, revision)
# Otherwise, if there's an exit code, or errmsg doesn't look like
# successful header, then call it a generic SCMError.
#
# If the .cvspass file doesn't exist, CVS will return an error message
# stating this. This is safe to ignore.
if ((failure and not errmsg.startswith('==========')) and
not '.cvspass does not exist - creating new file' in errmsg):
self.cleanup()
raise SCMError(errmsg)
self.cleanup()
return contents
def get_change(self, revision):
"""Get an individual change.
This returns a tuple with the commit message and the diff contents.
"""
cache_key = self.repository.get_commit_cache_key(revision)
revision = int(revision)
head_revision = Revision(opt_revision_kind.number, revision)
commit = cache.get(cache_key)
if commit:
message = commit.message
author_name = commit.author_name
date = commit.date
base_revision = Revision(opt_revision_kind.number, commit.parent)
else:
commits = self.client.log(
self.repopath,
revision_start=head_revision,
limit=2)
commit = commits[0]
message = commit['message']
author_name = commit['author']
date = datetime.datetime.utcfromtimestamp(commit['date']).\
isoformat()
try:
commit = commits[1]
base_revision = commit['revision']
except IndexError:
base_revision = Revision(opt_revision_kind.number, 0)
tmpdir = mkdtemp(prefix='reviewboard-svn.')
diff = self.client.diff(
tmpdir,
self.repopath,
revision1=base_revision,
revision2=head_revision,
diff_options=['-u'])
rmtree(tmpdir)
commit = Commit(author_name, six.text_type(head_revision.number), date,
message, six.text_type(base_revision.number))
commit.diff = diff
return commit
def check_repository(cls, path, username=None, password=None,
local_site_name=None):
"""
Performs checks on a repository to test its validity.
This should check if a repository exists and can be connected to.
This will also check if the repository requires an HTTPS certificate.
The result is returned as an exception. The exception may contain
extra information, such as a human-readable description of the problem.
If the repository is valid and can be connected to, no exception
will be thrown.
"""
if sshutils.is_ssh_uri(path):
username, hostname = SCMTool.get_auth_from_uri(path, username)
logging.debug(
"%s: Attempting ssh connection with host: %s, username: %s"
% (cls.__name__, hostname, username))
try:
sshutils.check_host(hostname, username, password,
local_site_name)
except SSHAuthenticationError as e:
# Represent an SSHAuthenticationError as a standard
# AuthenticationError.
raise AuthenticationError(e.allowed_types, six.text_type(e),
e.user_key)
except:
# Re-raise anything else
raise
def get_objects(modelName, timestampField, dateField):
"""Perform timestamp based queries.
This method receives a dynamic model name and performs a filter
query. Later the results are grouped by day and prepared for the
charting library.
"""
args = '%s__range' % timestampField
q = modelName.objects.filter(**{
args: (range_start, range_end)
})
q = q.extra({timestampField: dateField})
q = q.values(timestampField)
q = q.annotate(created_count=Count('pk'))
q = q.order_by(timestampField)
data = []
for obj in q:
data.append([
time.mktime(time.strptime(
six.text_type(obj[timestampField]),
"%Y-%m-%d")) * 1000,
obj['created_count']
])
return data
def _cat_file(self, path, revision, option):
"""
Call git-cat-file(1) to get content or type information for a
repository object.
If called with just "commit", gets the content of a blob (or
raises an exception if the commit is not a blob).
Otherwise, "option" can be used to pass a switch to git-cat-file,
e.g. to test or existence or get the type of "commit".
"""
commit = self._resolve_head(revision, path)
p = self._run_git(
['--git-dir=%s' % self.git_dir, 'cat-file', option, commit])
contents = p.stdout.read()
errmsg = six.text_type(p.stderr.read())
failure = p.wait()
if failure:
if errmsg.startswith("fatal: Not a valid object name"):
raise FileNotFoundError(commit)
else:
raise SCMError(errmsg)
return contents
def reply_section(context, entry, comment, context_type, context_id):
"""
Renders a template for displaying a reply.
This takes the same parameters as :tag:`reply_list`. The template
rendered by this function, :template:`reviews/review_reply_section.html`,
is responsible for invoking :tag:`reply_list` and as such passes these
variables through. It does not make use of them itself.
"""
if comment != "":
if type(comment) is ScreenshotComment:
context_id += 's'
elif type(comment) is FileAttachmentComment:
context_id += 'f'
context_id += six.text_type(comment.id)
return {
'entry': entry,
'comment': comment,
'context_type': context_type,
'context_id': context_id,
'user': context.get('user', None),
'local_site_name': context.get('local_site_name'),
}
def with_counts(self, user):
queryset = self
if user and user.is_authenticated():
select_dict = {}
select_dict['new_review_count'] = """
SELECT COUNT(*)
FROM reviews_review, accounts_reviewrequestvisit
WHERE reviews_review.public
AND reviews_review.review_request_id =
reviews_reviewrequest.id
AND accounts_reviewrequestvisit.review_request_id =
reviews_reviewrequest.id
AND accounts_reviewrequestvisit.user_id = %(user_id)s
AND reviews_review.timestamp >
accounts_reviewrequestvisit.timestamp
AND reviews_review.user_id != %(user_id)s
""" % {
'user_id': six.text_type(user.id)
}
queryset = self.extra(select=select_dict)
return queryset
def get_file(self, path, revision):
logging.debug('Plastic: get_file %s rev %s' % (path, revision))
repo = "rep:%s@repserver:%s:%s" % (self.reponame, self.hostname,
self.port)
# Work around a plastic bug, where 'cm cat --file=blah' gets an
# extra newline, but plain 'cm cat' doesn't
fd, tmpfile = mkstemp()
os.close(fd)
p = subprocess.Popen(
['cm', 'cat', revision + '@' + repo, '--file=' + tmpfile],
stderr=subprocess.PIPE, stdout=subprocess.PIPE,
close_fds=(os.name != 'nt'))
errmsg = six.text_type(p.stderr.read())
failure = p.wait()
if failure:
if not errmsg:
errmsg = p.stdout.read()
raise SCMError(errmsg)
with open(tmpfile, 'rb') as readtmp:
contents = readtmp.read()
os.unlink(tmpfile)
return contents
def _cat_file(self, path, revision, option):
"""
Call git-cat-file(1) to get content or type information for a
repository object.
If called with just "commit", gets the content of a blob (or
raises an exception if the commit is not a blob).
Otherwise, "option" can be used to pass a switch to git-cat-file,
e.g. to test or existence or get the type of "commit".
"""
commit = self._resolve_head(revision, path)
p = self._run_git(['--git-dir=%s' % self.git_dir, 'cat-file',
option, commit])
contents = p.stdout.read()
errmsg = six.text_type(p.stderr.read())
failure = p.wait()
if failure:
if errmsg.startswith("fatal: Not a valid object name"):
raise FileNotFoundError(commit)
else:
raise SCMError(errmsg)
return contents
def _build_form_data(self, fields, files):
"""Encodes data for use in an HTTP POST."""
BOUNDARY = mimetools.choose_boundary()
content = ""
for key in fields:
content += "--" + BOUNDARY + "\r\n"
content += "Content-Disposition: form-data; name=\"%s\"\r\n" % key
content += "\r\n"
content += six.text_type(fields[key]) + "\r\n"
for key in files:
filename = files[key]['filename']
value = files[key]['content']
content += "--" + BOUNDARY + "\r\n"
content += "Content-Disposition: form-data; name=\"%s\"; " % key
content += "filename=\"%s\"\r\n" % filename
content += "\r\n"
content += value + "\r\n"
content += "--" + BOUNDARY + "--\r\n"
content += "\r\n"
content_type = "multipart/form-data; boundary=%s" % BOUNDARY
return content_type, content
def test_get_with_repository_and_commit_id(self):
"""Testing the GET review-requests/?repository=&commit_id= API
with changenum backwards-compatibility
"""
review_request = self.create_review_request(create_repository=True,
publish=True)
review_request.changenum = 1234
review_request.save()
self.assertEqual(review_request.commit_id, None)
commit_id = six.text_type(review_request.changenum)
rsp = self.apiGet(get_review_request_list_url(), {
'repository': review_request.repository.id,
'commit_id': review_request.commit,
},
expected_mimetype=review_request_list_mimetype)
self.assertEqual(rsp['stat'], 'ok')
self.assertEqual(len(rsp['review_requests']), 1)
self.assertEqual(rsp['review_requests'][0]['id'],
review_request.display_id)
self.assertEqual(rsp['review_requests'][0]['summary'],
review_request.summary)
self.assertEqual(rsp['review_requests'][0]['changenum'],
review_request.changenum)
self.assertEqual(rsp['review_requests'][0]['commit_id'], commit_id)
def augment_queryset(self, queryset):
user = self.datagrid.request.user
if user.is_anonymous():
return queryset
query_dict = {
'user_id': six.text_type(user.id),
}
return queryset.extra(select={
'mycomments_my_reviews': """
SELECT COUNT(1)
FROM reviews_review
WHERE reviews_review.user_id = %(user_id)s
AND reviews_review.review_request_id =
reviews_reviewrequest.id
""" % query_dict,
'mycomments_private_reviews': """
SELECT COUNT(1)
FROM reviews_review
WHERE reviews_review.user_id = %(user_id)s
AND reviews_review.review_request_id =
reviews_reviewrequest.id
AND NOT reviews_review.public
""" % query_dict,
'mycomments_shipit_reviews': """
SELECT COUNT(1)
FROM reviews_review
WHERE reviews_review.user_id = %(user_id)s
AND reviews_review.review_request_id =
reviews_reviewrequest.id
AND reviews_review.ship_it
""" % query_dict,
})
def check_repository(cls,
path,
username=None,
password=None,
local_site_name=None):
"""
Performs checks on a repository to test its validity.
This should check if a repository exists and can be connected to.
This will also check if the repository requires an HTTPS certificate.
The result is returned as an exception. The exception may contain
extra information, such as a human-readable description of the problem.
If the repository is valid and can be connected to, no exception
will be thrown.
"""
if sshutils.is_ssh_uri(path):
username, hostname = SCMTool.get_auth_from_uri(path, username)
logging.debug(
"%s: Attempting ssh connection with host: %s, username: %s" %
(cls.__name__, hostname, username))
try:
sshutils.check_host(hostname, username, password,
local_site_name)
except SSHAuthenticationError as e:
# Represent an SSHAuthenticationError as a standard
# AuthenticationError.
raise AuthenticationError(e.allowed_types, six.text_type(e),
e.user_key)
except:
# Re-raise anything else
raise
def _build_form_data(self, fields, files):
"""Encodes data for use in an HTTP POST."""
BOUNDARY = mimetools.choose_boundary()
content = ""
for key in fields:
content += "--" + BOUNDARY + "\r\n"
content += "Content-Disposition: form-data; name=\"%s\"\r\n" % key
content += "\r\n"
content += six.text_type(fields[key]) + "\r\n"
for key in files:
filename = files[key]['filename']
value = files[key]['content']
content += "--" + BOUNDARY + "\r\n"
content += "Content-Disposition: form-data; name=\"%s\"; " % key
content += "filename=\"%s\"\r\n" % filename
content += "\r\n"
content += value + "\r\n"
content += "--" + BOUNDARY + "--\r\n"
content += "\r\n"
content_type = "multipart/form-data; boundary=%s" % BOUNDARY
return content_type, content
def with_counts(self, user):
queryset = self
if user and user.is_authenticated():
select_dict = {}
select_dict['new_review_count'] = """
SELECT COUNT(*)
FROM reviews_review, accounts_reviewrequestvisit
WHERE reviews_review.public
AND reviews_review.review_request_id =
reviews_reviewrequest.id
AND accounts_reviewrequestvisit.review_request_id =
reviews_reviewrequest.id
AND accounts_reviewrequestvisit.user_id = %(user_id)s
AND reviews_review.timestamp >
accounts_reviewrequestvisit.timestamp
AND reviews_review.user_id != %(user_id)s
""" % {
'user_id': six.text_type(user.id)
}
queryset = self.extra(select=select_dict)
return queryset
def _resolve_head(self, revision, path):
if revision == HEAD:
if path == "":
raise SCMError("path must be supplied if revision is %s" % HEAD)
return "HEAD:%s" % path
else:
return six.text_type(revision)
def test_get_with_repository_and_commit_id(self):
"""Testing the GET review-requests/?repository=&commit_id= API
with changenum backwards-compatibility
"""
review_request = self.create_review_request(create_repository=True,
publish=True)
review_request.changenum = 1234
review_request.save()
self.assertEqual(review_request.commit_id, None)
commit_id = six.text_type(review_request.changenum)
rsp = self.apiGet(get_review_request_list_url(), {
'repository': review_request.repository.id,
'commit_id': review_request.commit,
}, expected_mimetype=review_request_list_mimetype)
self.assertEqual(rsp['stat'], 'ok')
self.assertEqual(len(rsp['review_requests']), 1)
self.assertEqual(rsp['review_requests'][0]['id'],
review_request.display_id)
self.assertEqual(rsp['review_requests'][0]['summary'],
review_request.summary)
self.assertEqual(rsp['review_requests'][0]['changenum'],
review_request.changenum)
self.assertEqual(rsp['review_requests'][0]['commit_id'],
commit_id)
def reply_section(context, entry, comment, context_type, context_id):
"""
Renders a template for displaying a reply.
This takes the same parameters as :tag:`reply_list`. The template
rendered by this function, :template:`reviews/review_reply_section.html`,
is responsible for invoking :tag:`reply_list` and as such passes these
variables through. It does not make use of them itself.
"""
if comment != "":
if type(comment) is ScreenshotComment:
context_id += 's'
elif type(comment) is FileAttachmentComment:
context_id += 'f'
context_id += six.text_type(comment.id)
return {
'entry': entry,
'comment': comment,
'context_type': context_type,
'context_id': context_id,
'user': context.get('user', None),
'local_site_name': context.get('local_site_name'),
}
def setup_basic_put_test(self, user, with_local_site, local_site_name,
put_valid_data):
default_reviewer = DefaultReviewer.objects.create(
name='default1', file_regex='.*')
if with_local_site:
local_site = LocalSite.objects.get(name=local_site_name)
local_site.users.add(User.objects.get(username='******'))
local_site.users.add(User.objects.get(username='******'))
default_reviewer.local_site = local_site
default_reviewer.save()
default_reviewer.people.add(User.objects.get(username='******'))
default_reviewer.groups.add(
self.create_review_group(name='group1',
with_local_site=with_local_site))
repo1 = self.create_repository(with_local_site=with_local_site,
name='Test Repo 1',
path='test-repo-1')
default_reviewer.repository.add(repo1)
if put_valid_data:
self.create_review_group(name='group2',
with_local_site=with_local_site)
repo2 = self.create_repository(with_local_site=with_local_site,
name='Test Repo 2',
path='test-repo-2')
put_data = {
'name': 'New name',
'file_regex': '/foo/',
'users': 'doc,dopey',
'groups': 'group1,group2',
'repositories': ','.join([six.text_type(repo1.pk),
six.text_type(repo2.pk)]),
}
else:
put_data = {}
return (get_default_reviewer_item_url(default_reviewer.pk,
local_site_name),
default_reviewer_item_mimetype,
put_data,
default_reviewer,
[])
def _convert_p4exception_to_scmexception(e):
error = six.text_type(e)
if 'Perforce password' in error or 'Password must be set' in error:
raise AuthenticationError(msg=error)
elif 'check $P4PORT' in error:
raise RepositoryNotFoundError
else:
raise SCMError(error)
def _convert_p4exception_to_scmexception(e):
error = six.text_type(e)
if 'Perforce password' in error or 'Password must be set' in error:
raise AuthenticationError(msg=error)
elif 'check $P4PORT' in error:
raise RepositoryNotFoundError
else:
raise SCMError(error)
def _resolve_head(self, revision, path):
if revision == HEAD:
if path == "":
raise SCMError("path must be supplied if revision is %s" %
HEAD)
return "HEAD:%s" % path
else:
return six.text_type(revision)
def render_to_string(self, request, inline=True):
"""Renders the review UI to an HTML string.
This renders the review UI to a string for use in embedding into
either an existing page or a new page.
If inline is True, the rendered review UI will be embeddable into
an existing page.
If inline is False, it will be rendered for use as a full, standalone
page, compelte with Review Board chrome.
"""
self.request = request
last_activity_time, updated_object = \
self.review_request.get_last_activity()
draft = self.review_request.get_draft(request.user)
review_request_details = draft or self.review_request
context = {
'caption': self.get_caption(draft),
'comments': self.get_comments(),
'draft': draft,
'last_activity_time': last_activity_time,
'review_request_details': review_request_details,
'review_request': self.review_request,
'review_ui': self,
'review_ui_uuid': six.text_type(uuid4()),
self.object_key: self.obj,
self.diff_object_key: self.diff_against_obj,
}
if inline:
context.update({
'base_template': 'reviews/ui/base_inline.html',
'review_ui_inline': True,
})
else:
if self.review_request.repository_id:
diffset_count = DiffSet.objects.filter(
history__pk=self.review_request.diffset_history_id).count()
else:
diffset_count = 0
context.update({
'base_template': 'reviews/ui/base.html',
'has_diffs': (draft and draft.diffset) or diffset_count > 0,
'review': self.review_request.get_pending_review(request.user),
'review_ui_inline': False,
})
return render_to_string(
self.template_name,
RequestContext(
request,
make_review_request_context(request, self.review_request,
context),
**self.get_extra_context(request)))
def __init__(self, path, revision, *args, **kwargs):
InvalidRevisionFormatError.__init__(
self,
path=path,
revision=revision,
detail=six.text_type(_('The SHA1 is too short. Make sure the diff '
'is generated with `git diff '
'--full-index`.')),
*args, **kwargs)
def get_file(self, path, revision=HEAD):
if not path or revision != HEAD:
raise FileNotFoundError(path, revision)
try:
with open(self.repopath + '/' + path, 'rb') as f:
return f.read()
except IOError as e:
raise FileNotFoundError(path, revision, detail=six.text_type(e))
def render_to_string(self, request, inline=True):
"""Renders the review UI to an HTML string.
This renders the review UI to a string for use in embedding into
either an existing page or a new page.
If inline is True, the rendered review UI will be embeddable into
an existing page.
If inline is False, it will be rendered for use as a full, standalone
page, compelte with Review Board chrome.
"""
self.request = request
last_activity_time, updated_object = \
self.review_request.get_last_activity()
draft = self.review_request.get_draft(request.user)
review_request_details = draft or self.review_request
context = {
'caption': self.get_caption(draft),
'comments': self.get_comments(),
'draft': draft,
'last_activity_time': last_activity_time,
'review_request_details': review_request_details,
'review_request': self.review_request,
'review_ui': self,
'review_ui_uuid': six.text_type(uuid4()),
self.object_key: self.obj,
self.diff_object_key: self.diff_against_obj,
}
if inline:
context['base_template'] = 'reviews/ui/base_inline.html'
else:
if self.review_request.repository_id:
diffset_count = DiffSet.objects.filter(
history__pk=self.review_request.diffset_history_id).count(
)
else:
diffset_count = 0
context.update({
'base_template':
'reviews/ui/base.html',
'has_diffs': (draft and draft.diffset) or diffset_count > 0,
'review':
self.review_request.get_pending_review(request.user),
})
return render_to_string(
self.template_name,
RequestContext(
request,
make_review_request_context(request, self.review_request,
context),
**self.get_extra_context(request)))
def __normalize_revision(self, revision):
if revision == HEAD:
r = Revision(opt_revision_kind.head)
elif revision == PRE_CREATION:
raise FileNotFoundError('', revision)
else:
r = Revision(opt_revision_kind.number, six.text_type(revision))
return r
def render_listview_to_response(self, request=None, render_context=None):
"""
Renders the listview to a response, preventing caching in the
process.
"""
response = HttpResponse(
six.text_type(self.render_listview(render_context)))
patch_cache_control(response, no_cache=True, no_store=True, max_age=0,
must_revalidate=True)
return response
def __init__(self, path, revision, *args, **kwargs):
InvalidRevisionFormatError.__init__(
self,
path=path,
revision=revision,
detail=six.text_type(
_('The SHA1 is too short. Make sure the diff '
'is generated with `git diff '
'--full-index`.')),
*args,
**kwargs)
def build_client(cls, username=None, password=None, local_site_name=None):
config_dir = os.path.join(os.path.expanduser('~'), '.subversion')
if local_site_name:
# LocalSites can have their own Subversion config, used for
# per-LocalSite SSH keys.
config_dir = cls._prepare_local_site_config_dir(local_site_name)
elif not os.path.exists(config_dir):
cls._create_subversion_dir(config_dir)
import pysvn
client = pysvn.Client(config_dir)
if username:
client.set_default_username(six.text_type(username))
if password:
client.set_default_password(six.text_type(password))
return config_dir, client
def check_host(netloc, username=None, password=None, namespace=None):
"""
Checks if we can connect to a host with a known key.
This will raise an exception if we cannot connect to the host. The
exception will be one of BadHostKeyError, UnknownHostKeyError, or
SCMError.
"""
from django.conf import settings
client = SSHClient(namespace=namespace)
client.set_missing_host_key_policy(RaiseUnknownHostKeyPolicy())
kwargs = {}
if ':' in netloc:
hostname, port = netloc.split(':')
port = int(port)
else:
hostname = netloc
port = SSH_PORT
# We normally want to notify on unknown host keys, but not when running
# unit tests.
if getattr(settings, 'RUNNING_TEST', False):
client.set_missing_host_key_policy(paramiko.WarningPolicy())
kwargs['allow_agent'] = False
try:
client.connect(hostname,
port,
username=username,
password=password,
pkey=client.get_user_key(),
**kwargs)
except paramiko.BadHostKeyException as e:
raise BadHostKeyError(e.hostname, e.key, e.expected_key)
except paramiko.AuthenticationException as e:
# Some AuthenticationException instances have allowed_types set,
# and some don't.
allowed_types = getattr(e, 'allowed_types', [])
if 'publickey' in allowed_types:
key = client.get_user_key()
else:
key = None
raise SSHAuthenticationError(allowed_types=allowed_types, user_key=key)
except paramiko.SSHException as e:
msg = six.text_type(e)
if msg == 'No authentication methods available':
raise SSHAuthenticationError
else:
raise SSHError(msg)
def cat_file(self, path, rev="tip"):
if rev == HEAD:
rev = "tip"
elif rev == PRE_CREATION:
rev = ""
try:
return self.repo.changectx(rev).filectx(path).data()
except Exception as e:
# LookupError moves from repo to revlog in hg v0.9.4, so we
# catch the more general Exception to avoid the dependency.
raise FileNotFoundError(path, rev, detail=six.text_type(e))
def create(self,
request,
username,
service_id,
password=None,
hosting_url=None,
local_site_name=None,
*args,
**kwargs):
local_site = self._get_local_site(local_site_name)
if not HostingServiceAccount.objects.can_create(
request.user, local_site):
return self._no_access_error(request.user)
# Validate the service.
service = get_hosting_service(service_id)
if not service:
return INVALID_FORM_DATA, {
'fields': {
'service': ['This is not a valid service name'],
}
}
if service.self_hosted and not hosting_url:
return INVALID_FORM_DATA, {
'fields': {
'hosting_url': ['This field is required'],
}
}
account = HostingServiceAccount(service_name=service_id,
username=username,
hosting_url=hosting_url,
local_site=local_site)
service = account.service
if service.needs_authorization:
try:
service.authorize(request, username, password, hosting_url,
local_site_name)
except AuthorizationError as e:
return HOSTINGSVC_AUTH_ERROR, {
'reason': six.text_type(e),
}
service.save()
return 201, {
self.item_result_key: account,
}
def check_repository(cls, path, username=None, password=None,
local_site_name=None):
"""
Performs checks on a repository to test its validity.
This should check if a repository exists and can be connected to.
The result is returned as an exception. The exception may contain extra
information, such as a human-readable description of the problem. If the
repository is valid and can be connected to, no exception will be
thrown.
"""
super(PerforceTool, cls).check_repository(path, username, password,
local_site_name)
# 'p4 info' will succeed even if the server requires ticket auth and we
# don't run 'p4 login' first. We therefore don't go through all the
# trouble of handling tickets here.
client = cls._create_client(six.text_type(path),
six.text_type(username),
six.text_type(password))
client.get_info()
def _do_on_path(self, cb, path, revision=HEAD):
if not path:
raise FileNotFoundError(path, revision)
try:
normpath = self.__normalize_path(path)
# SVN expects to have URLs escaped. Take care to only
# escape the path part of the URL.
if self.client.is_url(normpath):
pathtuple = urlsplit(normpath)
path = pathtuple[2]
if isinstance(path, six.text_type):
path = path.encode('utf-8', 'ignore')
normpath = urlunsplit((pathtuple[0],
pathtuple[1],
quote(path),
'', ''))
normrev = self.__normalize_revision(revision)
return cb(normpath, normrev)
except ClientError as e:
stre = six.text_type(e)
if 'File not found' in stre or 'path not found' in stre:
raise FileNotFoundError(path, revision,
detail=six.text_type(e))
elif 'callback_ssl_server_trust_prompt required' in stre:
raise SCMError(
_('HTTPS certificate not accepted. Please ensure that '
'the proper certificate exists in %s '
'for the user that reviewboard is running as.')
% os.path.join(self.config_dir, 'auth'))
elif 'callback_get_login required' in stre:
raise AuthenticationError(
msg=_('Login to the SCM server failed.'))
else:
raise SCMError(e)
def update(self, request, caption=None, thumbnail=None, *args, **kwargs):
"""Updates the file's data.
This allows updating the file in a draft. The caption, currently,
is the only thing that can be updated.
"""
try:
review_request = \
resources.review_request.get_object(request, *args, **kwargs)
except ObjectDoesNotExist:
return DOES_NOT_EXIST
if not review_request.is_mutable_by(request.user):
return PERMISSION_DENIED
try:
file = resources.file_attachment.get_object(
request, *args, **kwargs)
except ObjectDoesNotExist:
return DOES_NOT_EXIST
if caption is not None:
try:
resources.review_request_draft.prepare_draft(
request, review_request)
except PermissionDenied:
return self._no_access_error(request.user)
file.draft_caption = caption
file.save()
if thumbnail is not None:
try:
file.thumbnail = thumbnail
except Exception as e:
logging.error(
'Failed to store thumbnail for attachment %d: %s',
file.pk,
e,
request=request)
return INVALID_FORM_DATA, {
'fields': {
'thumbnail': [six.text_type(e)],
}
}
return 200, {
self.item_result_key: file,
}
def _revspec_from_revision(self, revision):
"""Returns a revspec based on the revision found in the diff.
In addition to the standard date format from "bzr diff", this
function supports the revid: syntax provided by the bzr diff-revid plugin.
"""
if revision == HEAD:
revspec = 'last:1'
elif revision.startswith('revid:'):
revspec = revision
else:
revspec = 'date:' + six.text_type(
self._revision_timestamp_to_local(revision))
return revspec
def _api_get(self, url):
try:
data, headers = self._http_get(url)
return json.loads(data)
except (URLError, HTTPError) as e:
data = e.read()
try:
rsp = json.loads(data)
except:
rsp = None
if rsp and 'message' in rsp:
raise Exception(rsp['message'])
else:
raise Exception(six.text_type(e))
def create(self, request, *args, **kwargs):
"""Creates a new screenshot from an uploaded file.
This accepts any standard image format (PNG, GIF, JPEG) and associates
it with a draft of a review request.
It is expected that the client will send the data as part of a
:mimetype:`multipart/form-data` mimetype. The screenshot's name
and content should be stored in the ``path`` field. A typical request
may look like::
-- SoMe BoUnDaRy
Content-Disposition: form-data; name=path; filename="foo.png"
<PNG content here>
-- SoMe BoUnDaRy --
"""
try:
review_request = \
resources.review_request.get_object(request, *args, **kwargs)
except ObjectDoesNotExist:
return DOES_NOT_EXIST
if not review_request.is_mutable_by(request.user):
return self._no_access_error(request.user)
form_data = request.POST.copy()
form = UploadScreenshotForm(form_data, request.FILES)
if not form.is_valid():
return INVALID_FORM_DATA, {
'fields': self._get_form_errors(form),
}
try:
screenshot = form.create(request.FILES['path'], review_request)
except ValueError as e:
return INVALID_FORM_DATA, {
'fields': {
'path': [six.text_type(e)],
},
}
return 201, {
self.item_result_key: screenshot,
}
def get_related_links(self, obj=None, request=None, *args, **kwargs):
links = {}
if request and request.user.is_authenticated():
user_resource = get_resource_for_object(request.user)
href = user_resource.get_href(request.user, request,
*args, **kwargs)
links['user'] = {
'method': 'GET',
'href': href,
'title': six.text_type(request.user),
'resource': user_resource,
'list-resource': False,
}
return links
def make_cache_key(self):
"""Creates a cache key for any generated chunks."""
key = 'diff-sidebyside-'
if self.enable_syntax_highlighting:
key += 'hl-'
if not self.force_interdiff:
key += six.text_type(self.filediff.pk)
elif self.interfilediff:
key += 'interdiff-%s-%s' % (self.filediff.pk,
self.interfilediff.pk)
else:
key += 'interdiff-%s-none' % self.filediff.pk
key += '-%s' % get_language()
return key
def test_put_with_repository_invalid_site(self):
"""Testing the PUT default-reviewers/<id>/ API
with repository and invalid site
"""
repository = self.create_repository(with_local_site=True)
default_reviewer = DefaultReviewer.objects.create(
name='default1', file_regex='.*')
self._login_user(admin=True)
rsp = self.apiPut(
get_default_reviewer_item_url(default_reviewer.pk),
{'repositories': six.text_type(repository.pk)},
expected_status=400)
self.assertTrue('fields' in rsp)
self.assertTrue('repositories' in rsp['fields'])
def test_post_with_repository_invalid_site(self):
"""Testing the POST default-reviewers/ API
with repository and invalid site
"""
repository = self.create_repository(with_local_site=True)
self._login_user(admin=True)
rsp = self.apiPost(
get_default_reviewer_list_url(),
{
'name': 'default1',
'file_regex': '.*',
'repositories': six.text_type(repository.pk),
},
expected_status=400)
self.assertTrue('fields' in rsp)
self.assertTrue('repositories' in rsp['fields'])
def get_changeset(self, changesetid, allow_empty=False):
logging.debug('Plastic: get_changeset %s' % (changesetid))
changesetdata = self.client.get_changeset(changesetid)
logging.debug('Plastic: changesetdata %s' % (changesetdata))
# Changeset data is in the form of multiple lines of:
# <changesetid> <user> <revid> <file spec>
#
# We assume the user and comment will be the same for each item, so
# read it out of the first.
#
changeset = ChangeSet()
changeset.changenum = changesetid
split = changesetdata.split('\n')
m = self.CS_RE.match(split[0])
revid = m.group("revid")
changeset.username = m.group("user")
changeset.summary = self.client.get_changeset_comment(changesetid,
revid)
logging.debug('Plastic: changeset user %s summary %s' %
(changeset.username, changeset.summary))
for line in split:
if line:
m = self.CS_RE.match(line)
if not m:
logging.debug('Plastic: bad re %s failed to match %s' %
(self.CS_RE, line))
raise SCMError("Error looking up changeset")
if m.group("csid") != six.text_type(changesetid):
logging.debug('Plastic: csid %s != %s' % (m.group("csid"),
changesetid))
raise SCMError("The server returned a changeset ID that was not requested")
logging.debug('Plastic: adding file %s' % (m.group("file")))
changeset.files += m.group("file")
return changeset