class CommitData(models.Model):
"""A non user modifiable store for data about commits.
This acts as a simple store for data about commits which
a review request owner may not modify themselves. A simple
JSONField is used to mimic the extra_data present on many
built-in Review Board models to make transitioning as quick
as possible. Eventually we'll want to replace this with
something more sensible.
We use a second JSONField to represent extra_data for the
ReviewRequestDraft rather than deal with two tables.
"""
review_request = models.OneToOneField(
ReviewRequest,
primary_key=True)
extra_data = JSONField(
default='{}',
blank=False)
draft_extra_data = JSONField(
default='{}',
blank=False)
def get_for(self, review_request_details, key, default=None):
"""Return the extra data value for either a review request or a draft.
This helper method allows a caller to fetch a key from either
extra_data or draft_extra_data when they want the data associated
with a ReviewRequest or a ReviewRequestDraft respectively.
"""
if isinstance(review_request_details, ReviewRequestDraft):
return self.draft_extra_data.get(key, default)
return self.extra_data.get(key, default)
def set_for(self, review_request_details, key, value):
"""Set the extra data value for either a review request or a draft.
This helper method allows a caller to set a key on either
extra_data or draft_extra_data when they want the data associated
with a ReviewRequest or a ReviewRequestDraft respectively.
"""
if isinstance(review_request_details, ReviewRequestDraft):
self.draft_extra_data[key] = value
else:
self.extra_data[key] = value
class Meta:
app_label = 'mozreview'
class ToolExecution(models.Model):
"""Status of a tool execution.
This represents the request for and status of a tool's execution.
"""
QUEUED = 'Q'
RUNNING = 'R'
SUCCEEDED = 'S'
FAILED = 'F'
TIMED_OUT = 'T'
STATUSES = (
(QUEUED, _('Queued')),
(RUNNING, _('Running')),
(SUCCEEDED, _('Succeeded')),
(FAILED, _('Failed')),
(TIMED_OUT, _('Timed-out')),
)
profile = models.ForeignKey(Profile)
review_request_id = models.IntegerField(null=True)
diff_revision = models.IntegerField(null=True)
last_updated = ModificationTimestampField(_("last updated"))
status = models.CharField(max_length=1, choices=STATUSES, blank=True)
# Review Information
result = JSONField()
def __unicode__(self):
return '%s (%s)' % (self.profile.name, self.status)
class Meta:
ordering = ['-last_updated', 'review_request_id', 'diff_revision']
class DiffSetHistory(models.Model):
"""A collection of diffsets.
This gives us a way to store and keep track of multiple revisions of
diffsets belonging to an object.
"""
name = models.CharField(_('name'), max_length=256)
timestamp = models.DateTimeField(_("timestamp"), default=timezone.now)
last_diff_updated = models.DateTimeField(_("last updated"),
blank=True,
null=True,
default=None)
extra_data = JSONField(null=True)
def __str__(self):
"""Return a human-readable representation of the model.
Returns:
unicode:
A human-readable representation of the model.
"""
return 'Diff Set History (%s revisions)' % self.diffsets.count()
class Meta:
app_label = 'diffviewer'
db_table = 'diffviewer_diffsethistory'
verbose_name = _('Diff Set History')
verbose_name_plural = _('Diff Set Histories')
class Profile(models.Model):
"""A configuration of a tool.
Each Profile may have distinct settings for the associated tool and rules
about who may run the tool manually.
"""
tool = models.ForeignKey(Tool)
name = models.CharField(max_length=128, blank=False)
description = models.CharField(max_length=512, default="", blank=True)
allow_manual = models.BooleanField(default=False)
allow_manual_submitter = models.BooleanField(default=False)
allow_manual_group = models.BooleanField(default=False)
ship_it = models.BooleanField(default=False,
help_text=_("Ship it! If no issues raised."))
open_issues = models.BooleanField(default=False)
comment_unmodified = models.BooleanField(
default=False, verbose_name=_("Comment on unmodified code"))
tool_settings = JSONField()
local_site = models.ForeignKey(LocalSite,
blank=True,
null=True,
related_name='reviewbot_profiles')
def __unicode__(self):
return self.name
class Application(AbstractApplication):
"""An OAuth2 application.
This model is specialized so that it can be limited to a
:py:class:`~reviewboard.site.models.LocalSite`.
"""
local_site = models.ForeignKey(
verbose_name=_('Local Site'),
to=LocalSite,
related_name='oauth_applications',
blank=True,
null=True,
help_text=_('An optional LocalSite to limit this application to.'),
)
extra_data = JSONField(
_('Extra Data'),
null=True,
default=dict,
)
class Meta:
db_table = 'reviewboard_oauth_application'
verbose_name = _('OAuth Application')
verbose_name_plural = _('OAuth Applications')
class LocalSiteProfile(models.Model):
"""User profile information specific to a LocalSite."""
user = models.ForeignKey(User, related_name='site_profiles')
profile = models.ForeignKey(Profile, related_name='site_profiles')
local_site = models.ForeignKey(LocalSite, null=True, blank=True,
related_name='site_profiles')
# A dictionary of permission that the user has granted. Any permission
# missing is considered to be False.
permissions = JSONField(null=True)
# Counts for quickly knowing how many review requests are incoming
# (both directly and total), outgoing (pending and total ever made),
# and starred (public).
direct_incoming_request_count = CounterField(
_('direct incoming review request count'),
initializer=lambda p: (
ReviewRequest.objects.to_user_directly(
p.user, local_site=p.local_site).count()
if p.user_id else 0))
total_incoming_request_count = CounterField(
_('total incoming review request count'),
initializer=lambda p: (
ReviewRequest.objects.to_user(
p.user, local_site=p.local_site).count()
if p.user_id else 0))
pending_outgoing_request_count = CounterField(
_('pending outgoing review request count'),
initializer=lambda p: (
ReviewRequest.objects.from_user(
p.user, p.user, local_site=p.local_site).count()
if p.user_id else 0))
total_outgoing_request_count = CounterField(
_('total outgoing review request count'),
initializer=lambda p: (
ReviewRequest.objects.from_user(
p.user, p.user, None, local_site=p.local_site).count()
if p.user_id else 0))
starred_public_request_count = CounterField(
_('starred public review request count'),
initializer=lambda p: (
p.profile.starred_review_requests.public(
user=None, local_site=p.local_site).count()
if p.pk else 0))
def __str__(self):
"""Return a string used for the admin site listing."""
return '%s (%s)' % (self.user.username, self.local_site)
class Meta:
db_table = 'accounts_localsiteprofile'
unique_together = (('user', 'local_site'),
('profile', 'local_site'))
verbose_name = _('Local Site Profile')
verbose_name_plural = _('Local Site Profiles')
class LocalSite(models.Model):
"""
A division within a Review Board installation.
This allows the creation of independent, isolated divisions within a given
server. Users can be designated as members of a LocalSite, and optionally
as admins (which allows them to manipulate the repositories, groups and
users in the site).
Pretty much every other model in this module can all be assigned to a
single LocalSite, at which point only members will be able to see or
manipulate these objects. Access control is performed at every level, and
consistency is enforced through a liberal sprinkling of assertions and unit
tests.
"""
name = models.SlugField(_('name'), max_length=32, blank=False, unique=True)
public = models.BooleanField(
default=False,
help_text=_('Allow people outside the team to access and post '
'review requests and reviews.'))
users = models.ManyToManyField(User, blank=True,
related_name='local_site')
admins = models.ManyToManyField(User, blank=True,
related_name='local_site_admins')
extra_data = JSONField(null=True)
def is_accessible_by(self, user):
"""Returns whether or not the user has access to this LocalSite.
This checks that the user is logged in, and that they're listed in the
'users' field.
"""
return (self.public or
(user.is_authenticated and
(user.is_staff or self.users.filter(pk=user.pk).exists())))
def is_mutable_by(self, user, perm='site.change_localsite'):
"""Returns whether or not a user can modify settings in a LocalSite.
This checks that the user is either staff with the proper permissions,
or that they're listed in the 'admins' field.
By default, this is checking whether the LocalSite itself can be
modified, but a different permission can be passed to check for
another object.
"""
return user.has_perm(perm) or self.admins.filter(pk=user.pk).exists()
def __str__(self):
return self.name
class Meta:
db_table = 'site_localsite'
verbose_name = _('Local Site')
verbose_name_plural = _('Local Sites')
def test_init_with_custom_encoder_class(self):
"""Testing JSONField initialization with custom encoder class"""
class MyEncoder(json.JSONEncoder):
def __init__(self, default_msg, **kwargs):
self.default_msg = default_msg
super(MyEncoder, self).__init__(**kwargs)
def default(self, o):
return self.default_msg
with warnings.catch_warnings(record=True) as w:
field = JSONField(encoder_cls=MyEncoder,
encoder_kwargs={
'default_msg': 'What even is this?',
})
self.assertEqual(field.dumps(MyEncoder), '"What even is this?"')
self.assertEqual(len(w), 0)
class ChecklistTemplate(models.Model):
"""A checklist template defines a collection of checklist items.
Each template can be imported into a checklist. Items are stored in JSON
format as a single array of checklist items.
"""
owner = models.ForeignKey(User)
title = models.CharField(max_length=255)
items = JSONField()
class FileDiffData(models.Model):
"""
Contains hash and base64 pairs.
These pairs are used to reduce diff database storage.
"""
binary_hash = models.CharField(_("hash"), max_length=40, primary_key=True)
binary = Base64Field(_("base64"))
objects = FileDiffDataManager()
extra_data = JSONField(null=True)
@property
def insert_count(self):
return self.extra_data.get('insert_count')
@insert_count.setter
def insert_count(self, value):
self.extra_data['insert_count'] = value
@property
def delete_count(self):
return self.extra_data.get('delete_count')
@delete_count.setter
def delete_count(self, value):
self.extra_data['delete_count'] = value
def recalculate_line_counts(self, tool):
"""Recalculates the insert_count and delete_count values.
This will attempt to re-parse the stored diff and fetch the
line counts through the parser.
"""
logging.debug('Recalculating insert/delete line counts on '
'FileDiffData %s' % self.pk)
try:
files = tool.get_parser(self.binary).parse()
if len(files) != 1:
raise DiffParserError('Got wrong number of files (%d)' %
len(files))
except DiffParserError as e:
logging.error(
'Failed to correctly parse stored diff data in '
'FileDiffData ID %s when trying to get '
'insert/delete line counts: %s', self.pk, e)
else:
file_info = files[0]
self.insert_count = file_info.insert_count
self.delete_count = file_info.delete_count
if self.pk:
self.save(update_fields=['extra_data'])
class HostingServiceAccount(models.Model):
service_name = models.CharField(max_length=128)
hosting_url = models.CharField(max_length=255, blank=True, null=True)
username = models.CharField(max_length=128)
data = JSONField()
visible = models.BooleanField(default=True)
local_site = models.ForeignKey(LocalSite,
related_name='hosting_service_accounts',
verbose_name=_('Local site'),
blank=True,
null=True)
objects = HostingServiceAccountManager()
def __str__(self):
return self.username
@property
def service(self):
if not hasattr(self, '_service'):
cls = get_hosting_service(self.service_name)
if cls:
self._service = cls(self)
else:
self._service = None
return self._service
@property
def is_authorized(self):
service = self.service
if service:
return service.is_authorized()
else:
return False
def is_accessible_by(self, user):
"""Returns whether or not the user has access to the account.
The account is accessible by the user if the user has access to the
local site.
"""
return not self.local_site or self.local_site.is_accessible_by(user)
def is_mutable_by(self, user):
"""Returns whether or not the user can modify or delete the account.
The acount is mutable by the user if the user is an administrator
with proper permissions or the account is part of a LocalSite and
the user has permissions to modify it.
"""
return user.has_perm('hostingsvcs.change_hostingserviceaccount',
self.local_site)
def test_init_with_custom_encoder_instance(self):
"""Testing JSONField initialization with deprecated custom encoder
instance
"""
class MyEncoder(json.JSONEncoder):
def default(self, o):
return 'What even is this?'
with warnings.catch_warnings(record=True) as w:
field = JSONField(encoder=MyEncoder())
self.assertEqual(field.dumps(MyEncoder), '"What even is this?"')
self.assertEqual(len(w), 1)
message = w[0].message
self.assertIsInstance(message, RemovedInDjblets20Warning)
self.assertEqual(six.text_type(message),
'The encoder argument to JSONField has been '
'replaced by the encoder_cls and encoder_kwargs '
'arguments. Support for encoder is deprecated.')
def test_init_with_custom_encoder_class(self):
"""Testing JSONField initialization with custom encoder class"""
class MyEncoder(json.JSONEncoder):
def __init__(self, default_msg, **kwargs):
self.default_msg = default_msg
super(MyEncoder, self).__init__(**kwargs)
def default(self, o):
return self.default_msg
with warnings.catch_warnings(record=True) as w:
field = JSONField(
encoder_cls=MyEncoder,
encoder_kwargs={
'default_msg': 'What even is this?',
})
self.assertEqual(field.dumps(MyEncoder), '"What even is this?"')
self.assertEqual(len(w), 0)
def test_init_with_custom_encoder_instance(self):
"""Testing JSONField initialization with deprecated custom encoder
instance
"""
class MyEncoder(json.JSONEncoder):
def default(self, o):
return 'What even is this?'
with warnings.catch_warnings(record=True) as w:
field = JSONField(encoder=MyEncoder())
self.assertEqual(field.dumps(MyEncoder), '"What even is this?"')
self.assertEqual(len(w), 1)
message = w[0].message
self.assertIsInstance(message, DeprecationWarning)
self.assertEqual(
six.text_type(message),
'The encoder argument to JSONField has been '
'replaced by the encoder_cls and encoder_kwargs '
'arguments. Support for encoder is deprecated.')
class LegacyFileDiffData(models.Model):
"""Deprecated, legacy class for base64-encoded diff data.
This is no longer populated, and exists solely to store legacy data
that has not been migrated to :py:class:`RawFileDiffData`.
"""
binary_hash = models.CharField(_("hash"), max_length=40, primary_key=True)
binary = Base64Field(_("base64"))
extra_data = JSONField(null=True)
class Meta:
db_table = 'diffviewer_filediffdata'
class RegisteredExtension(models.Model):
"""Extension registration info.
An extension that was both installed and enabled at least once. This
may contain settings for the extension.
This does not contain full information for the extension, such as the
author or description. That is provided by the Extension object itself.
"""
class_name = models.CharField(max_length=128, unique=True)
name = models.CharField(max_length=32)
enabled = models.BooleanField(default=False)
installed = models.BooleanField(default=False)
settings = JSONField()
def __str__(self):
return self.name
def get_extension_class(self):
"""Retrieves the python object for the extensions class."""
if not hasattr(self, '_extension_class'):
cls = None
try:
# Import the function here to avoid a mutual
# dependency.
from djblets.extensions.manager import get_extension_managers
for manager in get_extension_managers():
try:
cls = manager.get_installed_extension(self.class_name)
break
except InvalidExtensionError:
continue
except:
return None
self._extension_class = cls
return self._extension_class
extension_class = property(get_extension_class)
class Meta:
# Djblets 0.9+ sets an app label of "djblets_extensions" on
# Django 1.7+, which would affect the table name. We need to retain
# the old name for backwards-compatibility.
db_table = 'extensions_registeredextension'
class Tool(models.Model):
"""Information about a tool installed on a worker.
Each entry in the database will be unique for the values of `entry_point`
and `version`. Any backwards incompatible changes to a Tool will result
in a version bump, allowing multiple versions of a tool to work with a
Review Board instance.
"""
name = models.CharField(max_length=128, blank=False)
entry_point = models.CharField(max_length=128, blank=False)
version = models.CharField(max_length=128, blank=False)
description = models.CharField(max_length=512, default='', blank=True)
enabled = models.BooleanField(default=True)
in_last_update = models.BooleanField()
timeout = models.IntegerField(blank=True, null=True)
working_directory_required = models.BooleanField(default=False)
#: A JSON list describing the options a tool make take. Each entry is a
#: dictionary which may define the following fields:
#:
#: {
#: 'name': The name of the option
#: 'field_type': The django form field class for the option
#: 'default': The default value
#: 'field_options': An object containing fields to be passed to
#: the form class, e.g.:
#: {
#: 'label': A label for the field
#: 'help_text': Help text
#: 'required': If the field is required
#: }
#: }
tool_options = JSONField()
def __str__(self):
"""Return a string representation of the tool.
Returns:
unicode:
The text representation for this model.
"""
return '%s - v%s' % (self.name, self.version)
class Meta:
app_label = 'reviewbotext'
unique_together = ('entry_point', 'version')
class LegacyFileDiffData(models.Model):
"""Deprecated, legacy class for base64-encoded diff data.
This is no longer populated, and exists solely to store legacy data
that has not been migrated to :py:class:`RawFileDiffData`.
"""
binary_hash = models.CharField(_('hash'), max_length=40, primary_key=True)
binary = Base64Field(_('base64'))
extra_data = JSONField(null=True)
class Meta:
app_label = 'diffviewer'
db_table = 'diffviewer_filediffdata'
verbose_name = _('Legacy File Diff Data')
verbose_name_plural = _('Legacy File Diff Data Blobs')
class DiffSetHistory(models.Model):
"""
A collection of diffsets.
This gives us a way to store and keep track of multiple revisions of
diffsets belonging to an object.
"""
name = models.CharField(_('name'), max_length=256)
timestamp = models.DateTimeField(_("timestamp"), default=timezone.now)
last_diff_updated = models.DateTimeField(_("last updated"),
blank=True,
null=True,
default=None)
extra_data = JSONField(null=True)
def __str__(self):
return 'Diff Set History (%s revisions)' % self.diffsets.count()
class Meta:
verbose_name_plural = "Diff set histories"
class Tool(models.Model):
"""Information about a tool installed on a worker.
`tool_option` is a JSON list describing the options a tool may take. Each
entry is a dictionary which may define the following fields:
{
'name': The name of the option
'field_type': The django form field class for the option
'default': The default value
'field_options': An object containing fields to be passed to the form
class, e.g.:
{
'label': A label for the field
'help_text': Help text
'required': If the field is required
}
}
Each entry in the database will be unique for the values of `entry_point`
and `version`. Any backwards incompatible changes to a Tool will result
in a version bump, allowing multiple versions of a tool to work with a
Review Board instance.
"""
name = models.CharField(max_length=128, blank=False)
entry_point = models.CharField(max_length=128, blank=False)
version = models.CharField(max_length=128, blank=False)
description = models.CharField(max_length=512, default="", blank=True)
enabled = models.BooleanField(default=True)
in_last_update = models.BooleanField()
tool_options = JSONField()
def __unicode__(self):
return "%s - v%s" % (self.name, self.version)
class Meta:
unique_together = ('entry_point', 'version')
class WebAPIToken(models.Model):
"""An access token used for authenticating with the API.
Each token can be used to authenticate the token's owner with the API,
without requiring a username or password to be provided. Tokens can
be revoked, and new tokens added.
Tokens can store policy information, which will later be used for
restricting access to the API.
"""
user = models.ForeignKey(User, related_name='webapi_tokens')
local_site = models.ForeignKey(LocalSite,
related_name='webapi_tokens',
blank=True,
null=True)
token = models.CharField(max_length=40, unique=True)
time_added = models.DateTimeField(default=timezone.now)
last_updated = models.DateTimeField(default=timezone.now)
note = models.TextField(blank=True)
policy = JSONField(default={})
extra_data = JSONField(default={})
objects = WebAPITokenManager()
def is_accessible_by(self, user):
return user.is_superuser or self.user == user
def is_mutable_by(self, user):
return user.is_superuser or self.user == user
def is_deletable_by(self, user):
return user.is_superuser or self.user == user
def __str__(self):
return 'Web API token for %s' % self.user
@classmethod
def validate_policy(cls, policy):
"""Validates an API policy.
This will check to ensure that the policy is in a suitable format
and contains the information required in a format that can be parsed.
If a failure is found, a ValidationError will be raised describing
the error and where it was found.
"""
from reviewboard.webapi.resources import resources
if not isinstance(policy, dict):
raise ValidationError(_('The policy must be a JSON object.'))
if policy == {}:
# Empty policies are equivalent to allowing full access. If this
# is empty, we can stop here.
return
if 'resources' not in policy:
raise ValidationError(
_("The policy is missing a 'resources' section."))
resources_section = policy['resources']
if not isinstance(resources_section, dict):
raise ValidationError(
_("The policy's 'resources' section must be a JSON object."))
if not resources_section:
raise ValidationError(
_("The policy's 'resources' section must not be empty."))
if '*' in resources_section:
cls._validate_policy_section(resources_section, '*', 'resources.*')
resource_policies = [
(section_name, section)
for section_name, section in six.iteritems(resources_section)
if section_name != '*'
]
if resource_policies:
valid_policy_ids = cls._get_valid_policy_ids(resources.root)
for policy_id, section in resource_policies:
if policy_id not in valid_policy_ids:
raise ValidationError(
_("'%s' is not a valid resource policy ID.") %
policy_id)
for subsection_name, subsection in six.iteritems(section):
if not isinstance(subsection_name, six.text_type):
raise ValidationError(
_("%s must be a string in 'resources.%s'") %
(subsection_name, policy_id))
cls._validate_policy_section(
section, subsection_name,
'resources.%s.%s' % (policy_id, subsection_name))
@classmethod
def _validate_policy_section(cls, parent_section, section_name,
full_section_name):
section = parent_section[section_name]
if not isinstance(section, dict):
raise ValidationError(
_("The '%s' section must be a JSON object.") %
full_section_name)
if 'allow' not in section and 'block' not in section:
raise ValidationError(
_("The '%s' section must have 'allow' and/or 'block' "
"rules.") % full_section_name)
if 'allow' in section and not isinstance(section['allow'], list):
raise ValidationError(
_("The '%s' section's 'allow' rule must be a list.") %
full_section_name)
if 'block' in section and not isinstance(section['block'], list):
raise ValidationError(
_("The '%s' section's 'block' rule must be a list.") %
full_section_name)
@classmethod
def _get_valid_policy_ids(cls, resource, result=None):
if result is None:
result = set()
if hasattr(resource, 'policy_id'):
result.add(resource.policy_id)
for child_resource in resource.list_child_resources:
cls._get_valid_policy_ids(child_resource, result)
for child_resource in resource.item_child_resources:
cls._get_valid_policy_ids(child_resource, result)
return result
class Meta:
verbose_name = _('Web API token')
verbose_name_plural = _('Web API tokens')
class FileDiff(models.Model):
"""A diff of a single file.
This contains the patch and information needed to produce original and
patched versions of a single file in a repository.
"""
_ANCESTORS_KEY = '__ancestors'
COPIED = 'C'
DELETED = 'D'
MODIFIED = 'M'
MOVED = 'V'
STATUSES = (
(COPIED, _('Copied')),
(DELETED, _('Deleted')),
(MODIFIED, _('Modified')),
(MOVED, _('Moved')),
)
_IS_PARENT_EMPTY_KEY = '__parent_diff_empty'
diffset = models.ForeignKey('DiffSet',
related_name='files',
verbose_name=_('diff set'))
commit = models.ForeignKey(DiffCommit,
related_name='files',
verbose_name=_('diff commit'),
null=True)
source_file = models.CharField(_('source file'), max_length=1024)
dest_file = models.CharField(_('destination file'), max_length=1024)
source_revision = models.CharField(_('source file revision'),
max_length=512)
dest_detail = models.CharField(_('destination file details'),
max_length=512)
binary = models.BooleanField(_('binary file'), default=False)
status = models.CharField(_('status'), max_length=1, choices=STATUSES)
diff64 = Base64Field(_('diff'), db_column='diff_base64', blank=True)
legacy_diff_hash = models.ForeignKey(LegacyFileDiffData,
db_column='diff_hash_id',
related_name='filediffs',
null=True,
blank=True)
diff_hash = models.ForeignKey(RawFileDiffData,
db_column='raw_diff_hash_id',
related_name='filediffs',
null=True,
blank=True)
parent_diff64 = Base64Field(_('parent diff'),
db_column='parent_diff_base64',
blank=True)
legacy_parent_diff_hash = models.ForeignKey(
LegacyFileDiffData,
db_column='parent_diff_hash_id',
related_name='parent_filediffs',
null=True,
blank=True)
parent_diff_hash = models.ForeignKey(RawFileDiffData,
db_column='raw_parent_diff_hash_id',
related_name='parent_filediffs',
null=True,
blank=True)
extra_data = JSONField(null=True)
objects = FileDiffManager()
@property
def source_file_display(self):
"""The displayed filename for the source/original file.
This may be different than :py:attr:`source_file`, as the associated
:py:class:`~reviewboard.scmtools.core.SCMTool` may normalize it for
display.
Type:
unicode
"""
tool = self.diffset.repository.get_scmtool()
return tool.normalize_path_for_display(self.source_file,
extra_data=self.extra_data)
@property
def dest_file_display(self):
"""The displayed filename for the destination/modified file.
This may be different than :py:attr:`dest_file`, as the associated
:py:class:`~reviewboard.scmtools.core.SCMTool` may normalize it for
display.
Type:
unicode
"""
tool = self.diffset.repository.get_scmtool()
return tool.normalize_path_for_display(self.dest_file,
extra_data=self.extra_data)
@property
def deleted(self):
return self.status == self.DELETED
@property
def copied(self):
return self.status == self.COPIED
@property
def moved(self):
return self.status == self.MOVED
@property
def modified(self):
"""Whether this file is a modification to an existing file."""
return self.status == self.MODIFIED
@property
def is_new(self):
return self.source_revision == PRE_CREATION
@property
def status_string(self):
"""The FileDiff's status as a human-readable string."""
if self.status == FileDiff.COPIED:
return 'copied'
elif self.status == FileDiff.DELETED:
return 'deleted'
elif self.status == FileDiff.MODIFIED:
return 'modified'
elif self.status == FileDiff.MOVED:
return 'moved'
else:
logging.error('Unknown FileDiff status %r for FileDiff %s',
self.status, self.pk)
return 'unknown'
def _get_diff(self):
if self._needs_diff_migration():
self._migrate_diff_data()
return self.diff_hash.content
def _set_diff(self, diff):
# Add hash to table if it doesn't exist, and set diff_hash to this.
self.diff_hash, is_new = \
RawFileDiffData.objects.get_or_create_from_data(diff)
self.diff64 = b''
return is_new
diff = property(_get_diff, _set_diff)
@property
def is_diff_empty(self):
"""Whether or not the diff is empty."""
line_counts = self.get_line_counts()
return (line_counts['raw_insert_count'] == 0
and line_counts['raw_delete_count'] == 0)
def _get_parent_diff(self):
if self._needs_parent_diff_migration():
self._migrate_diff_data()
if self.parent_diff_hash:
return self.parent_diff_hash.content
else:
return None
def _set_parent_diff(self, parent_diff):
if not parent_diff:
return False
# Add hash to table if it doesn't exist, and set diff_hash to this.
self.parent_diff_hash, is_new = \
RawFileDiffData.objects.get_or_create_from_data(parent_diff)
self.parent_diff64 = b''
return is_new
parent_diff = property(_get_parent_diff, _set_parent_diff)
def is_parent_diff_empty(self, cache_only=False):
"""Return whether or not the parent diff is empty.
Args:
cache_only (bool, optional):
Whether or not to only use cached results.
Returns:
bool:
Whether or not the parent diff is empty. This is true if either
there is no parent diff or if the parent diff has no insertions and
no deletions.
"""
assert self.parent_diff_hash_id is not None, 'No parent diff.'
if cache_only:
return self.extra_data.get(self._IS_PARENT_EMPTY_KEY, False)
if (not self.extra_data
or self._IS_PARENT_EMPTY_KEY not in self.extra_data):
parent_diff_hash = self.parent_diff_hash
if (parent_diff_hash.insert_count is None
or parent_diff_hash.delete_count is None):
tool = self.get_repository().get_scmtool()
parent_diff_hash.recalculate_line_counts(tool)
self.extra_data[self._IS_PARENT_EMPTY_KEY] = (
parent_diff_hash.insert_count == 0
and parent_diff_hash.delete_count == 0)
self.save(update_fields=('extra_data', ))
return self.extra_data[self._IS_PARENT_EMPTY_KEY]
@property
def orig_sha1(self):
"""The computed SHA1 hash of the original file.
This may be ``None``, in which case it will be populated when the
diff is next viewed.
"""
return self.extra_data.get('orig_sha1')
@property
def patched_sha1(self):
"""The computed SHA1 hash of the patched file.
This may be ``None``, in which case it will be populated when the
diff is next viewed.
"""
return self.extra_data.get('patched_sha1')
@property
def orig_sha256(self):
"""The computed SHA256 hash of the original file.
This may be ``None``, in which case it will be populated when the
diff is next viewed.
Version Added:
4.0
"""
return self.extra_data.get('orig_sha256')
@property
def patched_sha256(self):
"""The computed SHA256 hash of the patched file.
This may be ``None``, in which case it will be populated when the
diff is next viewed.
Version Added:
4.0
"""
return self.extra_data.get('patched_sha256')
@property
def encoding(self):
"""The encoding of the source and patched file.
This will check the ``encoding`` key in :py:attr:`extra_data`.
If not available, then this will be ``None``.
Version Added:
4.0
"""
return self.extra_data.get('encoding')
def get_line_counts(self):
"""Return the stored line counts for the diff.
This will return all the types of line counts that can be set.
Returns:
dict:
A dictionary with the following keys:
* ``raw_insert_count``
* ``raw_delete_count``
* ``insert_count``
* ``delete_count``
* ``replace_count``
* ``equal_count``
* ``total_line_count``
These are not all guaranteed to have values set, and may instead be
``None``. Only ``raw_insert_count``, ``raw_delete_count``
``insert_count``, and ``delete_count`` are guaranteed to have
values set.
If there isn't a processed number of inserts or deletes stored,
then ``insert_count`` and ``delete_count`` will be equal to the raw
versions.
"""
if ('raw_insert_count' not in self.extra_data
or 'raw_delete_count' not in self.extra_data):
if not self.diff_hash:
self._migrate_diff_data()
if self.diff_hash.insert_count is None:
self._recalculate_line_counts(self.diff_hash)
self.extra_data.update({
'raw_insert_count':
self.diff_hash.insert_count,
'raw_delete_count':
self.diff_hash.delete_count,
})
if self.pk:
self.save(update_fields=['extra_data'])
raw_insert_count = self.extra_data['raw_insert_count']
raw_delete_count = self.extra_data['raw_delete_count']
return {
'raw_insert_count': raw_insert_count,
'raw_delete_count': raw_delete_count,
'insert_count': self.extra_data.get('insert_count',
raw_insert_count),
'delete_count': self.extra_data.get('delete_count',
raw_delete_count),
'replace_count': self.extra_data.get('replace_count'),
'equal_count': self.extra_data.get('equal_count'),
'total_line_count': self.extra_data.get('total_line_count'),
}
def set_line_counts(self,
raw_insert_count=None,
raw_delete_count=None,
insert_count=None,
delete_count=None,
replace_count=None,
equal_count=None,
total_line_count=None):
"""Set the line counts on the FileDiff.
There are many types of useful line counts that can be set.
Args:
raw_insert_count (int, optional):
The insert count on the original patch.
This will be set on the
:py:class:`reviewboard.diffviewer.models.raw_file_diff_data.RawFileDiffData`
as well.
raw_delete_count (int, optional):
The delete count in the original patch.
This will be set on the
:py:class:`reviewboard.diffviewer.models.raw_file_diff_data.RawFileDiffData`
as well.
insert_count (int, optional):
The number of lines that were inserted in the diff.
delete_count (int, optional):
The number of lines that were deleted in the diff.
replace_count (int, optional):
The number of lines that were replaced in the diff.
equal_count (int, optional):
The number of lines that were identical in the diff.
total_line_count (int, optional):
The total line count.
"""
updated = False
if not self.diff_hash_id:
# This really shouldn't happen, but if it does, we should handle
# it gracefully.
logging.warning('Attempting to call set_line_counts on '
'un-migrated FileDiff %s' % self.pk)
self._migrate_diff_data(False)
if (insert_count is not None and raw_insert_count is not None
and self.diff_hash.insert_count is not None
and self.diff_hash.insert_count != insert_count):
# Allow overriding, but warn. This really shouldn't be called.
logging.warning('Attempting to override insert count on '
'RawFileDiffData %s from %s to %s (FileDiff %s)' %
(self.diff_hash.pk, self.diff_hash.insert_count,
insert_count, self.pk))
if (delete_count is not None and raw_delete_count is not None
and self.diff_hash.delete_count is not None
and self.diff_hash.delete_count != delete_count):
# Allow overriding, but warn. This really shouldn't be called.
logging.warning('Attempting to override delete count on '
'RawFileDiffData %s from %s to %s (FileDiff %s)' %
(self.diff_hash.pk, self.diff_hash.delete_count,
delete_count, self.pk))
if raw_insert_count is not None or raw_delete_count is not None:
# New raw counts have been provided. These apply to the actual
# diff file itself, and will be common across all diffs sharing
# the diff_hash instance. Set it there.
if raw_insert_count is not None:
self.diff_hash.insert_count = raw_insert_count
self.extra_data['raw_insert_count'] = raw_insert_count
updated = True
if raw_delete_count is not None:
self.diff_hash.delete_count = raw_delete_count
self.extra_data['raw_delete_count'] = raw_delete_count
updated = True
self.diff_hash.save()
for key, cur_value in (('insert_count', insert_count), ('delete_count',
delete_count),
('replace_count',
replace_count), ('equal_count', equal_count),
('total_line_count', total_line_count)):
if cur_value is not None and cur_value != self.extra_data.get(key):
self.extra_data[key] = cur_value
updated = True
if updated and self.pk:
self.save(update_fields=['extra_data'])
def get_ancestors(self, minimal, filediffs=None, update=True):
"""Return the ancestors of this FileDiff.
This will update the ancestors of this :py:class:`FileDiff` and all its
ancestors if they are not already cached.
Args:
minimal (bool):
Whether or not the minimal set of ancestors are returned.
The minimal set of ancestors does not include ancestors where
the file was deleted. In other words, if a file is deleted and
then re-created, the deletion will not appear in the re-created
file's ancestor list.
filediffs (iterable of FileDiff, optional):
An optional list of FileDiffs to check for ancestors so that a
query does not have to be performed.
update (bool, optional):
Whether or not to cache the results in the database.
If ``True`` and the results have not already been cached, this
FileDiff and its ancestors will all be updated.
Returns:
list of FileDiff:
The ancestor :py:class:`FileDiffs <FileDiff>`, in application
order.
"""
if self.commit_id is None:
return []
if (self.extra_data is None
or self._ANCESTORS_KEY not in self.extra_data):
if filediffs is None:
filediffs = list(
FileDiff.objects.filter(diffset_id=self.diffset_id))
compliment_ids, minimal_ids = self._compute_ancestors(
filediffs, update)
else:
compliment_ids, minimal_ids = self.extra_data[self._ANCESTORS_KEY]
if filediffs is None:
filediffs = FileDiff.objects.filter(pk__in=compliment_ids +
minimal_ids)
by_id = {filediff.pk: filediff for filediff in filediffs}
if minimal:
ids = minimal_ids
else:
ids = chain(compliment_ids, minimal_ids)
return [by_id[pk] for pk in ids]
def get_repository(self):
"""Return the repository this diff applies to.
Version Added:
4.0
Returns:
reviewboard.scmtools.models.Repository:
The repository used for any operations for this FileDiff.
"""
return self.diffset.repository
def get_base_filediff(self, base_commit, ancestors=None):
"""Return the base FileDiff within a commit range.
This looks through the FileDiff's ancestors, looking for the most
recent one that equals or precedes ``base_commit``.
Args:
base_commit (reviewboard.diffviewer.models.diffcommit.DiffCommit):
The newest commit that the resulting base FileDiff can be
associated with.
ancestors (list of FileDiff, optional):
A pre-fetched list of FileDiff ancestors. If not provided, a
list will be fetched.
Returns:
FileDiff:
The base FileDiff, if one could be found. ``None`` will be
returned if no base could be found.
"""
if base_commit and self.commit_id:
if ancestors is None:
ancestors = self.get_ancestors(minimal=False) or []
for ancestor in reversed(ancestors):
if ancestor.commit_id <= base_commit.pk:
return ancestor
return None
def _compute_ancestors(self, filediffs, update):
"""Compute the ancestors of this FileDiff.
Args:
filediffs (iterable of FileDiff):
The list of FileDiffs to check for ancestors.
update (bool):
Whether or not to cache the results.
Returns:
list of FileDiff:
The ancestor FileDiffs in application order.
"""
by_dest_file = {}
by_id = {}
for filediff in filediffs:
by_detail = by_dest_file.setdefault(filediff.dest_file, {})
by_commit = by_detail.setdefault(filediff.dest_detail, {})
by_commit[filediff.commit_id] = filediff
by_id[filediff.pk] = filediff
current = self
ancestor_ids = []
should_update = {self.pk}
while True:
# If the ancestors have already been computed for the direct
# ancestor, we can stop there and re-use those results.
try:
rest = current.extra_data[self._ANCESTORS_KEY]
except (KeyError, TypeError):
should_update.add(current.pk)
else:
# We reverse below, so we have to add these in reverse order.
ancestor_ids.extend(chain(reversed(rest[1]),
reversed(rest[0])))
break
if current.is_new:
# If the FileDiff is new there may have been a previous
# FileDiff with the same name that was deleted.
try:
by_detail = by_dest_file[current.source_file]
prev_set = (filediff
for by_commit in six.itervalues(by_detail)
for filediff in six.itervalues(by_commit)
if filediff.deleted)
except KeyError:
break
else:
# Otherwise we need to walk the graph to find the previous
# FileDiff.
try:
by_detail = by_dest_file[current.source_file]
prev_set = six.itervalues(
by_detail[current.source_revision])
except KeyError:
# There is no previous FileDiff created by the commit series.
break
# The only information we know is the previous revision and name,
# of which there might be multiple matches. We need to find the
# most recent FileDiff that matches that criteria that belongs to a
# commit that comes before the current FileDiff's commit in
# application order.
try:
prev = max((filediff for filediff in prev_set
if filediff.commit_id < current.commit_id),
key=lambda filediff: filediff.commit_id)
except ValueError:
# max() raises ValueError if it is given an empty iterable.
# This means there is no previous FileDiff created by the
# commit series.
break
ancestor_ids.append(prev.pk)
current = prev
# We computed the list of ancestors in reverse order (i.e., most recent
# to least recent) above.
ancestor_ids.reverse()
compliment_ids, minimal_ids = self._split_ancestors(
by_id, ancestor_ids)
if update:
for i, pk in enumerate(ancestor_ids):
if pk in should_update:
filediff = by_id[pk]
if filediff.extra_data is None:
filediff.extra_data = {}
# We need to split the history at the point of the last
# deletion. That way we will have the minimal set of
# ancestors, which we can use to compute the diff for this
# FileDiff, and the maximal set of ancestors, which we can
# use to compute cumulative diffs.
filediff.extra_data[self._ANCESTORS_KEY] = \
list(self._split_ancestors(by_id, ancestor_ids[:i]))
filediff.save(update_fields=('extra_data', ))
self.extra_data[self._ANCESTORS_KEY] = [
compliment_ids, minimal_ids
]
self.save(update_fields=('extra_data', ))
return compliment_ids, minimal_ids
def _split_ancestors(self, by_id, ancestor_ids):
"""Split the ancestor IDs into pre-delete and post-delete sets.
Args:
by_id (dict):
A mapping of primary keys to :py:class:`FileDiffs
<reviewboard.diffviewer.models.filediff.FileDiff>`.
ancestor_ids (list of int):
The ancestors to split.
Returns:
tuple:
A 2-tuple of:
* The compliment of the minimal ancestors (py:class:`list` of
:py:class:`int`).
* The list of minimal ancestors (py:class:`list` of
:py:class:`int`).
"""
i = 0
# We traverse the list backwards to find the last deleted FileDiff.
# Everything after that is in the set of minimal ancestors.
for i in range(len(ancestor_ids) - 1, -1, -1):
filediff = by_id[ancestor_ids[i]]
if filediff.deleted:
i += 1
break
return ancestor_ids[:i], ancestor_ids[i:]
def _needs_diff_migration(self):
return self.diff_hash_id is None
def _needs_parent_diff_migration(self):
return (self.parent_diff_hash_id is None
and (self.parent_diff64 or self.legacy_parent_diff_hash_id))
def _migrate_diff_data(self, recalculate_counts=True):
"""Migrates diff data associated with a FileDiff to RawFileDiffData.
If the diff data is stored directly on the FileDiff, it will be
removed and stored on a RawFileDiffData instead.
If the diff data is stored on an associated LegacyFileDiffData,
that will be converted into a RawFileDiffData. The LegacyFileDiffData
will then be removed, if nothing else is using it.
Args:
recalculate_line_counts (bool, optional):
Whether or not line counts should be recalculated during the
migration.
"""
needs_save = False
diff_hash_is_new = False
parent_diff_hash_is_new = False
fix_refs = False
legacy_pks = []
needs_diff_migration = self._needs_diff_migration()
needs_parent_diff_migration = self._needs_parent_diff_migration()
if needs_diff_migration:
recalculate_diff_counts = recalculate_counts
needs_save = True
if self.legacy_diff_hash_id:
logging.debug(
'Migrating LegacyFileDiffData %s to '
'RawFileDiffData for diff in FileDiff %s',
self.legacy_diff_hash_id, self.pk)
try:
legacy_data = self.legacy_diff_hash.binary
except LegacyFileDiffData.DoesNotExist:
# Another process migrated this before we could.
# We'll need to fix the references.
fix_refs = True
recalculate_diff_counts = False
else:
diff_hash_is_new = self._set_diff(legacy_data)
legacy_pks.append(self.legacy_diff_hash_id)
self.legacy_diff_hash = None
else:
logging.debug(
'Migrating FileDiff %s diff data to '
'RawFileDiffData', self.pk)
diff_hash_is_new = self._set_diff(self.diff64)
if recalculate_diff_counts:
self._recalculate_line_counts(self.diff_hash)
if needs_parent_diff_migration:
recalculate_parent_diff_counts = recalculate_counts
needs_save = True
if self.legacy_parent_diff_hash_id:
logging.debug(
'Migrating LegacyFileDiffData %s to '
'RawFileDiffData for parent diff in FileDiff %s',
self.legacy_parent_diff_hash_id, self.pk)
try:
legacy_parent_data = self.legacy_parent_diff_hash.binary
except LegacyFileDiffData.DoesNotExist:
# Another process migrated this before we could.
# We'll need to fix the references.
fix_refs = True
recalculate_parent_diff_counts = False
else:
parent_diff_hash_is_new = \
self._set_parent_diff(legacy_parent_data)
legacy_pks.append(self.legacy_parent_diff_hash_id)
self.legacy_parent_diff_hash = None
else:
logging.debug(
'Migrating FileDiff %s parent diff data to '
'RawFileDiffData', self.pk)
parent_diff_hash_is_new = \
self._set_parent_diff(self.parent_diff64)
if recalculate_parent_diff_counts:
self._recalculate_line_counts(self.parent_diff_hash)
if fix_refs:
# Another server/process/thread got to this before we could.
# We need to pull the latest refs and make sure they're set here.
diff_hash, parent_diff_hash = (FileDiff.objects.filter(
pk=self.pk).values_list('diff_hash_id',
'parent_diff_hash_id')[0])
if needs_diff_migration:
if diff_hash:
self.diff_hash_id = diff_hash
self.legacy_diff_hash = None
self.diff64 = ''
else:
logging.error(
'Unable to migrate diff for FileDiff %s: '
'LegacyFileDiffData "%s" is missing, and '
'database entry does not have a new '
'diff_hash! Data may be missing.', self.pk,
self.legacy_diff_hash_id)
if needs_parent_diff_migration:
if parent_diff_hash:
self.parent_diff_hash_id = parent_diff_hash
self.legacy_parent_diff_hash = None
self.parent_diff64 = ''
else:
logging.error(
'Unable to migrate parent diff for '
'FileDiff %s: LegacyFileDiffData "%s" is '
'missing, and database entry does not have '
'a new parent_diff_hash! Data may be '
'missing.', self.pk, self.legacy_parent_diff_hash_id)
if needs_save:
if self.pk:
self.save(update_fields=[
'diff64',
'parent_diff64',
'diff_hash',
'parent_diff_hash',
'legacy_diff_hash',
'legacy_parent_diff_hash',
])
else:
self.save()
if legacy_pks:
# Delete any LegacyFileDiffData objects no longer associated
# with any FileDiffs.
LegacyFileDiffData.objects \
.filter(pk__in=legacy_pks) \
.exclude(Q(filediffs__pk__gt=0) |
Q(parent_filediffs__pk__gt=0)) \
.delete()
return diff_hash_is_new, parent_diff_hash_is_new
def _recalculate_line_counts(self, diff_hash):
"""Recalculate line counts for the raw data.
Args:
diff_hash (reviewboard.diffviewer.models.raw_file_diff_data.
RawFileDiffData):
The raw data to recalculate line counts for.
"""
diff_hash.recalculate_line_counts(self.get_repository().get_scmtool())
def __str__(self):
"""Return a human-readable representation of the model.
Returns:
unicode:
A human-readable representation of the model.
"""
return "%s (%s) -> %s (%s)" % (self.source_file, self.source_revision,
self.dest_file, self.dest_detail)
class Meta:
app_label = 'diffviewer'
db_table = 'diffviewer_filediff'
verbose_name = _('File Diff')
verbose_name_plural = _('File Diffs')
class StatusUpdate(models.Model):
"""A status update from a third-party service or extension.
This status model allows external services (such as continuous integration
services, Review Bot, etc.) to provide an update on their status. An
example of this would be a CI tool which does experimental builds of
changes. While the build is running, that tool would set its status to
pending, and when it was done, would set it to one of the done states,
and potentially associate it with a review containing issues.
"""
#: The pending state.
PENDING = 'P'
#: The completed successfully state.
DONE_SUCCESS = 'S'
#: The completed with reported failures state.
DONE_FAILURE = 'F'
#: The error state.
ERROR = 'E'
#: Timeout state.
TIMEOUT = 'T'
STATUSES = (
(PENDING, _('Pending')),
(DONE_SUCCESS, _('Done (Success)')),
(DONE_FAILURE, _('Done (Failure)')),
(ERROR, _('Error')),
(TIMEOUT, _('Timed Out')),
)
#: An identifier for the service posting this status update.
#:
#: This ID is self-assigned, and just needs to be unique to that service.
#: Possible values can be an extension ID, webhook URL, or a script name.
service_id = models.CharField(_('Service ID'), max_length=255)
#: The user who created this status update.
user = models.ForeignKey(
User,
related_name='status_updates',
verbose_name=_('User'),
blank=True,
null=True)
#: The timestamp of the status update.
timestamp = models.DateTimeField(_('Timestamp'), default=timezone.now)
#: A user-visible short summary of the status update.
#:
#: This is typically the name of the integration or tool that was run.
summary = models.CharField(_('Summary'), max_length=255)
#: A user-visible description on the status update.
#:
#: This is shown in the UI adjacent to the summary. Typical results might
#: be things like "running." or "failed.". This should include punctuation.
description = models.CharField(_('Description'), max_length=255,
blank=True)
#: An optional link.
#:
#: This is used in case the tool has some external page, such as a build
#: results page on a CI system.
url = models.URLField(_('Link URL'), max_length=255, blank=True)
#: Text for the link. If ``url`` is empty, this will not be used.
url_text = models.CharField(_('Link text'), max_length=64, blank=True)
#: The current state of this status update.
#:
#: This should be set to :py:attr:`PENDING` while the service is
#: processing the update, and then to either :py:attr:`DONE_SUCCESS` or
#: :py:attr:`DONE_FAILURE` once complete. If the service encountered some
#: error which prevented completion, this should be set to
#: :py:attr:`ERROR`.
state = models.CharField(_('State'), max_length=1, choices=STATUSES)
#: The review request that this status update is for.
review_request = models.ForeignKey(
ReviewRequest,
related_name='status_updates',
verbose_name=_('Review Request'))
#: The change to the review request that this status update is for.
#:
#: If this is ``None``, this status update refers to the review request as
#: a whole (for example, the initial diff that was posted).
change_description = models.ForeignKey(
ChangeDescription,
related_name='status_updates',
verbose_name=_('Change Description'),
null=True,
blank=True)
#: An optional review created for this status update.
#:
#: This allows the third-party service to create comments and open issues.
review = models.OneToOneField(
Review,
related_name='status_update',
verbose_name=_('Review'),
null=True,
blank=True)
#: Any extra data that the service wants to store for this status update.
extra_data = JSONField(null=True)
#: An (optional) timeout, in seconds. If this is non-None and the state has
#: been ``PENDING`` for longer than this period (computed from the
#: :py:attr:`timestamp` field), :py:attr:`effective_state` will be
#: ``TIMEOUT``.
timeout = models.IntegerField(null=True, blank=True)
@staticmethod
def state_to_string(state):
"""Return a string representation of a status update state.
Args:
state (unicode):
A single-character string representing the state.
Returns:
unicode:
A longer string representation of the state suitable for use in
the API.
"""
if state == StatusUpdate.PENDING:
return 'pending'
elif state == StatusUpdate.DONE_SUCCESS:
return 'done-success'
elif state == StatusUpdate.DONE_FAILURE:
return 'done-failure'
elif state is StatusUpdate.ERROR:
return 'error'
elif state is StatusUpdate.TIMEOUT:
return 'timed-out'
else:
raise ValueError('Invalid state "%s"' % state)
@staticmethod
def string_to_state(state):
"""Return a status update state from an API string.
Args:
state (unicode):
A string from the API representing the state.
Returns:
unicode:
A single-character string representing the state, suitable for
storage in the ``state`` field.
"""
if state == 'pending':
return StatusUpdate.PENDING
elif state == 'done-success':
return StatusUpdate.DONE_SUCCESS
elif state == 'done-failure':
return StatusUpdate.DONE_FAILURE
elif state == 'error':
return StatusUpdate.ERROR
elif state == 'timed-out':
return StatusUpdate.TIMEOUT
else:
raise ValueError('Invalid state string "%s"' % state)
def is_mutable_by(self, user):
"""Return whether the user can modify this status update.
Args:
user (django.contrib.auth.models.User):
The user to check.
Returns:
bool:
True if the user can modify this status update.
"""
return (self.user == user or
user.has_perm('reviews.can_edit_status',
self.review_request.local_site))
@property
def effective_state(self):
"""The state of the status update, taking into account timeouts."""
if self.state == self.PENDING and self.timeout is not None:
timeout = self.timestamp + datetime.timedelta(seconds=self.timeout)
if timezone.now() > timeout:
return self.TIMEOUT
return self.state
class Meta:
app_label = 'reviews'
ordering = ['timestamp']
get_latest_by = 'timestamp'
class BaseReviewRequestDetails(models.Model):
"""Base information for a review request and draft.
ReviewRequest and ReviewRequestDraft share a lot of fields and
methods. This class provides those fields and methods for those
classes.
"""
MAX_SUMMARY_LENGTH = 300
description = models.TextField(_("description"), blank=True)
description_rich_text = models.BooleanField(_('description in rich text'),
default=False)
testing_done = models.TextField(_("testing done"), blank=True)
testing_done_rich_text = models.BooleanField(
_('testing done in rich text'), default=False)
bugs_closed = models.CharField(_("bugs"), max_length=300, blank=True)
branch = models.CharField(_("branch"), max_length=300, blank=True)
commit_id = models.CharField(_('commit ID'),
max_length=64,
blank=True,
null=True,
db_index=True)
extra_data = JSONField(null=True)
# Deprecated and no longer used for new review requests as of 2.0.9.
rich_text = models.BooleanField(_("rich text"), default=False)
def get_review_request(self):
raise NotImplementedError
def get_bug_list(self):
"""Returns a list of bugs associated with this review request."""
if self.bugs_closed == "":
return []
bugs = list(set(re.split(r"[, ]+", self.bugs_closed)))
# First try a numeric sort, to show the best results for the majority
# case of bug trackers with numeric IDs. If that fails, sort
# alphabetically.
try:
bugs.sort(key=int)
except ValueError:
bugs.sort()
return bugs
def get_screenshots(self):
"""Returns the list of all screenshots on a review request.
This includes all current screenshots, but not previous ones.
By accessing screenshots through this method, future review request
lookups from the screenshots will be avoided.
"""
review_request = self.get_review_request()
for screenshot in self.screenshots.all():
screenshot._review_request = review_request
yield screenshot
def get_inactive_screenshots(self):
"""Returns the list of all inactive screenshots on a review request.
This only includes screenshots that were previously visible but
have since been removed.
By accessing screenshots through this method, future review request
lookups from the screenshots will be avoided.
"""
review_request = self.get_review_request()
for screenshot in self.inactive_screenshots.all():
screenshot._review_request = review_request
yield screenshot
def get_file_attachments(self):
"""Returns the list of all file attachments on a review request.
This includes all current file attachments, but not previous ones.
By accessing file attachments through this method, future review
request lookups from the file attachments will be avoided.
"""
review_request = self.get_review_request()
def get_attachments():
for file_attachment in self.file_attachments.all():
file_attachment._review_request = review_request
# Handle legacy entries which don't have an associated
# FileAttachmentHistory entry.
if (not file_attachment.is_from_diff
and file_attachment.attachment_history is None):
history = FileAttachmentHistory.objects.create(
display_position=FileAttachmentHistory.
compute_next_display_position(review_request))
review_request.file_attachment_histories.add(history)
file_attachment.attachment_history = history
file_attachment.save(update_fields=['attachment_history'])
yield file_attachment
def get_display_position(attachment):
if attachment.attachment_history_id is not None:
return attachment.attachment_history.display_position
else:
return 0
return sorted(list(get_attachments()), key=get_display_position)
def get_inactive_file_attachments(self):
"""Returns all inactive file attachments on a review request.
This only includes file attachments that were previously visible
but have since been removed.
By accessing file attachments through this method, future review
request lookups from the file attachments will be avoided.
"""
review_request = self.get_review_request()
for file_attachment in self.inactive_file_attachments.all():
file_attachment._review_request = review_request
yield file_attachment
def add_default_reviewers(self):
"""Add default reviewers based on the diffset.
This method goes through the DefaultReviewer objects in the database
and adds any missing reviewers based on regular expression comparisons
with the set of files in the diff.
"""
diffset = self.get_latest_diffset()
if not diffset:
return
people = set()
groups = set()
# TODO: This is kind of inefficient, and could maybe be optimized in
# some fancy way. Certainly the most superficial optimization that
# could be made would be to cache the compiled regexes somewhere.
files = diffset.files.all()
reviewers = DefaultReviewer.objects.for_repository(
self.repository, self.local_site)
for default in reviewers:
try:
regex = re.compile(default.file_regex)
except:
continue
for filediff in files:
if regex.match(filediff.source_file or filediff.dest_file):
for person in default.people.all():
if person.is_active:
people.add(person)
for group in default.groups.all():
groups.add(group)
break
existing_people = self.target_people.all()
for person in people:
if person not in existing_people:
self.target_people.add(person)
existing_groups = self.target_groups.all()
for group in groups:
if group not in existing_groups:
self.target_groups.add(group)
def update_from_commit_id(self, commit_id):
"""Updates the data from a server-side changeset.
If the commit ID refers to a pending changeset on an SCM which stores
such things server-side (like perforce), the details like the summary
and description will be updated with the latest information.
If the change number is the commit ID of a change which exists on the
server, the summary and description will be set from the commit's
message, and the diff will be fetched from the SCM.
"""
scmtool = self.repository.get_scmtool()
changeset = None
if scmtool.supports_pending_changesets:
changeset = scmtool.get_changeset(commit_id, allow_empty=True)
if changeset and changeset.pending:
self.update_from_pending_change(commit_id, changeset)
elif self.repository.supports_post_commit:
self.update_from_committed_change(commit_id)
else:
if changeset:
raise InvalidChangeNumberError()
else:
raise NotImplementedError()
def update_from_pending_change(self, commit_id, changeset):
"""Updates the data from a server-side pending changeset.
This will fetch the metadata from the server and update the fields on
the review request.
"""
if not changeset:
raise InvalidChangeNumberError()
# If the SCM supports changesets, they should always include a number,
# summary and description, parsed from the changeset description. Some
# specialized systems may support the other fields, but we don't want
# to clobber the user-entered values if they don't.
self.commit = commit_id
description = changeset.description
testing_done = changeset.testing_done
self.summary = changeset.summary
self.description = description
self.description_rich_text = False
if testing_done:
self.testing_done = testing_done
self.testing_done_rich_text = False
if changeset.branch:
self.branch = changeset.branch
if changeset.bugs_closed:
self.bugs_closed = ','.join(changeset.bugs_closed)
def update_from_committed_change(self, commit_id):
"""Updates from a committed change present on the server.
Fetches the commit message and diff from the repository and sets the
relevant fields.
"""
commit = self.repository.get_change(commit_id)
summary, message = commit.split_message()
message = message.strip()
self.commit = commit_id
self.summary = summary.strip()
self.description = message
self.description_rich_text = False
DiffSet.objects.create_from_data(
repository=self.repository,
diff_file_name='diff',
diff_file_contents=commit.diff.encode('utf-8'),
parent_diff_file_name=None,
parent_diff_file_contents=None,
diffset_history=self.get_review_request().diffset_history,
basedir='/',
request=None,
base_commit_id=commit.base_commit_id)
def save(self, **kwargs):
self.bugs_closed = self.bugs_closed.strip()
self.summary = self._truncate(self.summary, self.MAX_SUMMARY_LENGTH)
super(BaseReviewRequestDetails, self).save(**kwargs)
def _truncate(self, string, num):
if len(string) > num:
string = string[0:num]
i = string.rfind('.')
if i != -1:
string = string[0:i + 1]
return string
def __str__(self):
if self.summary:
return six.text_type(self.summary)
else:
return six.text_type(_('(no summary)'))
class Meta:
abstract = True
app_label = 'reviews'
class Profile(models.Model):
"""User profile which contains some basic configurable settings."""
user = models.ForeignKey(User, unique=True)
# This will redirect new users to the account settings page the first time
# they log in (or immediately after creating an account). This allows
# people to fix their real name and join groups.
first_time_setup_done = models.BooleanField(
default=False,
verbose_name=_("first time setup done"),
help_text=_("Indicates whether the user has already gone through "
"the first time setup process by saving their user "
"preferences."))
# Whether the user wants to receive emails
should_send_email = models.BooleanField(
default=True,
verbose_name=_("send email"),
help_text=_("Indicates whether the user wishes to receive emails."))
should_send_own_updates = models.BooleanField(
default=True,
verbose_name=_("receive emails about own actions"),
help_text=_("Indicates whether the user wishes to receive emails "
"about their own activity."))
collapsed_diffs = models.BooleanField(
default=True,
verbose_name=_("collapsed diffs"),
help_text=_("Indicates whether diffs should be shown in their "
"collapsed state by default."))
wordwrapped_diffs = models.BooleanField(
default=True,
help_text=_("This field is unused and will be removed in a future "
"version."))
syntax_highlighting = models.BooleanField(
default=True,
verbose_name=_("syntax highlighting"),
help_text=_("Indicates whether the user wishes to see "
"syntax highlighting in the diffs."))
is_private = models.BooleanField(
default=False,
verbose_name=_("profile private"),
help_text=_("Indicates whether the user wishes to keep his/her "
"profile private."))
open_an_issue = models.BooleanField(
default=True,
verbose_name=_("opens an issue"),
help_text=_("Indicates whether the user wishes to default "
"to opening an issue or not."))
default_use_rich_text = models.NullBooleanField(
default=None,
verbose_name=_('enable Markdown by default'),
help_text=_('Indicates whether new posts or comments should default '
'to being in Markdown format.'))
# Indicate whether closed review requests should appear in the
# review request lists (excluding the dashboard).
show_closed = models.BooleanField(default=True)
sort_review_request_columns = models.CharField(max_length=256, blank=True)
sort_dashboard_columns = models.CharField(max_length=256, blank=True)
sort_submitter_columns = models.CharField(max_length=256, blank=True)
sort_group_columns = models.CharField(max_length=256, blank=True)
review_request_columns = models.CharField(max_length=256, blank=True)
dashboard_columns = models.CharField(max_length=256, blank=True)
submitter_columns = models.CharField(max_length=256, blank=True)
group_columns = models.CharField(max_length=256, blank=True)
# A list of starred review requests. This allows users to monitor a
# review request and receive e-mails on updates without actually being
# on the reviewer list or commenting on the review. This is similar to
# adding yourself to a CC list.
starred_review_requests = models.ManyToManyField(ReviewRequest,
blank=True,
related_name="starred_by")
# A list of watched groups. This is so that users can monitor groups
# without actually joining them, preventing e-mails being sent to the
# user and review requests from entering the Incoming Reviews list.
starred_groups = models.ManyToManyField(Group,
blank=True,
related_name="starred_by")
# Allows per-user timezone settings
timezone = models.CharField(choices=TIMEZONE_CHOICES,
default='UTC',
max_length=30)
settings = JSONField(null=True, default=dict)
extra_data = JSONField(null=True, default=dict)
objects = ProfileManager()
@property
def should_use_rich_text(self):
"""Get whether rich text should be used by default for this user.
If the user has chosen whether or not to use rich text explicitly,
then that choice will be respected. Otherwise, the system default is
used.
"""
if self.default_use_rich_text is None:
siteconfig = SiteConfiguration.objects.get_current()
return siteconfig.get('default_use_rich_text')
else:
return self.default_use_rich_text
@property
def should_enable_desktop_notifications(self):
"""Return whether desktop notifications should be used for this user.
If the user has chosen whether or not to use desktop notifications
explicitly, then that choice will be respected. Otherwise, we
enable desktop notifications by default.
Returns:
bool:
If the user has set whether they wish to recieve desktop
notifications, then use their preference. Otherwise, we return
``True``.
"""
return (not self.settings
or self.settings.get('enable_desktop_notifications', True))
def star_review_request(self, review_request):
"""Mark a review request as starred.
This will mark a review request as starred for this user and
immediately save to the database.
"""
self.starred_review_requests.add(review_request)
if (review_request.public
and (review_request.status == ReviewRequest.PENDING_REVIEW
or review_request.status == ReviewRequest.SUBMITTED)):
site_profile, is_new = LocalSiteProfile.objects.get_or_create(
user=self.user,
local_site=review_request.local_site,
profile=self)
if is_new:
site_profile.save()
site_profile.increment_starred_public_request_count()
self.save()
def unstar_review_request(self, review_request):
"""Mark a review request as unstarred.
This will mark a review request as starred for this user and
immediately save to the database.
"""
q = self.starred_review_requests.filter(pk=review_request.pk)
if q.count() > 0:
self.starred_review_requests.remove(review_request)
if (review_request.public
and (review_request.status == ReviewRequest.PENDING_REVIEW
or review_request.status == ReviewRequest.SUBMITTED)):
site_profile, is_new = LocalSiteProfile.objects.get_or_create(
user=self.user,
local_site=review_request.local_site,
profile=self)
if is_new:
site_profile.save()
site_profile.decrement_starred_public_request_count()
self.save()
def star_review_group(self, review_group):
"""Mark a review group as starred.
This will mark a review group as starred for this user and
immediately save to the database.
"""
if self.starred_groups.filter(pk=review_group.pk).count() == 0:
self.starred_groups.add(review_group)
def unstar_review_group(self, review_group):
"""Mark a review group as unstarred.
This will mark a review group as starred for this user and
immediately save to the database.
"""
if self.starred_groups.filter(pk=review_group.pk).count() > 0:
self.starred_groups.remove(review_group)
def __str__(self):
"""Return a string used for the admin site listing."""
return self.user.username
class JSONFieldTests(TestCase):
"""Unit tests for JSONField."""
def setUp(self):
self.field = JSONField()
def test_init_with_custom_encoder_class(self):
"""Testing JSONField initialization with custom encoder class"""
class MyEncoder(json.JSONEncoder):
def __init__(self, default_msg, **kwargs):
self.default_msg = default_msg
super(MyEncoder, self).__init__(**kwargs)
def default(self, o):
return self.default_msg
with warnings.catch_warnings(record=True) as w:
field = JSONField(encoder_cls=MyEncoder,
encoder_kwargs={
'default_msg': 'What even is this?',
})
self.assertEqual(field.dumps(MyEncoder), '"What even is this?"')
self.assertEqual(len(w), 0)
def test_init_with_custom_encoder_instance(self):
"""Testing JSONField initialization with deprecated custom encoder
instance
"""
class MyEncoder(json.JSONEncoder):
def default(self, o):
return 'What even is this?'
with warnings.catch_warnings(record=True) as w:
field = JSONField(encoder=MyEncoder())
self.assertEqual(field.dumps(MyEncoder), '"What even is this?"')
self.assertEqual(len(w), 1)
message = w[0].message
self.assertIsInstance(message, DeprecationWarning)
self.assertEqual(
six.text_type(message),
'The encoder argument to JSONField has been '
'replaced by the encoder_cls and encoder_kwargs '
'arguments. Support for encoder is deprecated.')
def test_dumps_with_json_dict(self):
"""Testing JSONField with dumping a JSON dictionary"""
result = self.field.dumps({'a': 1, 'b': 2})
self.assertTrue(isinstance(result, six.string_types))
self.assertEqual(result, '{"a": 1, "b": 2}')
def test_dumps_with_json_string(self):
"""Testing JSONField with dumping a JSON string"""
result = self.field.dumps('{"a": 1, "b": 2}')
self.assertTrue(isinstance(result, six.string_types))
self.assertEqual(result, '{"a": 1, "b": 2}')
def test_loading_json_dict(self):
"""Testing JSONField with loading a JSON dictionary"""
result = self.field.loads('{"a": 1, "b": 2}')
self.assertTrue(isinstance(result, dict))
self.assertTrue('a' in result)
self.assertTrue('b' in result)
def test_loading_json_broken_dict(self):
"""Testing JSONField with loading a badly serialized JSON dictionary"""
result = self.field.loads('{u"a": 1, u"b": 2}')
self.assertTrue(isinstance(result, dict))
self.assertTrue('a' in result)
self.assertTrue('b' in result)
def test_loading_json_array(self):
"""Testing JSONField with loading a JSON array"""
result = self.field.loads('[1, 2, 3]')
self.assertTrue(isinstance(result, list))
self.assertEqual(result, [1, 2, 3])
def test_loading_string(self):
"""Testing JSONField with loading a stored string"""
result = self.field.loads('"foo"')
self.assertTrue(isinstance(result, dict))
self.assertEqual(result, {})
def test_loading_broken_string(self):
"""Testing JSONField with loading a broken stored string"""
result = self.field.loads('u"foo"')
self.assertTrue(isinstance(result, dict))
self.assertEqual(result, {})
def test_loading_python_code(self):
"""Testing JSONField with loading Python code"""
result = self.field.loads('locals()')
self.assertTrue(isinstance(result, dict))
self.assertEqual(result, {})
def test_get_json(self):
"""Testing JSONField with get_{fieldname}_json"""
class MyModel(Model):
myfield = JSONField()
model = MyModel()
model.myfield = {
'a': 1,
'b': 2,
}
self.assertEqual(model.get_myfield_json(), '{"a": 1, "b": 2}')
def test_set_json(self):
"""Testing JSONField with set_{fieldname}_json"""
class MyModel(Model):
myfield = JSONField()
model = MyModel()
model.set_myfield_json('{"a": 1, "b": 2}')
self.assertEqual(model.myfield, {
'a': 1,
'b': 2,
})
def test_validate_with_valid_json_string(self):
"""Testing JSONField with validating a valid JSON string"""
self.field.run_validators('{"a": 1, "b": 2}')
def test_validate_with_invalid_json_string(self):
"""Testing JSONField with validating an invalid JSON string"""
self.assertRaises(ValidationError,
lambda: self.field.run_validators('foo'))
def test_validate_with_json_dict(self):
"""Testing JSONField with validating a JSON dictionary"""
self.field.run_validators({'a': 1, 'b': 2})
class MyModel(Model):
myfield = JSONField()
class Group(models.Model):
"""A group of people who can be targetted for review.
This is usually used to separate teams at a company or components of a
project.
Each group can have an e-mail address associated with it, sending
all review requests and replies to that address. If that e-mail address is
blank, e-mails are sent individually to each member of that group.
"""
name = models.SlugField(_("name"), max_length=64, blank=False)
display_name = models.CharField(_("display name"), max_length=64)
mailing_list = models.CharField(
_("mailing list"),
blank=True,
max_length=254,
help_text=_("The mailing list review requests and discussions "
"are sent to."))
email_list_only = models.BooleanField(
_('send e-mail only to the mailing list'),
default=True,
help_text=_('If a mailing list is specified and this option is '
'checked, group members will not be individually '
'included on e-mails, and only the mailing list '
'will be used. This is highly recommended for '
'large groups.'))
users = models.ManyToManyField(User,
blank=True,
related_name="review_groups",
verbose_name=_("users"))
local_site = models.ForeignKey(LocalSite,
blank=True,
null=True,
related_name='groups')
is_default_group = models.BooleanField(
_('add new users by default'),
default=False,
help_text=_('If a local site is set, this will automatically add '
'users to this group when those users are added to the '
'local site. If there is no local site, users will be '
'automatically added to this group when they are '
'registered.'))
incoming_request_count = CounterField(
_('incoming review request count'),
initializer=_initialize_incoming_request_count)
invite_only = models.BooleanField(
_('invite only'),
default=False,
help_text=_('If checked, only the users listed below will be able '
'to view review requests sent to this group.'))
visible = models.BooleanField(default=True)
extra_data = JSONField(null=True)
objects = ReviewGroupManager()
def is_accessible_by(self, user, request=None, silent=False):
"""Returns true if the user can access this group."""
if self.local_site and not self.local_site.is_accessible_by(user):
if not silent:
logging.warning(
'Group pk=%d (%s) is not accessible by user '
'%s because its local_site is not accessible '
'by that user.',
self.pk,
self.name,
user,
request=request)
return False
if not self.invite_only or user.is_superuser:
return True
if user.is_authenticated() and self.users.filter(pk=user.pk).exists():
return True
if not silent:
logging.warning(
'Group pk=%d (%s) is not accessible by user %s '
'because it is invite only, and the user is not a '
'member.',
self.pk,
self.name,
user,
request=request)
return False
def is_mutable_by(self, user):
"""Returns whether or not the user can modify or delete the group.
The group is mutable by the user if they are an administrator with
proper permissions, or the group is part of a LocalSite and the user is
in the admin list.
"""
return user.has_perm('reviews.change_group', self.local_site)
def __str__(self):
return self.name
def get_absolute_url(self):
if self.local_site_id:
local_site_name = self.local_site.name
else:
local_site_name = None
return local_site_reverse('group',
local_site_name=local_site_name,
kwargs={'name': self.name})
def clean(self):
"""Clean method for checking null unique_together constraints.
Django has a bug where unique_together constraints for foreign keys
aren't checked properly if one of the relations is null. This means
that users who aren't using local sites could create multiple groups
with the same name.
"""
super(Group, self).clean()
if (self.local_site is None and Group.objects.filter(
name=self.name).exclude(pk=self.pk).exists()):
raise ValidationError(_('A group with this name already exists'),
params={'field': 'name'})
class Meta:
app_label = 'reviews'
db_table = 'reviews_group'
unique_together = (('name', 'local_site'), )
verbose_name = _('Review Group')
verbose_name_plural = _('Review Groups')
ordering = ['name']
def setUp(self):
self.field = JSONField()
class JSONFieldTests(unittest.TestCase):
"""Unit tests for JSONField."""
def setUp(self):
self.field = JSONField()
def test_dumps_with_json_dict(self):
"""Testing JSONField with dumping a JSON dictionary"""
result = self.field.dumps({'a': 1})
self.assertTrue(isinstance(result, six.string_types))
self.assertEqual(result, '{"a": 1}')
def test_dumps_with_json_string(self):
"""Testing JSONField with dumping a JSON string"""
result = self.field.dumps('{"a": 1, "b": 2}')
self.assertTrue(isinstance(result, six.string_types))
self.assertEqual(result, '{"a": 1, "b": 2}')
def test_loading_json_dict(self):
"""Testing JSONField with loading a JSON dictionary"""
result = self.field.loads('{"a": 1, "b": 2}')
self.assertTrue(isinstance(result, dict))
self.assertTrue('a' in result)
self.assertTrue('b' in result)
def test_loading_json_broken_dict(self):
"""Testing JSONField with loading a badly serialized JSON dictionary"""
result = self.field.loads('{u"a": 1, u"b": 2}')
self.assertTrue(isinstance(result, dict))
self.assertTrue('a' in result)
self.assertTrue('b' in result)
def test_loading_json_array(self):
"""Testing JSONField with loading a JSON array"""
result = self.field.loads('[1, 2, 3]')
self.assertTrue(isinstance(result, list))
self.assertEqual(result, [1, 2, 3])
def test_loading_string(self):
"""Testing JSONField with loading a stored string"""
result = self.field.loads('"foo"')
self.assertTrue(isinstance(result, dict))
self.assertEqual(result, {})
def test_loading_broken_string(self):
"""Testing JSONField with loading a broken stored string"""
result = self.field.loads('u"foo"')
self.assertTrue(isinstance(result, dict))
self.assertEqual(result, {})
def test_loading_python_code(self):
"""Testing JSONField with loading Python code"""
result = self.field.loads('locals()')
self.assertTrue(isinstance(result, dict))
self.assertEqual(result, {})
def test_validate_with_valid_json_string(self):
"""Testing JSONField with validating a valid JSON string"""
self.field.run_validators('{"a": 1, "b": 2}')
def test_validate_with_invalid_json_string(self):
"""Testing JSONField with validating an invalid JSON string"""
self.assertRaises(ValidationError,
lambda: self.field.run_validators('foo'))
def test_validate_with_json_dict(self):
"""Testing JSONField with validating a JSON dictionary"""
self.field.run_validators({'a': 1, 'b': 2})
class Repository(models.Model):
ENCRYPTED_PASSWORD_PREFIX = '\t'
name = models.CharField(_('Name'), max_length=255)
path = models.CharField(_('Path'), max_length=255)
mirror_path = models.CharField(max_length=255, blank=True)
raw_file_url = models.CharField(
_('Raw file URL mask'),
max_length=255,
blank=True,
help_text=_("A URL mask used to check out a particular revision of a "
"file using HTTP. This is needed for repository types "
"that can't access remote files natively. "
"Use <tt><revision></tt> and "
"<tt><filename></tt> in the URL in place of the "
"revision and filename parts of the path."))
username = models.CharField(max_length=32, blank=True)
encrypted_password = models.CharField(max_length=128,
blank=True,
db_column='password')
extra_data = JSONField(null=True)
tool = models.ForeignKey(Tool, related_name="repositories")
hosting_account = models.ForeignKey(
HostingServiceAccount,
related_name='repositories',
verbose_name=_('Hosting service account'),
blank=True,
null=True)
bug_tracker = models.CharField(
_('Bug tracker URL'),
max_length=256,
blank=True,
help_text=_("This should be the full path to a bug in the bug tracker "
"for this repository, using '%s' in place of the bug ID."))
encoding = models.CharField(
max_length=32,
blank=True,
help_text=_("The encoding used for files in this repository. This is "
"an advanced setting and should only be used if you're "
"sure you need it."))
visible = models.BooleanField(
_('Show this repository'),
default=True,
help_text=_('Use this to control whether or not a repository is '
'shown when creating new review requests. Existing '
'review requests are unaffected.'))
archived = models.BooleanField(
_('Archived'),
default=False,
help_text=_("Archived repositories do not show up in lists of "
"repositories, and aren't open to new review requests."))
archived_timestamp = models.DateTimeField(null=True, blank=True)
# Access control
local_site = models.ForeignKey(LocalSite,
verbose_name=_('Local site'),
blank=True,
null=True)
public = models.BooleanField(
_('publicly accessible'),
default=True,
help_text=_('Review requests and files on public repositories are '
'visible to anyone. Private repositories must explicitly '
'list the users and groups that can access them.'))
users = models.ManyToManyField(
User,
limit_choices_to={'is_active': True},
blank=True,
related_name='repositories',
verbose_name=_('Users with access'),
help_text=_('A list of users with explicit access to the repository.'))
review_groups = models.ManyToManyField(
'reviews.Group',
limit_choices_to={'invite_only': True},
blank=True,
related_name='repositories',
verbose_name=_('Review groups with access'),
help_text=_('A list of invite-only review groups whose members have '
'explicit access to the repository.'))
hooks_uuid = models.CharField(
_('Hooks UUID'),
max_length=32,
null=True,
blank=True,
help_text=_('Unique identifier used for validating incoming '
'webhooks.'))
objects = RepositoryManager()
BRANCHES_CACHE_PERIOD = 60 * 5 # 5 minutes
COMMITS_CACHE_PERIOD_SHORT = 60 * 5 # 5 minutes
COMMITS_CACHE_PERIOD_LONG = 60 * 60 * 24 # 1 day
NAME_CONFLICT_ERROR = _('A repository with this name already exists')
PATH_CONFLICT_ERROR = _('A repository with this path already exists')
def _set_password(self, value):
"""Sets the password for the repository.
The password will be stored as an encrypted value, prefixed with a
tab character in order to differentiate between legacy plain-text
passwords.
"""
if value:
value = '%s%s' % (self.ENCRYPTED_PASSWORD_PREFIX,
encrypt_password(value))
else:
value = ''
self.encrypted_password = value
def _get_password(self):
"""Returns the password for the repository.
If a password is stored and encrypted, it will be decrypted and
returned.
If the stored password is in plain-text, then it will be encrypted,
stored in the database, and returned.
"""
password = self.encrypted_password
# NOTE: Due to a bug in 2.0.9, it was possible to get a string of
# "\tNone", indicating no password. We have to check for this.
if not password or password == '\tNone':
password = None
elif password.startswith(self.ENCRYPTED_PASSWORD_PREFIX):
password = password[len(self.ENCRYPTED_PASSWORD_PREFIX):]
if password:
password = decrypt_password(password)
else:
password = None
else:
# This is a plain-text password. Convert it.
self.password = password
self.save(update_fields=['encrypted_password'])
return password
password = property(_get_password, _set_password)
@property
def scmtool_class(self):
"""The SCMTool subclass used for this repository."""
return self.tool.get_scmtool_class()
def get_scmtool(self):
"""Return an instance of the SCMTool for this repository.
Returns:
reviewboard.scmtools.core.SCMTool:
A new instance of the SCMTool for this repository.
"""
return self.scmtool_class(self)
@cached_property
def hosting_service(self):
if self.hosting_account:
return self.hosting_account.service
return None
@cached_property
def bug_tracker_service(self):
"""Returns selected bug tracker service if one exists."""
if self.extra_data.get('bug_tracker_use_hosting'):
return self.hosting_service
else:
bug_tracker_type = self.extra_data.get('bug_tracker_type')
if bug_tracker_type:
bug_tracker_cls = get_hosting_service(bug_tracker_type)
# TODO: we need to figure out some way of storing a second
# hosting service account for bug trackers.
return bug_tracker_cls(HostingServiceAccount())
return None
@property
def supports_post_commit(self):
"""Whether or not this repository supports post-commit creation.
If this is ``True``, the :py:meth:`get_branches` and
:py:meth:`get_commits` methods will be implemented to fetch information
about the committed revisions, and get_change will be implemented to
fetch the actual diff. This is used by
:py:meth:`ReviewRequestDraft.update_from_commit_id
<reviewboard.reviews.models.ReviewRequestDraft.update_from_commit_id>`.
"""
hosting_service = self.hosting_service
if hosting_service:
return hosting_service.supports_post_commit
else:
return self.scmtool_class.supports_post_commit
@property
def supports_pending_changesets(self):
"""Whether this repository supports server-aware pending changesets."""
return self.scmtool_class.supports_pending_changesets
@property
def diffs_use_absolute_paths(self):
"""Whether or not diffs for this repository contain absolute paths.
Some types of source code management systems generate diffs that
contain paths relative to the directory where the diff was generated.
Most contain absolute paths. This flag indicates which path format
this repository can expect.
"""
# Ideally, we won't have to instantiate the class, as that can end up
# performing some expensive calls or HTTP requests. If the SCMTool is
# modern (doesn't define a get_diffs_use_absolute_paths), it will have
# all the information we need on the class. If not, we might have to
# instantiate it, but do this as a last resort.
scmtool_cls = self.scmtool_class
if isinstance(scmtool_cls.diffs_use_absolute_paths, bool):
return scmtool_cls.diffs_use_absolute_paths
elif hasattr(scmtool_cls, 'get_diffs_use_absolute_paths'):
# This will trigger a deprecation warning.
return self.get_scmtool().diffs_use_absolute_paths
else:
return False
def get_credentials(self):
"""Returns the credentials for this repository.
This returns a dictionary with 'username' and 'password' keys.
By default, these will be the values stored for the repository,
but if a hosting service is used and the repository doesn't have
values for one or both of these, the hosting service's credentials
(if available) will be used instead.
"""
username = self.username
password = self.password
if self.hosting_account and self.hosting_account.service:
username = username or self.hosting_account.username
password = password or self.hosting_account.service.get_password()
return {
'username': username,
'password': password,
}
def get_or_create_hooks_uuid(self, max_attempts=20):
"""Returns a hooks UUID, creating one if necessary.
If a hooks UUID isn't already saved, then this will try to generate one
that doesn't conflict with any other registered hooks UUID. It will try
up to `max_attempts` times, and if it fails, None will be returned.
"""
if not self.hooks_uuid:
for attempt in range(max_attempts):
self.hooks_uuid = uuid.uuid4().hex
try:
self.save(update_fields=['hooks_uuid'])
break
except IntegrityError:
# We hit a collision with the token value. Try again.
self.hooks_uuid = None
if not self.hooks_uuid:
s = ('Unable to generate a unique hooks UUID for '
'repository %s after %d attempts' %
(self.pk, max_attempts))
logging.error(s)
raise Exception(s)
return self.hooks_uuid
def archive(self, save=True):
"""Archive a repository.
The repository won't appear in any public lists of repositories,
and won't be used when looking up repositories. Review requests
can't be posted against an archived repository.
New repositories can be created with the same information as the
archived repository.
Args:
save (bool, optional):
Whether to save the repository immediately.
"""
# This should be sufficiently unlikely to create duplicates. time()
# will use up a max of 8 characters, so we slice the name down to
# make the result fit in 64 characters
max_name_len = self._meta.get_field('name').max_length
encoded_time = '%x' % int(time())
reserved_len = len('ar::') + len(encoded_time)
self.name = 'ar:%s:%s' % (self.name[:max_name_len - reserved_len],
encoded_time)
self.archived = True
self.public = False
self.archived_timestamp = timezone.now()
if save:
self.save(update_fields=('name', 'archived', 'public',
'archived_timestamp'))
def get_file(self, path, revision, base_commit_id=None, request=None):
"""Returns a file from the repository.
This will attempt to retrieve the file from the repository. If the
repository is backed by a hosting service, it will go through that.
Otherwise, it will attempt to directly access the repository.
"""
# We wrap the result of get_file in a list and then return the first
# element after getting the result from the cache. This prevents the
# cache backend from converting to unicode, since we're no longer
# passing in a string and the cache backend doesn't recursively look
# through the list in order to convert the elements inside.
#
# Basically, this fixes the massive regressions introduced by the
# Django unicode changes.
if not isinstance(path, six.text_type):
raise TypeError('"path" must be a Unicode string, not %s' %
type(path))
if not isinstance(revision, six.text_type):
raise TypeError('"revision" must be a Unicode string, not %s' %
type(revision))
if (base_commit_id is not None
and not isinstance(base_commit_id, six.text_type)):
raise TypeError('"base_commit_id" must be a Unicode string, '
'not %s' % type(base_commit_id))
return cache_memoize(
self._make_file_cache_key(path, revision, base_commit_id),
lambda:
[self._get_file_uncached(path, revision, base_commit_id, request)],
large_data=True)[0]
def get_file_exists(self,
path,
revision,
base_commit_id=None,
request=None):
"""Returns whether or not a file exists in the repository.
If the repository is backed by a hosting service, this will go
through that. Otherwise, it will attempt to directly access the
repository.
The result of this call will be cached, making future lookups
of this path and revision on this repository faster.
"""
if not isinstance(path, six.text_type):
raise TypeError('"path" must be a Unicode string, not %s' %
type(path))
if not isinstance(revision, six.text_type):
raise TypeError('"revision" must be a Unicode string, not %s' %
type(revision))
if (base_commit_id is not None
and not isinstance(base_commit_id, six.text_type)):
raise TypeError('"base_commit_id" must be a Unicode string, '
'not %s' % type(base_commit_id))
key = self._make_file_exists_cache_key(path, revision, base_commit_id)
if cache.get(make_cache_key(key)) == '1':
return True
exists = self._get_file_exists_uncached(path, revision, base_commit_id,
request)
if exists:
cache_memoize(key, lambda: '1')
return exists
def get_branches(self):
"""Returns a list of branches."""
hosting_service = self.hosting_service
cache_key = make_cache_key('repository-branches:%s' % self.pk)
if hosting_service:
branches_callable = lambda: hosting_service.get_branches(self)
else:
branches_callable = self.get_scmtool().get_branches
return cache_memoize(cache_key, branches_callable,
self.BRANCHES_CACHE_PERIOD)
def get_commit_cache_key(self, commit):
return 'repository-commit:%s:%s' % (self.pk, commit)
def get_commits(self, branch=None, start=None):
"""Returns a list of commits.
This is paginated via the 'start' parameter. Any exceptions are
expected to be handled by the caller.
"""
hosting_service = self.hosting_service
commits_kwargs = {
'branch': branch,
'start': start,
}
if hosting_service:
commits_callable = \
lambda: hosting_service.get_commits(self, **commits_kwargs)
else:
commits_callable = \
lambda: self.get_scmtool().get_commits(**commits_kwargs)
# We cache both the entire list for 'start', as well as each individual
# commit. This allows us to reduce API load when people are looking at
# the "new review request" page more frequently than they're pushing
# code, and will usually save 1 API request when they go to actually
# create a new review request.
if branch and start:
cache_period = self.COMMITS_CACHE_PERIOD_LONG
else:
cache_period = self.COMMITS_CACHE_PERIOD_SHORT
cache_key = make_cache_key('repository-commits:%s:%s:%s' %
(self.pk, branch, start))
commits = cache_memoize(cache_key, commits_callable, cache_period)
for commit in commits:
cache.set(self.get_commit_cache_key(commit.id), commit,
self.COMMITS_CACHE_PERIOD_LONG)
return commits
def get_change(self, revision):
"""Get an individual change.
This returns a tuple of (commit message, diff).
"""
hosting_service = self.hosting_service
if hosting_service:
return hosting_service.get_change(self, revision)
else:
return self.get_scmtool().get_change(revision)
def is_accessible_by(self, user):
"""Returns whether or not the user has access to the repository.
The repository is accessibly by the user if it is public or
the user has access to it (either by being explicitly on the allowed
users list, or by being a member of a review group on that list).
"""
if self.local_site and not self.local_site.is_accessible_by(user):
return False
return (self.public or user.is_superuser
or (user.is_authenticated() and
(self.review_groups.filter(users__pk=user.pk).exists()
or self.users.filter(pk=user.pk).exists())))
def is_mutable_by(self, user):
"""Returns whether or not the user can modify or delete the repository.
The repository is mutable by the user if the user is an administrator
with proper permissions or the repository is part of a LocalSite and
the user has permissions to modify it.
"""
return user.has_perm('scmtools.change_repository', self.local_site)
def save(self, **kwargs):
"""Saves the repository.
This will perform any data normalization needed, and then save the
repository to the database.
"""
# Prevent empty strings from saving in the admin UI, which could lead
# to database-level validation errors.
if self.hooks_uuid == '':
self.hooks_uuid = None
return super(Repository, self).save(**kwargs)
def __str__(self):
return self.name
def _make_file_cache_key(self, path, revision, base_commit_id):
"""Makes a cache key for fetched files."""
return 'file:%s:%s:%s:%s:%s' % (
self.pk, urlquote(path), urlquote(revision),
urlquote(base_commit_id or ''), urlquote(self.raw_file_url or ''))
def _make_file_exists_cache_key(self, path, revision, base_commit_id):
"""Makes a cache key for file existence checks."""
return 'file-exists:%s:%s:%s:%s:%s' % (
self.pk, urlquote(path), urlquote(revision),
urlquote(base_commit_id or ''), urlquote(self.raw_file_url or ''))
def _get_file_uncached(self, path, revision, base_commit_id, request):
"""Internal function for fetching an uncached file.
This is called by get_file if the file isn't already in the cache.
"""
fetching_file.send(sender=self,
path=path,
revision=revision,
base_commit_id=base_commit_id,
request=request)
if base_commit_id:
timer_msg = "Fetching file '%s' r%s (base commit ID %s) from %s" \
% (path, revision, base_commit_id, self)
else:
timer_msg = "Fetching file '%s' r%s from %s" \
% (path, revision, self)
log_timer = log_timed(timer_msg, request=request)
hosting_service = self.hosting_service
if hosting_service:
data = hosting_service.get_file(self,
path,
revision,
base_commit_id=base_commit_id)
assert isinstance(
data,
bytes), ('%s.get_file() must return a byte string, not %s' %
(type(hosting_service).__name__, type(data)))
else:
tool = self.get_scmtool()
data = tool.get_file(path, revision, base_commit_id=base_commit_id)
assert isinstance(
data,
bytes), ('%s.get_file() must return a byte string, not %s' %
(type(tool).__name__, type(data)))
log_timer.done()
fetched_file.send(sender=self,
path=path,
revision=revision,
base_commit_id=base_commit_id,
request=request,
data=data)
return data
def _get_file_exists_uncached(self, path, revision, base_commit_id,
request):
"""Internal function for checking that a file exists.
This is called by get_file_exists if the file isn't already in the
cache.
This function is smart enough to check if the file exists in cache,
and will use that for the result instead of making a separate call.
"""
# First we check to see if we've fetched the file before. If so,
# it's in there and we can just return that we have it.
file_cache_key = make_cache_key(
self._make_file_cache_key(path, revision, base_commit_id))
if file_cache_key in cache:
exists = True
else:
# We didn't have that in the cache, so check from the repository.
checking_file_exists.send(sender=self,
path=path,
revision=revision,
base_commit_id=base_commit_id,
request=request)
hosting_service = self.hosting_service
if hosting_service:
exists = hosting_service.get_file_exists(
self, path, revision, base_commit_id=base_commit_id)
else:
tool = self.get_scmtool()
exists = tool.file_exists(path,
revision,
base_commit_id=base_commit_id)
checked_file_exists.send(sender=self,
path=path,
revision=revision,
base_commit_id=base_commit_id,
request=request,
exists=exists)
return exists
def get_encoding_list(self):
"""Returns a list of candidate text encodings for files"""
encodings = []
for e in self.encoding.split(','):
e = e.strip()
if e:
encodings.append(e)
return encodings or ['iso-8859-15']
def clean(self):
"""Clean method for checking null unique_together constraints.
Django has a bug where unique_together constraints for foreign keys
aren't checked properly if one of the relations is null. This means
that users who aren't using local sites could create multiple groups
with the same name.
Raises:
django.core.exceptions.ValidationError:
Validation of the model's data failed. Details are in the
exception.
"""
super(Repository, self).clean()
if self.local_site is None:
existing_repos = (Repository.objects.exclude(pk=self.pk).filter(
Q(name=self.name)
| (Q(archived=False) & Q(path=self.path))).values(
'name', 'path'))
errors = {}
for repo_info in existing_repos:
if repo_info['name'] == self.name:
errors['name'] = [
ValidationError(self.NAME_CONFLICT_ERROR,
code='repository_name_exists'),
]
if repo_info['path'] == self.path:
errors['path'] = [
ValidationError(self.PATH_CONFLICT_ERROR,
code='repository_path_exists'),
]
if errors:
raise ValidationError(errors)
class Meta:
db_table = 'scmtools_repository'
unique_together = (('name', 'local_site'), ('archived_timestamp',
'path', 'local_site'),
('hooks_uuid', 'local_site'))
verbose_name = _('Repository')
verbose_name_plural = _('Repositories')
class Repository(models.Model):
name = models.CharField(max_length=64)
path = models.CharField(max_length=255)
mirror_path = models.CharField(max_length=255, blank=True)
raw_file_url = models.CharField(
_('Raw file URL mask'),
max_length=255,
blank=True,
help_text=_("A URL mask used to check out a particular revision of a "
"file using HTTP. This is needed for repository types "
"that can't access remote files natively. "
"Use <tt><revision></tt> and "
"<tt><filename></tt> in the URL in place of the "
"revision and filename parts of the path."))
username = models.CharField(max_length=32, blank=True)
password = models.CharField(max_length=128, blank=True)
extra_data = JSONField(null=True)
tool = models.ForeignKey(Tool, related_name="repositories")
hosting_account = models.ForeignKey(
HostingServiceAccount,
related_name='repositories',
verbose_name=_('Hosting service account'),
blank=True,
null=True)
bug_tracker = models.CharField(
_('Bug tracker URL'),
max_length=256,
blank=True,
help_text=_("This should be the full path to a bug in the bug tracker "
"for this repository, using '%s' in place of the bug ID."))
encoding = models.CharField(
max_length=32,
blank=True,
help_text=_("The encoding used for files in this repository. This is "
"an advanced setting and should only be used if you're "
"sure you need it."))
visible = models.BooleanField(
_('Show this repository'),
default=True,
help_text=_('Use this to control whether or not a repository is '
'shown when creating new review requests. Existing '
'review requests are unaffected.'))
# Access control
local_site = models.ForeignKey(LocalSite,
verbose_name=_('Local site'),
blank=True,
null=True)
public = models.BooleanField(
_('publicly accessible'),
default=True,
help_text=_('Review requests and files on public repositories are '
'visible to anyone. Private repositories must explicitly '
'list the users and groups that can access them.'))
users = models.ManyToManyField(
User,
limit_choices_to={'is_active': True},
blank=True,
related_name='repositories',
verbose_name=_('Users with access'),
help_text=_('A list of users with explicit access to the repository.'))
review_groups = models.ManyToManyField(
'reviews.Group',
limit_choices_to={'invite_only': True},
blank=True,
related_name='repositories',
verbose_name=_('Review groups with access'),
help_text=_('A list of invite-only review groups whose members have '
'explicit access to the repository.'))
objects = RepositoryManager()
BRANCHES_CACHE_PERIOD = 60 * 5 # 5 minutes
COMMITS_CACHE_PERIOD = 60 * 60 * 24 # 1 day
def get_scmtool(self):
cls = self.tool.get_scmtool_class()
return cls(self)
@property
def hosting_service(self):
if self.hosting_account:
return self.hosting_account.service
return None
@property
def supports_post_commit(self):
"""Whether or not this repository supports post-commit creation.
If this is True, the get_branches and get_commits methods will be
implemented to fetch information about the committed revisions, and
get_change will be implemented to fetch the actual diff. This is used
by ReviewRequest.update_from_commit_id.
"""
hosting_service = self.hosting_service
if hosting_service:
return hosting_service.supports_post_commit
else:
return self.get_scmtool().supports_post_commit
def get_credentials(self):
"""Returns the credentials for this repository.
This returns a dictionary with 'username' and 'password' keys.
By default, these will be the values stored for the repository,
but if a hosting service is used and the repository doesn't have
values for one or both of these, the hosting service's credentials
(if available) will be used instead.
"""
username = self.username
password = self.password
if self.hosting_account and self.hosting_account.service:
username = username or self.hosting_account.username
password = password or self.hosting_account.service.get_password()
return {
'username': username,
'password': password,
}
def get_file(self, path, revision, base_commit_id=None, request=None):
"""Returns a file from the repository.
This will attempt to retrieve the file from the repository. If the
repository is backed by a hosting service, it will go through that.
Otherwise, it will attempt to directly access the repository.
"""
# We wrap the result of get_file in a list and then return the first
# element after getting the result from the cache. This prevents the
# cache backend from converting to unicode, since we're no longer
# passing in a string and the cache backend doesn't recursively look
# through the list in order to convert the elements inside.
#
# Basically, this fixes the massive regressions introduced by the
# Django unicode changes.
return cache_memoize(
self._make_file_cache_key(path, revision, base_commit_id),
lambda:
[self._get_file_uncached(path, revision, base_commit_id, request)],
large_data=True)[0]
def get_file_exists(self,
path,
revision,
base_commit_id=None,
request=None):
"""Returns whether or not a file exists in the repository.
If the repository is backed by a hosting service, this will go
through that. Otherwise, it will attempt to directly access the
repository.
The result of this call will be cached, making future lookups
of this path and revision on this repository faster.
"""
key = self._make_file_exists_cache_key(path, revision, base_commit_id)
if cache.get(make_cache_key(key)) == '1':
return True
exists = self._get_file_exists_uncached(path, revision, base_commit_id,
request)
if exists:
cache_memoize(key, lambda: '1')
return exists
def get_branches(self):
"""Returns a list of branches."""
hosting_service = self.hosting_service
cache_key = make_cache_key('repository-branches:%s' % self.pk)
if hosting_service:
branches_callable = lambda: hosting_service.get_branches(self)
else:
branches_callable = self.get_scmtool().get_branches
return cache_memoize(cache_key, branches_callable,
self.BRANCHES_CACHE_PERIOD)
def get_commit_cache_key(self, commit):
return 'repository-commit:%s:%s' % (self.pk, commit)
def get_commits(self, start=None):
"""Returns a list of commits.
This is paginated via the 'start' parameter. Any exceptions are
expected to be handled by the caller.
"""
hosting_service = self.hosting_service
cache_key = make_cache_key('repository-commits:%s:%s' %
(self.pk, start))
if hosting_service:
commits_callable = lambda: hosting_service.get_commits(self, start)
else:
commits_callable = lambda: self.get_scmtool().get_commits(start)
# We cache both the entire list for 'start', as well as each individual
# commit. This allows us to reduce API load when people are looking at
# the "new review request" page more frequently than they're pushing
# code, and will usually save 1 API request when they go to actually
# create a new review request.
commits = cache_memoize(cache_key, commits_callable)
for commit in commits:
cache.set(self.get_commit_cache_key(commit.id), commit,
self.COMMITS_CACHE_PERIOD)
return commits
def get_change(self, revision):
"""Get an individual change.
This returns a tuple of (commit message, diff).
"""
hosting_service = self.hosting_service
if hosting_service:
return hosting_service.get_change(self, revision)
else:
return self.get_scmtool().get_change(revision)
def is_accessible_by(self, user):
"""Returns whether or not the user has access to the repository.
The repository is accessibly by the user if it is public or
the user has access to it (either by being explicitly on the allowed
users list, or by being a member of a review group on that list).
"""
if self.local_site and not self.local_site.is_accessible_by(user):
return False
return (self.public or user.is_superuser
or (user.is_authenticated() and
(self.review_groups.filter(users__pk=user.pk).count() > 0
or self.users.filter(pk=user.pk).count() > 0)))
def is_mutable_by(self, user):
"""Returns whether or not the user can modify or delete the repository.
The repository is mutable by the user if the user is an administrator
with proper permissions or the repository is part of a LocalSite and
the user has permissions to modify it.
"""
return user.has_perm('scmtools.change_repository', self.local_site)
def __str__(self):
return self.name
def _make_file_cache_key(self, path, revision, base_commit_id):
"""Makes a cache key for fetched files."""
return "file:%s:%s:%s:%s" % (self.pk, urlquote(path),
urlquote(revision),
urlquote(base_commit_id or ''))
def _make_file_exists_cache_key(self, path, revision, base_commit_id):
"""Makes a cache key for file existence checks."""
return "file-exists:%s:%s:%s:%s" % (self.pk, urlquote(path),
urlquote(revision),
urlquote(base_commit_id or ''))
def _get_file_uncached(self, path, revision, base_commit_id, request):
"""Internal function for fetching an uncached file.
This is called by get_file if the file isn't already in the cache.
"""
fetching_file.send(sender=self,
path=path,
revision=revision,
base_commit_id=base_commit_id,
request=request)
if base_commit_id:
timer_msg = "Fetching file '%s' r%s (base commit ID %s) from %s" \
% (path, revision, base_commit_id, self)
else:
timer_msg = "Fetching file '%s' r%s from %s" \
% (path, revision, self)
log_timer = log_timed(timer_msg, request=request)
hosting_service = self.hosting_service
if hosting_service:
data = hosting_service.get_file(self,
path,
revision,
base_commit_id=base_commit_id)
else:
data = self.get_scmtool().get_file(path, revision)
log_timer.done()
fetched_file.send(sender=self,
path=path,
revision=revision,
base_commit_id=base_commit_id,
request=request,
data=data)
return data
def _get_file_exists_uncached(self, path, revision, base_commit_id,
request):
"""Internal function for checking that a file exists.
This is called by get_file_eixsts if the file isn't already in the
cache.
This function is smart enough to check if the file exists in cache,
and will use that for the result instead of making a separate call.
"""
# First we check to see if we've fetched the file before. If so,
# it's in there and we can just return that we have it.
file_cache_key = make_cache_key(
self._make_file_cache_key(path, revision, base_commit_id))
if file_cache_key in cache:
exists = True
else:
# We didn't have that in the cache, so check from the repository.
checking_file_exists.send(sender=self,
path=path,
revision=revision,
base_commit_id=base_commit_id,
request=request)
hosting_service = self.hosting_service
if hosting_service:
exists = hosting_service.get_file_exists(
self, path, revision, base_commit_id=base_commit_id)
else:
exists = self.get_scmtool().file_exists(path, revision)
checked_file_exists.send(sender=self,
path=path,
revision=revision,
base_commit_id=base_commit_id,
request=request,
exists=exists)
return exists
def get_encoding_list(self):
"""Returns a list of candidate text encodings for files"""
return self.encoding.split(',') or ['iso-8859-15']
def clean(self):
"""Clean method for checking null unique_together constraints.
Django has a bug where unique_together constraints for foreign keys
aren't checked properly if one of the relations is null. This means
that users who aren't using local sites could create multiple groups
with the same name.
"""
super(Repository, self).clean()
if self.local_site is None:
q = Repository.objects.exclude(pk=self.pk)
if q.filter(name=self.name).exists():
raise ValidationError(
_('A repository with this name already exists'),
params={'field': 'name'})
if q.filter(path=self.path).exists():
raise ValidationError(
_('A repository with this path already exists'),
parames={'field': 'path'})
class Meta:
verbose_name_plural = "Repositories"
# TODO: the path:local_site unique constraint causes problems when
# archiving repositories. We should really remove this constraint from
# the tables and enforce it in code whenever visible=True
unique_together = (('name', 'local_site'), ('path', 'local_site'))
def setUp(self):
self.field = JSONField()
class ChangeDescription(models.Model):
"""
The recorded set of changes, containing optional description text
and fields that have changed.
This is a general model that can be used in applications for recording
changes how they see fit. A helper function, 'record_field_changed',
can be used to record information in a standard way for most value types,
but the 'fields_changed' dictionary can be manipulated however the caller
chooses.
A ChangeDescription is not bound to a particular model. It is up to models
to establish relationships with a ChangeDescription.
Each field in 'fields_changed' represents a changed field.
For string fields, the following fields will be available:
* 'old': The old value of the field
* 'new': The new value of the field
For list and set fields, the following fields will be available:
* 'removed': The fields that were removed, if any.
* 'added': The fields that were added, if any.
"""
timestamp = models.DateTimeField(_('timestamp'), default=timezone.now)
public = models.BooleanField(_("public"), default=False)
text = models.TextField(_("change text"), blank=True)
rich_text = models.BooleanField(_("rich text"), default=True)
fields_changed = JSONField(_("fields changed"))
def record_field_change(self,
field,
old_value,
new_value,
name_field=None):
"""
Records a field change.
This will encode field changes following the rules in the overlying
'ChangeDescription' documentation.
'name_field' can be specified for lists or other iterables. When
specified, each list item will be a tuple in the form of
(object_name, object_url, object_id). Otherwise, it will be a
tuple in the form of (item,).
It is generally expected that fields with lists of model objects will
have 'name_field' set, whereas lists of numbers or some other
value type will not. Specifying a 'name_field' for non-objects will
cause an AttributeError.
"""
def serialize_changed_obj_list(items, name_field):
if name_field:
return [(getattr(item,
name_field), item.get_absolute_url(), item.id)
for item in list(items)]
else:
return [(item, ) for item in list(items)]
if (type(old_value) != type(new_value)
and not (isinstance(old_value, six.string_types)
and isinstance(new_value, six.string_types))
and old_value is not None and new_value is not None):
raise ValueError(
"%s (%s) and %s (%s) are of two different value "
"types." %
(old_value, type(old_value), new_value, type(new_value)))
if hasattr(old_value, "__iter__"):
old_set = set(old_value)
new_set = set(new_value)
self.fields_changed[field] = {
'old':
serialize_changed_obj_list(old_value, name_field),
'new':
serialize_changed_obj_list(new_value, name_field),
'added':
serialize_changed_obj_list(new_set - old_set, name_field),
'removed':
serialize_changed_obj_list(old_set - new_set, name_field),
}
else:
self.fields_changed[field] = {
'old': (old_value, ),
'new': (new_value, ),
}
def __str__(self):
return self.text
def has_modified_fields(self):
"""Determines if the 'fields_changed' variable is non-empty
Uses the 'fields_changed' variable to determine if there are any
current modifications being tracked to this ChangedDescription object.
"""
return bool(self.fields_changed)
class Meta:
ordering = ['-timestamp']
get_latest_by = "timestamp"
class JSONFieldTests(TestCase):
"""Unit tests for JSONField."""
def setUp(self):
self.field = JSONField()
def test_init_with_custom_encoder_class(self):
"""Testing JSONField initialization with custom encoder class"""
class MyEncoder(json.JSONEncoder):
def __init__(self, default_msg, **kwargs):
self.default_msg = default_msg
super(MyEncoder, self).__init__(**kwargs)
def default(self, o):
return self.default_msg
with warnings.catch_warnings(record=True) as w:
field = JSONField(
encoder_cls=MyEncoder,
encoder_kwargs={
'default_msg': 'What even is this?',
})
self.assertEqual(field.dumps(MyEncoder), '"What even is this?"')
self.assertEqual(len(w), 0)
def test_init_with_custom_encoder_instance(self):
"""Testing JSONField initialization with deprecated custom encoder
instance
"""
class MyEncoder(json.JSONEncoder):
def default(self, o):
return 'What even is this?'
with warnings.catch_warnings(record=True) as w:
field = JSONField(encoder=MyEncoder())
self.assertEqual(field.dumps(MyEncoder), '"What even is this?"')
self.assertEqual(len(w), 1)
message = w[0].message
self.assertIsInstance(message, RemovedInDjblets20Warning)
self.assertEqual(six.text_type(message),
'The encoder argument to JSONField has been '
'replaced by the encoder_cls and encoder_kwargs '
'arguments. Support for encoder is deprecated.')
def test_init_with_dict_value(self):
"""Testing JSONField initialization with initial dict value"""
class MyModel(Model):
myfield = JSONField()
value = {
'a': 1,
'b': 2,
}
model = MyModel(myfield=value)
# Make sure we're working with a copy in the model.
value['c'] = 3
self.assertEqual(model.myfield, {
'a': 1,
'b': 2,
})
self.assertEqual(model.get_myfield_json(),
'{"a": 1, "b": 2}')
def test_init_with_dict_value_empty(self):
"""Testing JSONField initialization with initial empty dict value"""
class MyModel(Model):
myfield = JSONField()
model = MyModel(myfield={})
self.assertEqual(model.myfield, {})
self.assertEqual(model.get_myfield_json(), '{}')
def test_init_with_list_value(self):
"""Testing JSONField initialization with initial list value"""
class MyModel(Model):
myfield = JSONField()
value = [1, 2]
model = MyModel(myfield=value)
# Make sure we're working with a copy in the model.
value.append(3)
self.assertEqual(model.myfield, [1, 2])
self.assertEqual(model.get_myfield_json(), '[1, 2]')
def test_init_with_list_value_empty(self):
"""Testing JSONField initialization with initial empty list value"""
class MyModel(Model):
myfield = JSONField()
model = MyModel(myfield=[])
self.assertEqual(model.myfield, [])
self.assertEqual(model.get_myfield_json(), '[]')
def test_init_with_json_string_value(self):
"""Testing JSONField initialization with initial JSON string value"""
class MyModel(Model):
myfield = JSONField()
model = MyModel(myfield='{"a": 1, "b": 2}')
self.assertEqual(model.myfield, {
'a': 1,
'b': 2,
})
self.assertEqual(model.get_myfield_json(),
'{"a": 1, "b": 2}')
def test_init_with_json_string_value_empty(self):
"""Testing JSONField initialization with initial empty JSON string
value
"""
class MyModel(Model):
myfield = JSONField()
model = MyModel(myfield='')
self.assertEqual(model.myfield, {})
self.assertEqual(model.get_myfield_json(), '{}')
def test_init_with_bad_type(self):
"""Testing JSONField initialization with initial unsupported value
type
"""
class MyModel(Model):
myfield = JSONField()
message = (
"<class 'djblets.db.tests.test_json_field.MyModel'> is not a "
"supported value type."
)
with self.assertRaisesMessage(ValidationError, message):
MyModel(myfield=MyModel())
def test_dumps_with_json_dict(self):
"""Testing JSONField with dumping a JSON dictionary"""
result = self.field.dumps({'a': 1, 'b': 2})
self.assertTrue(isinstance(result, six.string_types))
self.assertEqual(result, '{"a": 1, "b": 2}')
def test_dumps_with_json_string(self):
"""Testing JSONField with dumping a JSON string"""
result = self.field.dumps('{"a": 1, "b": 2}')
self.assertTrue(isinstance(result, six.string_types))
self.assertEqual(result, '{"a": 1, "b": 2}')
def test_loading_json_dict(self):
"""Testing JSONField with loading a JSON dictionary"""
result = self.field.loads('{"a": 1, "b": 2}')
self.assertTrue(isinstance(result, dict))
self.assertTrue('a' in result)
self.assertTrue('b' in result)
def test_loading_json_broken_dict(self):
"""Testing JSONField with loading a badly serialized JSON dictionary"""
result = self.field.loads('{u"a": 1, u"b": 2}')
self.assertTrue(isinstance(result, dict))
self.assertTrue('a' in result)
self.assertTrue('b' in result)
def test_loading_json_array(self):
"""Testing JSONField with loading a JSON array"""
result = self.field.loads('[1, 2, 3]')
self.assertTrue(isinstance(result, list))
self.assertEqual(result, [1, 2, 3])
def test_loading_string(self):
"""Testing JSONField with loading a stored string"""
result = self.field.loads('"foo"')
self.assertTrue(isinstance(result, dict))
self.assertEqual(result, {})
def test_loading_broken_string(self):
"""Testing JSONField with loading a broken stored string"""
result = self.field.loads('u"foo"')
self.assertTrue(isinstance(result, dict))
self.assertEqual(result, {})
def test_loading_python_code(self):
"""Testing JSONField with loading Python code"""
result = self.field.loads('locals()')
self.assertTrue(isinstance(result, dict))
self.assertEqual(result, {})
def test_get_json(self):
"""Testing JSONField with get_{fieldname}_json"""
class MyModel(Model):
myfield = JSONField()
model = MyModel()
model.myfield = {
'a': 1,
'b': 2,
}
self.assertEqual(model.get_myfield_json(),
'{"a": 1, "b": 2}')
def test_set_json(self):
"""Testing JSONField with set_{fieldname}_json"""
class MyModel(Model):
myfield = JSONField()
model = MyModel()
model.set_myfield_json('{"a": 1, "b": 2}')
self.assertEqual(
model.myfield,
{
'a': 1,
'b': 2,
})
def test_validate_with_valid_json_string(self):
"""Testing JSONField with validating a valid JSON string"""
self.field.run_validators('{"a": 1, "b": 2}')
def test_validate_with_invalid_json_string(self):
"""Testing JSONField with validating an invalid JSON string"""
self.assertRaises(ValidationError,
lambda: self.field.run_validators('foo'))
def test_validate_with_json_dict(self):
"""Testing JSONField with validating a JSON dictionary"""
self.field.run_validators({'a': 1, 'b': 2})
