class AssetVersion(models.Model):
uid = KpiUidField(uid_prefix='v')
asset = models.ForeignKey('Asset',
related_name='asset_versions',
on_delete=models.CASCADE)
name = models.CharField(null=True, max_length=255)
date_modified = models.DateTimeField(default=timezone.now)
# preserving _reversion_version in case we don't save all that we
# need to in the first migration from reversion to AssetVersion
_reversion_version = models.OneToOneField(
Version,
null=True,
on_delete=models.SET_NULL,
)
version_content = JSONBField()
uid_aliases = JSONBField(null=True)
deployed_content = JSONBField(null=True)
_deployment_data = JSONBField(default=dict)
deployed = models.BooleanField(default=False)
class Meta:
ordering = ['-date_modified']
def _deployed_content(self):
if self.deployed_content is not None:
return self.deployed_content
legacy_names = self._reversion_version is not None
if legacy_names:
return to_xlsform_structure(self.version_content,
deprecated_autoname=True)
else:
return to_xlsform_structure(self.version_content,
move_autonames=True)
def to_formpack_schema(self):
return {
'content': expand_content(self._deployed_content()),
'version': self.uid,
'version_id_key': '__version__',
}
@property
def content_hash(self):
# used to determine changes in the content from version to version
# not saved, only compared with other asset_versions
_json_string = json.dumps(self.version_content, sort_keys=True)
return hashlib.sha1(hashable_str(_json_string)).hexdigest()
def __str__(self):
return '{}@{} T{}{}'.format(
self.asset.uid, self.uid,
self.date_modified.strftime('%Y-%m-%d %H:%M'),
' (deployed)' if self.deployed else '')
class KobocatUserProfile(ShadowModel):
"""
From onadata/apps/main/models/user_profile.py
Not read-only because we need write access to `require_auth`
"""
class Meta(ShadowModel.Meta):
db_table = 'main_userprofile'
verbose_name = 'user profile'
verbose_name_plural = 'user profiles'
# This field is required.
user = models.OneToOneField(KobocatUser,
related_name='profile',
on_delete=models.CASCADE)
# Other fields here
name = models.CharField(max_length=255, blank=True)
city = models.CharField(max_length=255, blank=True)
country = models.CharField(max_length=2, blank=True)
organization = models.CharField(max_length=255, blank=True)
home_page = models.CharField(max_length=255, blank=True)
twitter = models.CharField(max_length=255, blank=True)
description = models.CharField(max_length=255, blank=True)
require_auth = models.BooleanField(
default=False,
verbose_name="Require authentication to see forms and submit data")
address = models.CharField(max_length=255, blank=True)
phonenumber = models.CharField(max_length=30, blank=True)
created_by = models.ForeignKey(User,
null=True,
blank=True,
on_delete=models.CASCADE)
num_of_submissions = models.IntegerField(default=0)
metadata = JSONBField(default=dict, blank=True)
class ExtraUserDetail(models.Model):
user = models.OneToOneField(settings.AUTH_USER_MODEL,
related_name='extra_details',
on_delete=models.CASCADE)
data = JSONBField(default=dict)
def __str__(self):
return '{}\'s data: {}'.format(self.user.__str__(), repr(self.data))
class Migration(migrations.Migration):
dependencies = [
('hub', '0006_remove_formbuilder_preference_table'),
]
operations = [
migrations.AlterField(
model_name='extrauserdetail',
name='data',
field=JSONBField(default=dict),
),
migrations.AlterField(
model_name='perusersetting',
name='user_queries',
field=JSONBField(
help_text=
'A JSON representation of a *list* of Django queries, e.g. `[{"email__iendswith": "@kobotoolbox.org"}, {"email__iendswith": "@kbtdev.org"}]`. A matching user is one who would be returned by ANY of the queries in the list.'
),
),
]
class PerUserSetting(models.Model):
"""
A configuration setting that has different values depending on whether not
a user matches certain criteria
"""
user_queries = JSONBField(
help_text='A JSON representation of a *list* of Django queries, '
'e.g. `[{"email__iendswith": "@kobotoolbox.org"}, '
'{"email__iendswith": "@kbtdev.org"}]`. '
'A matching user is one who would be returned by ANY of '
'the queries in the list.'
)
name = models.CharField(max_length=255, unique=True,
default='INTERCOM_APP_ID') # The only one for now!
value_when_matched = models.CharField(max_length=2048, blank=True)
value_when_not_matched = models.CharField(max_length=2048, blank=True)
def user_matches(self, user, ignore_invalid_queries=True):
if user.is_anonymous:
user = get_anonymous_user()
manager = user._meta.model.objects
queryset = manager.none()
for user_query in self.user_queries:
try:
queryset |= manager.filter(**user_query)
except (FieldError, TypeError):
if ignore_invalid_queries:
return False
else:
raise
return queryset.filter(pk=user.pk).exists()
def get_for_user(self, user):
if self.user_matches(user):
return self.value_when_matched
else:
return self.value_when_not_matched
def clean(self):
user = User.objects.first()
if not user:
return
try:
self.user_matches(user, ignore_invalid_queries=False)
except FieldError as e:
raise ValidationError({'user_queries': e.message})
except TypeError:
raise ValidationError(
{'user_queries': 'JSON structure is incorrect.'})
def __str__(self):
return self.name
class Migration(migrations.Migration):
dependencies = [
('kpi', '0016_asset_settings'),
]
operations = [
migrations.AddField(
model_name='assetversion',
name='uid_aliases',
field=JSONBField(null=True),
),
]
class Migration(migrations.Migration):
dependencies = [
('kpi', '0015_assetversion'),
]
operations = [
migrations.AddField(
model_name='asset',
name='settings',
field=JSONBField(default=dict),
),
]
class Migration(migrations.Migration):
dependencies = [
('kpi', '0018_export_task'),
]
operations = [
migrations.AddField(
model_name='asset',
name='report_custom',
field=JSONBField(default=dict),
),
]
class KobocatUserProfile(ShadowModel):
"""
From onadata/apps/main/models/user_profile.py
Not read-only because we need write access to `require_auth`
"""
class Meta(ShadowModel.Meta):
db_table = 'main_userprofile'
verbose_name = 'user profile'
verbose_name_plural = 'user profiles'
# This field is required.
user = models.OneToOneField(KobocatUser,
related_name='profile',
on_delete=models.CASCADE)
# Other fields here
name = models.CharField(max_length=255, blank=True)
city = models.CharField(max_length=255, blank=True)
country = models.CharField(max_length=2, blank=True)
organization = models.CharField(max_length=255, blank=True)
home_page = models.CharField(max_length=255, blank=True)
twitter = models.CharField(max_length=255, blank=True)
description = models.CharField(max_length=255, blank=True)
require_auth = models.BooleanField(
default=False,
verbose_name="Require authentication to see forms and submit data"
)
address = models.CharField(max_length=255, blank=True)
phonenumber = models.CharField(max_length=30, blank=True)
created_by = models.ForeignKey(KobocatUser, null=True, blank=True,
on_delete=models.CASCADE)
num_of_submissions = models.IntegerField(default=0)
metadata = JSONBField(default=dict, blank=True)
# We need to cast `is_active` to an (positive small) integer because KoBoCAT
# is using `LazyBooleanField` which is an integer behind the scene.
# We do not want to port this class to KPI only for one line of code.
is_mfa_active = models.PositiveSmallIntegerField(default=False)
@classmethod
def set_mfa_status(cls, user_id: int, is_active: bool):
try:
user_profile, created = cls.objects.get_or_create(user_id=user_id)
except cls.DoesNotExist:
pass
else:
user_profile.is_mfa_active = int(is_active)
user_profile.save(update_fields=['is_mfa_active'])
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('kpi', '0021_map-custom-styles'),
]
operations = [
migrations.CreateModel(
name='AssetFile',
fields=[
('id',
models.AutoField(verbose_name='ID',
serialize=False,
auto_created=True,
primary_key=True)),
('uid', kpi.fields.KpiUidField(uid_prefix=b'af')),
('file_type',
models.CharField(max_length=32,
choices=[(b'map_layer', b'map_layer')])),
('name', models.CharField(max_length=255)),
('date_created',
models.DateTimeField(default=django.utils.timezone.now)),
('content',
private_storage.fields.PrivateFileField(
storage=private_storage.storage.files.
PrivateFileSystemStorage(),
max_length=380,
upload_to=kpi.models.asset_file.upload_to)),
('metadata', JSONBField(default=dict)),
],
),
# Why did `manage.py makemigrations` create these as separate operations?
migrations.AddField(
model_name='assetfile',
name='asset',
field=models.ForeignKey(related_name='asset_files',
to='kpi.Asset',
on_delete=models.CASCADE),
),
migrations.AddField(
model_name='assetfile',
name='user',
field=models.ForeignKey(related_name='asset_files',
to=settings.AUTH_USER_MODEL,
on_delete=models.CASCADE),
),
]
class InAppMessageUserInteractions(models.Model):
message = models.ForeignKey(InAppMessage, on_delete=models.CASCADE)
user = models.ForeignKey(settings.AUTH_USER_MODEL,
on_delete=models.CASCADE)
interactions = JSONBField(default=dict)
class Meta:
unique_together = ('message', 'user')
def __str__(self):
return '{} with {} ({}): {}'.format(
self.user.username,
self.message.title,
self.message.uid,
self.interactions,
)
class Migration(migrations.Migration):
dependencies = [
('kpi', '0022_assetfile'),
]
operations = [
migrations.CreateModel(
name='Hook',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('uid', kpi.fields.KpiUidField(uid_prefix=b'h')),
('name', models.CharField(max_length=255)),
('endpoint', models.CharField(max_length=500)),
('active', models.BooleanField(default=True)),
('export_type', models.CharField(default=b'json', max_length=10, choices=[(b'xml', b'xml'), (b'json', b'json')])),
('auth_level', models.CharField(default=b'no_auth', max_length=10, choices=[(b'no_auth', b'no_auth'), (b'basic_auth', b'basic_auth')])),
('settings', JSONBField(default=dict)),
('date_created', models.DateTimeField(default=django.utils.timezone.now)),
('date_modified', models.DateTimeField(default=django.utils.timezone.now)),
('asset', models.ForeignKey(related_name='hooks', to='kpi.Asset', on_delete=models.CASCADE)),
],
options={
'ordering': ['name'],
},
),
migrations.CreateModel(
name='HookLog',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('uid', kpi.fields.KpiUidField(uid_prefix=b'hl')),
('instance_id', models.IntegerField(default=0, db_index=True)),
('tries', models.PositiveSmallIntegerField(default=0)),
('status', models.PositiveSmallIntegerField(default=1)),
('status_code', models.IntegerField(default=None, null=True, blank=True)),
('message', models.TextField(default=b'')),
('date_created', models.DateTimeField(auto_now_add=True)),
('date_modified', models.DateTimeField(auto_now_add=True)),
('hook', models.ForeignKey(related_name='logs', to='hook.Hook', on_delete=models.CASCADE)),
],
options={
'ordering': ['-date_created'],
},
),
]
class Migration(migrations.Migration):
dependencies = [
('hub', '0004_configurationfile'),
]
operations = [
migrations.CreateModel(
name='PerUserSetting',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('user_queries', JSONBField(help_text='A JSON representation of a *list* of Django queries, e.g. `[{"email__endswith": "@kobotoolbox.org"}, {"email__endswith": "@kbtdev.org"}]`. A matching user is one who would be returned by ANY of the queries in the list.')),
('name', models.CharField(unique=True, max_length=255)),
('value_when_matched', models.CharField(max_length=2048, blank=True)),
('value_when_not_matched', models.CharField(max_length=2048, blank=True)),
],
),
]
class AssetExportSettings(models.Model):
uid = KpiUidField(uid_prefix='es')
asset = models.ForeignKey('Asset',
related_name='asset_export_settings',
on_delete=models.CASCADE)
date_modified = models.DateTimeField()
name = models.CharField(max_length=255, blank=True, default='')
export_settings = JSONBField(default=dict)
def save(self, *args, **kwargs):
self.date_modified = timezone.now()
super().save(*args, **kwargs)
class Meta:
ordering = ['-date_modified']
unique_together = ('asset', 'name')
def __str__(self):
return f'{self.name} {self.uid}'
class AssetFile(models.Model):
# More to come!
MAP_LAYER = 'map_layer'
TYPE_CHOICES = ((MAP_LAYER, MAP_LAYER), )
uid = KpiUidField(uid_prefix='af')
asset = models.ForeignKey('Asset',
related_name='asset_files',
on_delete=models.CASCADE)
# Keep track of the uploading user, who could be anyone with `change_asset`
# rights, not just the asset owner
user = models.ForeignKey('auth.User',
related_name='asset_files',
on_delete=models.CASCADE)
file_type = models.CharField(choices=TYPE_CHOICES, max_length=32)
name = models.CharField(max_length=255)
date_created = models.DateTimeField(default=timezone.now)
# TODO: Handle deletion! The file won't be deleted automatically when the
# object is removed from the database
content = PrivateFileField(upload_to=upload_to, max_length=380)
metadata = JSONBField(default=dict)
class AssetUserPartialPermission(models.Model):
"""
Many-to-Many table which provides users' permissions
on other users' submissions
For example,
- Asset:
- uid: aAAAAAA
- id: 1
- User:
- username: someuser
- id: 1
We want someuser to be able to view otheruser's submissions
Records should be
`permissions` is dict formatted as is:
asset_id | user_id | permissions
1 | 1 | {"someuser": ["view_submissions"]}
Using a list per user for permissions, gives the opportunity to add other permissions
such as `change_submissions`, `delete_submissions` for later purpose
"""
class Meta:
unique_together = [['asset', 'user']]
asset = models.ForeignKey('Asset',
related_name='asset_partial_permissions',
on_delete=models.CASCADE)
user = models.ForeignKey('auth.User',
related_name='user_partial_permissions',
on_delete=models.CASCADE)
permissions = JSONBField(default=dict)
date_created = models.DateTimeField(default=timezone.now)
date_modified = models.DateTimeField(default=timezone.now)
def save(self, *args, **kwargs):
if self.pk is not None:
self.date_modified = timezone.now()
super().save(*args, **kwargs)
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='InAppMessage',
fields=[
('id',
models.AutoField(verbose_name='ID',
serialize=False,
auto_created=True,
primary_key=True)),
('uid', kpi.fields.KpiUidField(uid_prefix='iam')),
('title', models.CharField(max_length=255)),
('snippet', markdownx.models.MarkdownxField()),
('body', markdownx.models.MarkdownxField()),
('published',
models.BooleanField(
default=False,
help_text=
'When published, this message appears to all users. It otherwise appears only to the last editor'
)),
('valid_from',
models.DateTimeField(
default=datetime.datetime(1970, 1, 1, 0, 0))),
('valid_until',
models.DateTimeField(
default=datetime.datetime(1970, 1, 1, 0, 0))),
('last_editor',
models.ForeignKey(to=settings.AUTH_USER_MODEL,
on_delete=models.CASCADE)),
],
),
migrations.CreateModel(
name='InAppMessageFile',
fields=[
('id',
models.AutoField(verbose_name='ID',
serialize=False,
auto_created=True,
primary_key=True)),
('content',
private_storage.fields.PrivateFileField(
storage=private_storage.storage.files.
PrivateFileSystemStorage(),
upload_to='__in_app_message/%Y/%m/%d/')),
],
),
migrations.CreateModel(
name='InAppMessageUserInteractions',
fields=[
('id',
models.AutoField(verbose_name='ID',
serialize=False,
auto_created=True,
primary_key=True)),
('interactions', JSONBField(default=dict)),
('message',
models.ForeignKey(to='help.InAppMessage',
on_delete=models.CASCADE)),
('user',
models.ForeignKey(to=settings.AUTH_USER_MODEL,
on_delete=models.CASCADE)),
],
),
migrations.AlterUniqueTogether(
name='inappmessageuserinteractions',
unique_together=set([('message', 'user')]),
),
]
class AssetSnapshot(
models.Model,
AbstractFormList,
XlsExportableMixin,
FormpackXLSFormUtilsMixin,
):
"""
This model serves as a cache of the XML that was exported by the installed
version of pyxform.
TODO: come up with a policy to clear this cache out.
DO NOT: depend on these snapshots existing for more than a day
until a policy is set.
Done with https://github.com/kobotoolbox/kpi/pull/2434.
Remove above lines when PR is merged
"""
xml = models.TextField()
source = JSONBField(default=dict)
details = JSONBField(default=dict)
owner = models.ForeignKey('auth.User', related_name='asset_snapshots',
null=True, on_delete=models.CASCADE)
asset = models.ForeignKey('Asset', null=True, on_delete=models.CASCADE)
_reversion_version_id = models.IntegerField(null=True)
asset_version = models.OneToOneField('AssetVersion',
on_delete=models.CASCADE,
null=True)
date_created = models.DateTimeField(auto_now_add=True)
uid = KpiUidField(uid_prefix='s')
@property
def content(self):
return self.source
@property
def description(self):
"""
Implements `OpenRosaFormListInterface.description`
"""
return self.asset.settings.get('description', '')
@property
def form_id(self):
"""
Implements `OpenRosaFormListInterface.form_id()`
"""
return self.uid
def get_download_url(self, request):
"""
Implements `OpenRosaFormListInterface.get_download_url()`
"""
return reverse(
viewname='assetsnapshot-detail',
format='xml',
kwargs={'uid': self.uid},
request=request
)
def get_manifest_url(self, request):
"""
Implements `OpenRosaFormListInterface.get_manifest_url()`
"""
return reverse(
viewname='assetsnapshot-manifest',
format='xml',
kwargs={'uid': self.uid},
request=request
)
@property
def md5_hash(self):
"""
Implements `OpenRosaFormListInterface.md5_hash()`
"""
return f'{calculate_hash(self.xml, prefix=True)}'
@property
def name(self):
"""
Implements `OpenRosaFormListInterface.name()`
"""
return self.asset.name
def save(self, *args, **kwargs):
if self.asset is not None:
# Previously, `self.source` was a nullable field. It must now
# either contain valid content or be an empty dictionary.
assert self.asset is not None
if not self.source:
if self.asset_version is None:
self.asset_version = self.asset.latest_version
self.source = self.asset_version.version_content
if self.owner is None:
self.owner = self.asset.owner
_note = self.details.pop('note', None)
_source = copy.deepcopy(self.source)
self._standardize(_source)
self._make_default_translation_first(_source)
self._strip_empty_rows(_source)
self._autoname(_source)
self._remove_empty_expressions(_source)
# TODO: move these inside `generate_xml_from_source()`?
_settings = _source.get('settings', {})
form_title = _settings.get('form_title')
id_string = _settings.get('id_string')
root_node_name = _settings.get('name')
self.xml, self.details = self.generate_xml_from_source(
_source,
include_note=_note,
root_node_name=root_node_name,
form_title=form_title,
id_string=id_string,
)
self.source = _source
return super().save(*args, **kwargs)
def generate_xml_from_source(self,
source,
include_note=False,
root_node_name=None,
form_title=None,
id_string=None):
if not root_node_name:
if self.asset and self.asset.uid:
root_node_name = self.asset.uid
else:
root_node_name = 'snapshot_xml'
if not form_title:
if self.asset and self.asset.name:
form_title = self.asset.name
else:
form_title = 'Snapshot XML'
if id_string is None:
id_string = root_node_name
if include_note and 'survey' in source:
_translations = source.get('translations', [])
_label = include_note
if len(_translations) > 0:
_label = [_label for t in _translations]
source['survey'].append({'type': 'note',
'name': 'prepended_note',
'label': _label})
source_copy = copy.deepcopy(source)
self._expand_kobo_qs(source_copy)
self._populate_fields_with_autofields(source_copy)
self._strip_kuids(source_copy)
allow_choice_duplicates(source_copy)
warnings = []
details = {}
try:
xml = FormPack({'content': source_copy},
root_node_name=root_node_name,
id_string=id_string,
title=form_title)[0].to_xml(warnings=warnings)
details.update({
'status': 'success',
'warnings': warnings,
})
except Exception as err:
err_message = str(err)
logging.error('Failed to generate xform for asset', extra={
'src': source,
'id_string': id_string,
'uid': self.uid,
'_msg': err_message,
'warnings': warnings,
})
xml = ''
details.update({
'status': 'failure',
'error_type': type(err).__name__,
'error': err_message,
'warnings': warnings,
})
return xml, details
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('kpi', '0022_assetfile'),
]
operations = [
migrations.CreateModel(
name='AssetUserPartialPermission',
fields=[
('id',
models.AutoField(verbose_name='ID',
serialize=False,
auto_created=True,
primary_key=True)),
('permissions', JSONBField(default=dict)),
('date_created',
models.DateTimeField(default=django.utils.timezone.now)),
('date_modified',
models.DateTimeField(default=django.utils.timezone.now)),
],
),
migrations.AlterModelOptions(
name='asset',
options={
'ordering': ('-date_modified', ),
'permissions':
(('view_asset', 'Can view asset'),
('share_asset', "Can change asset's sharing settings"),
('add_submissions', 'Can submit data to asset'),
('view_submissions', 'Can view submitted data for asset'),
('partial_submissions',
'Can make partial actions on submitted data for asset for specific users'
), ('change_submissions',
'Can modify submitted data for asset'),
('delete_submissions', 'Can delete submitted data for asset'),
('share_submissions',
"Can change sharing settings for asset's submitted data"),
('validate_submissions', 'Can validate submitted data asset'),
('from_kc_only', 'INTERNAL USE ONLY; DO NOT ASSIGN'))
},
),
migrations.AddField(
model_name='assetuserpartialpermission',
name='asset',
field=models.ForeignKey(related_name='asset_partial_permissions',
to='kpi.Asset',
on_delete=models.CASCADE),
),
migrations.AddField(
model_name='assetuserpartialpermission',
name='user',
field=models.ForeignKey(related_name='user_partial_permissions',
to=settings.AUTH_USER_MODEL,
on_delete=models.CASCADE),
),
migrations.AlterUniqueTogether(
name='assetuserpartialpermission',
unique_together=set([('asset', 'user')]),
),
]
class AssetFile(models.Model, AbstractFormMedia):
# More to come!
MAP_LAYER = 'map_layer'
FORM_MEDIA = 'form_media'
PAIRED_DATA = 'paired_data'
TYPE_CHOICES = (
(MAP_LAYER, MAP_LAYER),
(FORM_MEDIA, FORM_MEDIA),
(PAIRED_DATA, PAIRED_DATA),
)
ALLOWED_MIME_TYPES = {
FORM_MEDIA: (
'image',
'audio',
'video',
'text/csv',
'application/xml',
'application/zip',
),
PAIRED_DATA: ('application/xml', ),
MAP_LAYER: (
'text/csv',
'application/vnd.google-earth.kml+xml',
'application/vnd.google-earth.kmz',
'application/wkt',
'application/geo+json',
'application/json',
),
}
uid = KpiUidField(uid_prefix='af')
asset = models.ForeignKey('Asset',
related_name='asset_files',
on_delete=models.CASCADE)
# Keep track of the uploading user, who could be anyone with `change_asset`
# rights, not just the asset owner
user = models.ForeignKey('auth.User',
related_name='asset_files',
on_delete=models.CASCADE)
file_type = models.CharField(choices=TYPE_CHOICES, max_length=32)
description = models.CharField(max_length=255)
date_created = models.DateTimeField(default=timezone.now)
content = PrivateFileField(upload_to=upload_to, max_length=380, null=True)
metadata = JSONBField(default=dict)
date_deleted = models.DateTimeField(null=True, default=None)
date_modified = models.DateTimeField(default=timezone.now)
synced_with_backend = models.BooleanField(default=False)
@property
def backend_media_id(self):
"""
Implements `SyncBackendMediaInterface.backend_media_id()`
"""
return (self.metadata['redirect_url']
if self.is_remote_url else self.filename)
def delete(self, using=None, keep_parents=False, force=False):
# Delete object and files on storage if `force` is True or file type
# is anything else than 'form_media'
if force or self.file_type != self.FORM_MEDIA:
if not self.is_remote_url:
self.content.delete(save=False)
return super().delete(using=using, keep_parents=keep_parents)
# Otherwise, just flag the file as deleted.
self.date_deleted = timezone.now()
self.synced_with_backend = False
self.save(update_fields=['date_deleted', 'synced_with_backend'])
@property
def deleted_at(self):
"""
Implements:
- `SyncBackendMediaInterface.deleted_at()`
"""
return self.date_deleted
@property
def filename(self):
"""
Implements:
- `OpenRosaManifestInterface.filename()`
- `SyncBackendMediaInterface.filename()`
"""
self.set_filename()
return self.metadata['filename']
def get_download_url(self, request):
"""
Implements `OpenRosaManifestInterface.get_download_url()`
"""
return reverse('asset-file-content',
args=(self.asset.uid, self.uid),
request=request)
@staticmethod
def get_path(asset, file_type, filename):
return posixpath.join(asset.owner.username, 'asset_files', asset.uid,
file_type, filename)
@property
def md5_hash(self):
"""
Implements:
- `OpenRosaManifestInterface.md5_hash()`
- `SyncBackendMediaInterface.md5_hash()`
"""
if not self.metadata.get('hash'):
self.set_md5_hash()
return self.metadata['hash']
@property
def is_remote_url(self):
"""
Implements `SyncBackendMediaInterface.is_remote_url()`
"""
try:
self.metadata['redirect_url']
except KeyError:
return False
return True
@property
def mimetype(self):
"""
Implements `SyncBackendMediaInterface.mimetype()`
"""
self.set_mimetype()
return self.metadata['mimetype']
def save(self,
force_insert=False,
force_update=False,
using=None,
update_fields=None):
if self.pk is None:
self.set_filename()
self.set_md5_hash()
self.set_mimetype()
else:
self.date_modified = timezone.now()
return super().save(force_insert, force_update, using, update_fields)
def set_filename(self):
if not self.metadata.get('filename'):
self.metadata['filename'] = self.content.name
def set_md5_hash(self, md5_hash: Optional[str] = None):
"""
Calculate md5 hash and store it in `metadata` field if it does not exist
or empty.
Value can be also set with the optional `md5_hash` parameter.
If `md5_hash` is an empty string, the hash is recalculated.
"""
if md5_hash is not None:
self.metadata['hash'] = md5_hash
if not self.metadata.get('hash'):
if self.is_remote_url:
md5_hash = calculate_hash(self.metadata['redirect_url'],
prefix=True)
else:
try:
md5_hash = calculate_hash(self.content.file.read(),
prefix=True)
except ValueError:
md5_hash = None
self.metadata['hash'] = md5_hash
def set_mimetype(self):
if not self.metadata.get('mimetype'):
mimetype, _ = guess_type(self.filename)
self.metadata['mimetype'] = mimetype
class Migration(migrations.Migration):
dependencies = [
('reversion', '0002_auto_20141216_1509'),
('kpi', '0014_discoverable_subscribable_collections'),
]
operations = [
migrations.CreateModel(
name='AssetVersion',
fields=[
('id',
models.AutoField(verbose_name='ID',
serialize=False,
auto_created=True,
primary_key=True)),
('uid', kpi.fields.KpiUidField(uid_prefix='v')),
('name', models.CharField(max_length=255, null=True)),
('date_modified', models.DateTimeField(default=timezone.now)),
('version_content', JSONBField()),
('deployed_content', JSONBField(null=True)),
('_deployment_data', JSONBField(default=False)),
('deployed', models.BooleanField(default=False)),
('_reversion_version',
models.OneToOneField(null=True,
on_delete=models.SET_NULL,
to='reversion.Version')),
('asset',
models.ForeignKey(related_name='asset_versions',
to='kpi.Asset',
on_delete=models.CASCADE)),
],
options={
'ordering': ['-date_modified'],
},
),
migrations.AlterField(
model_name='asset',
name='summary',
field=JSONField(default=dict, null=True),
),
migrations.AddField(
model_name='asset',
name='report_styles',
field=JSONBField(default=dict),
),
migrations.RenameField(
model_name='assetsnapshot',
old_name='asset_version_id',
new_name='_reversion_version_id',
),
migrations.AddField(
model_name='assetsnapshot',
name='asset_version',
field=models.OneToOneField(null=True,
on_delete=models.CASCADE,
to='kpi.AssetVersion'),
),
migrations.RunPython(
copy_reversion_to_assetversion,
noop,
),
]
class Asset(ObjectPermissionMixin, TagStringMixin, DeployableMixin,
XlsExportable, FormpackXLSFormUtils, models.Model):
name = models.CharField(max_length=255, blank=True, default='')
date_created = models.DateTimeField(auto_now_add=True)
date_modified = models.DateTimeField(auto_now=True)
content = JSONBField(default=dict)
summary = JSONBField(default=dict)
report_styles = JSONBField(default=dict)
report_custom = JSONBField(default=dict)
map_styles = LazyDefaultJSONBField(default=dict)
map_custom = LazyDefaultJSONBField(default=dict)
asset_type = models.CharField(choices=ASSET_TYPES,
max_length=20,
default=ASSET_TYPE_SURVEY)
parent = models.ForeignKey('Collection',
related_name='assets',
null=True,
blank=True,
on_delete=models.CASCADE)
owner = models.ForeignKey('auth.User',
related_name='assets',
null=True,
on_delete=models.CASCADE)
# TODO: remove this flag; support for it has been removed from
# ObjectPermissionMixin
editors_can_change_permissions = models.BooleanField(default=False)
uid = KpiUidField(uid_prefix='a')
tags = TaggableManager(manager=KpiTaggableManager)
settings = JSONBField(default=dict)
# _deployment_data should be accessed through the `deployment` property
# provided by `DeployableMixin`
_deployment_data = JSONBField(default=dict)
permissions = GenericRelation(ObjectPermission)
objects = AssetManager()
@property
def kind(self):
return 'asset'
class Meta:
ordering = ('-date_modified', )
permissions = (
# change_, add_, and delete_asset are provided automatically
# by Django
(PERM_VIEW_ASSET, _('Can view asset')),
(PERM_SHARE_ASSET, _("Can change asset's sharing settings")),
# Permissions for collected data, i.e. submissions
(PERM_ADD_SUBMISSIONS, _('Can submit data to asset')),
(PERM_VIEW_SUBMISSIONS, _('Can view submitted data for asset')),
(PERM_PARTIAL_SUBMISSIONS,
_('Can make partial actions on '
'submitted data for asset '
'for specific users')),
(PERM_CHANGE_SUBMISSIONS,
_('Can modify submitted data for asset')),
(PERM_DELETE_SUBMISSIONS,
_('Can delete submitted data for asset')),
(PERM_SHARE_SUBMISSIONS,
_("Can change sharing settings for "
"asset's submitted data")),
(PERM_VALIDATE_SUBMISSIONS,
_("Can validate submitted data asset")),
# TEMPORARY Issue #1161: A flag to indicate that permissions came
# solely from `sync_kobocat_xforms` and not from any user
# interaction with KPI
(PERM_FROM_KC_ONLY, 'INTERNAL USE ONLY; DO NOT ASSIGN'))
# Since Django 2.1, 4 permissions are added for each registered model:
# - add
# - change
# - delete
# - view
# See https://docs.djangoproject.com/en/2.2/topics/auth/default/#default-permissions
# for more detail.
# `view_asset` clashes with newly built-in one.
# The simplest way to fix this is to keep old behaviour
default_permissions = ('add', 'change', 'delete')
# Labels for each `asset_type` as they should be presented to users
ASSET_TYPE_LABELS = {
ASSET_TYPE_SURVEY: _('form'),
ASSET_TYPE_TEMPLATE: _('template'),
ASSET_TYPE_BLOCK: _('block'),
ASSET_TYPE_QUESTION: _('question'),
ASSET_TYPE_TEXT: _('text'), # unused?
ASSET_TYPE_EMPTY: _('empty'), # unused?
#ASSET_TYPE_COLLECTION: _('collection'),
}
# Assignable permissions that are stored in the database.
# The labels are templates used by `get_label_for_permission()`, which you
# should call instead of accessing this dictionary directly
ASSIGNABLE_PERMISSIONS_WITH_LABELS = {
PERM_VIEW_ASSET: _('View ##asset_type_label##'),
PERM_CHANGE_ASSET: _('Edit ##asset_type_label##'),
PERM_ADD_SUBMISSIONS: _('Add submissions'),
PERM_VIEW_SUBMISSIONS: _('View submissions'),
PERM_PARTIAL_SUBMISSIONS:
_('View submissions only from specific users'),
PERM_CHANGE_SUBMISSIONS: _('Edit submissions'),
PERM_DELETE_SUBMISSIONS: _('Delete submissions'),
PERM_VALIDATE_SUBMISSIONS: _('Validate submissions'),
}
ASSIGNABLE_PERMISSIONS = tuple(ASSIGNABLE_PERMISSIONS_WITH_LABELS.keys())
# Depending on our `asset_type`, only some permissions might be applicable
ASSIGNABLE_PERMISSIONS_BY_TYPE = {
ASSET_TYPE_SURVEY: ASSIGNABLE_PERMISSIONS, # all of them
ASSET_TYPE_TEMPLATE: (PERM_VIEW_ASSET, PERM_CHANGE_ASSET),
ASSET_TYPE_BLOCK: (PERM_VIEW_ASSET, PERM_CHANGE_ASSET),
ASSET_TYPE_QUESTION: (PERM_VIEW_ASSET, PERM_CHANGE_ASSET),
ASSET_TYPE_TEXT: (), # unused?
ASSET_TYPE_EMPTY: (), # unused?
#ASSET_TYPE_COLLECTION: # tbd
}
# Calculated permissions that are neither directly assignable nor stored
# in the database, but instead implied by assignable permissions
CALCULATED_PERMISSIONS = (PERM_SHARE_ASSET, PERM_DELETE_ASSET,
PERM_SHARE_SUBMISSIONS)
# Certain Collection permissions carry over to Asset
MAPPED_PARENT_PERMISSIONS = {
PERM_VIEW_COLLECTION: PERM_VIEW_ASSET,
PERM_CHANGE_COLLECTION: PERM_CHANGE_ASSET
}
# Granting some permissions implies also granting other permissions
IMPLIED_PERMISSIONS = {
# Format: explicit: (implied, implied, ...)
PERM_CHANGE_ASSET: (PERM_VIEW_ASSET, ),
PERM_ADD_SUBMISSIONS: (PERM_VIEW_ASSET, ),
PERM_VIEW_SUBMISSIONS: (PERM_VIEW_ASSET, ),
PERM_PARTIAL_SUBMISSIONS: (PERM_VIEW_ASSET, ),
PERM_CHANGE_SUBMISSIONS: (PERM_VIEW_SUBMISSIONS, ),
PERM_DELETE_SUBMISSIONS: (PERM_VIEW_SUBMISSIONS, ),
PERM_VALIDATE_SUBMISSIONS: (PERM_VIEW_SUBMISSIONS, )
}
CONTRADICTORY_PERMISSIONS = {
PERM_PARTIAL_SUBMISSIONS: (
PERM_VIEW_SUBMISSIONS,
PERM_CHANGE_SUBMISSIONS,
PERM_DELETE_SUBMISSIONS,
PERM_VALIDATE_SUBMISSIONS,
),
PERM_VIEW_SUBMISSIONS: (PERM_PARTIAL_SUBMISSIONS, ),
PERM_CHANGE_SUBMISSIONS: (PERM_PARTIAL_SUBMISSIONS, ),
PERM_DELETE_SUBMISSIONS: (PERM_PARTIAL_SUBMISSIONS, ),
PERM_VALIDATE_SUBMISSIONS: (PERM_PARTIAL_SUBMISSIONS, ),
}
# Some permissions must be copied to KC
KC_PERMISSIONS_MAP = { # keys are KPI's codenames, values are KC's
PERM_CHANGE_SUBMISSIONS: 'change_xform', # "Can Edit" in KC UI
PERM_VIEW_SUBMISSIONS: 'view_xform', # "Can View" in KC UI
PERM_ADD_SUBMISSIONS: 'report_xform', # "Can submit to" in KC UI
PERM_DELETE_SUBMISSIONS:
'delete_data_xform', # "Can Delete Data" in KC UI
PERM_VALIDATE_SUBMISSIONS: 'validate_xform', # "Can Validate" in KC UI
}
KC_CONTENT_TYPE_KWARGS = {'app_label': 'logger', 'model': 'xform'}
# KC records anonymous access as flags on the `XForm`
KC_ANONYMOUS_PERMISSIONS_XFORM_FLAGS = {
PERM_VIEW_SUBMISSIONS: {
'shared': True,
'shared_data': True
}
}
def __str__(self):
return '{} ({})'.format(self.name, self.uid)
def adjust_content_on_save(self):
"""
This is called on save by default if content exists.
Can be disabled / skipped by calling with parameter:
asset.save(adjust_content=False)
"""
self._standardize(self.content)
self._make_default_translation_first(self.content)
self._strip_empty_rows(self.content)
self._assign_kuids(self.content)
self._autoname(self.content)
self._unlink_list_items(self.content)
self._remove_empty_expressions(self.content)
settings = self.content['settings']
_title = settings.pop('form_title', None)
id_string = settings.get('id_string')
filename = self.summary.pop('filename', None)
if filename:
# if we have filename available, set the id_string
# and/or form_title from the filename.
if not id_string:
id_string = sluggify_label(filename)
settings['id_string'] = id_string
if not _title:
_title = filename
if self.asset_type not in [ASSET_TYPE_SURVEY, ASSET_TYPE_TEMPLATE]:
# instead of deleting the settings, simply clear them out
self.content['settings'] = {}
if _title is not None:
self.name = _title
def clone(self, version_uid=None):
# not currently used, but this is how "to_clone_dict" should work
return Asset.objects.create(**self.to_clone_dict(version_uid))
@property
def deployed_versions(self):
return self.asset_versions.filter(
deployed=True).order_by('-date_modified')
def get_ancestors_or_none(self):
# ancestors are ordered from farthest to nearest
if self.parent is not None:
return self.parent.get_ancestors(include_self=True)
else:
return None
def get_filters_for_partial_perm(self,
user_id,
perm=PERM_VIEW_SUBMISSIONS):
"""
Returns the list of filters for a specific permission `perm`
and this specific asset.
:param user_id:
:param perm: see `constants.*_SUBMISSIONS`
:return:
"""
if not perm.endswith(
SUFFIX_SUBMISSIONS_PERMS) or perm == PERM_PARTIAL_SUBMISSIONS:
raise BadPermissionsException(
_('Only partial permissions for '
'submissions are supported'))
perms = self.get_partial_perms(user_id, with_filters=True)
if perms:
return perms.get(perm)
return None
def get_label_for_permission(self, permission_or_codename):
try:
codename = permission_or_codename.codename
permission = permission_or_codename
except AttributeError:
codename = permission_or_codename
permission = None
try:
label = self.ASSIGNABLE_PERMISSIONS_WITH_LABELS[codename]
except KeyError:
if not permission:
# Seems expensive. Cache it?
permission = Permission.objects.get(
# `content_type` and `codename` are `unique_together`
# https://github.com/django/django/blob/e893c0ad8b0b5b0a1e5be3345c287044868effc4/django/contrib/auth/models.py#L69
content_type=ContentType.objects.get_for_model(self),
codename=codename)
label = permission.name
label = label.replace(
'##asset_type_label##',
# Raises TypeError if not coerced explicitly
str(self.ASSET_TYPE_LABELS[self.asset_type]))
return label
def get_partial_perms(self, user_id, with_filters=False):
"""
Returns the list of permissions the user is restricted to,
for this specific asset.
If `with_filters` is `True`, it returns a dict of permissions (as keys) and
the filters (as values) to apply on query to narrow down the results.
For example:
`get_partial_perms(user1_obj.id)` would return
```
['view_submissions',]
```
`get_partial_perms(user1_obj.id, with_filters=True)` would return
```
{
'view_submissions: [
{'_submitted_by': {'$in': ['user1', 'user2']}},
{'_submitted_by': 'user3'}
],
}
```
If user doesn't have any partial permissions, it returns `None`.
:param user_obj: auth.User
:param with_filters: boolean. Optional
:return: list|dict|None
"""
perms = self.asset_partial_permissions.filter(user_id=user_id)\
.values_list("permissions", flat=True).first()
if perms:
if with_filters:
return perms
else:
return list(perms)
return None
@property
def has_active_hooks(self):
"""
Returns if asset has active hooks.
Useful to update `kc.XForm.has_kpi_hooks` field.
:return: {boolean}
"""
return self.hooks.filter(active=True).exists()
@property
def latest_deployed_version(self):
return self.deployed_versions.first()
@property
def latest_version(self):
versions = None
try:
versions = self.prefetched_latest_versions
except AttributeError:
versions = self.asset_versions.order_by('-date_modified')
try:
return versions[0]
except IndexError:
return None
@staticmethod
def optimize_queryset_for_list(queryset):
""" Used by serializers to improve performance when listing assets """
queryset = queryset.defer(
# Avoid pulling these from the database because they are often huge
# and we don't need them for list views.
'content',
'report_styles'
).select_related(
# We only need `username`, but `select_related('owner__username')`
# actually pulled in the entire `auth_user` table under Django 1.8.
# In Django 1.9+, "select_related() prohibits non-relational fields
# for nested relations."
'owner',
).prefetch_related(
# We previously prefetched `permissions__content_object`, but that
# actually pulled the entirety of each permission's linked asset
# from the database! For now, the solution is to remove
# `content_object` here *and* from
# `ObjectPermissionNestedSerializer`.
'permissions__permission',
'permissions__user',
# `Prefetch(..., to_attr='prefetched_list')` stores the prefetched
# related objects in a list (`prefetched_list`) that we can use in
# other methods to avoid additional queries; see:
# https://docs.djangoproject.com/en/1.8/ref/models/querysets/#prefetch-objects
Prefetch('tags', to_attr='prefetched_tags'),
Prefetch(
'asset_versions',
queryset=AssetVersion.objects.order_by('-date_modified').only(
'uid', 'asset', 'date_modified', 'deployed'),
to_attr='prefetched_latest_versions',
),
)
return queryset
def rename_translation(self, _from, _to):
if not self._has_translations(self.content, 2):
raise ValueError('no translations available')
self._rename_translation(self.content, _from, _to)
# todo: test and implement this method
# todo 2019-04-25: Still needed, `revert_to_version` does the same?
# def restore_version(self, uid):
# _version_to_restore = self.asset_versions.get(uid=uid)
# self.content = _version_to_restore.version_content
# self.name = _version_to_restore.name
def revert_to_version(self, version_uid):
av = self.asset_versions.get(uid=version_uid)
self.content = av.version_content
self.save()
def save(self, *args, **kwargs):
if self.content is None:
self.content = {}
# in certain circumstances, we don't want content to
# be altered on save. (e.g. on asset.deploy())
if kwargs.pop('adjust_content', True):
self.adjust_content_on_save()
# populate summary
self._populate_summary()
# infer asset_type only between question and block
if self.asset_type in [ASSET_TYPE_QUESTION, ASSET_TYPE_BLOCK]:
try:
row_count = int(self.summary.get('row_count'))
except TypeError:
pass
else:
if row_count == 1:
self.asset_type = ASSET_TYPE_QUESTION
elif row_count > 1:
self.asset_type = ASSET_TYPE_BLOCK
self._populate_report_styles()
_create_version = kwargs.pop('create_version', True)
super().save(*args, **kwargs)
if _create_version:
self.asset_versions.create(
name=self.name,
version_content=self.content,
_deployment_data=self._deployment_data,
# asset_version.deployed is set in the
# DeploymentSerializer
deployed=False,
)
@property
def snapshot(self):
return self._snapshot(regenerate=False)
def to_clone_dict(self, version_uid=None, version=None):
"""
Returns a dictionary of the asset based on version_uid or version.
If `version` is specified, there are no needs to provide `version_uid` and make another request to DB.
:param version_uid: string
:param version: AssetVersion
:return: dict
"""
if not isinstance(version, AssetVersion):
if version_uid:
version = self.asset_versions.get(uid=version_uid)
else:
version = self.asset_versions.first()
return {
'name': version.name,
'content': version.version_content,
'asset_type': self.asset_type,
'tag_string': self.tag_string,
}
def to_ss_structure(self):
return flatten_content(self.content, in_place=False)
@property
def version__content_hash(self):
# Avoid reading the propery `self.latest_version` more than once, since
# it may execute a database query each time it's read
latest_version = self.latest_version
if latest_version:
return latest_version.content_hash
@property
def version_id(self):
# Avoid reading the propery `self.latest_version` more than once, since
# it may execute a database query each time it's read
latest_version = self.latest_version
if latest_version:
return latest_version.uid
def _populate_report_styles(self):
default = self.report_styles.get(DEFAULT_REPORTS_KEY, {})
specifieds = self.report_styles.get(SPECIFIC_REPORTS_KEY, {})
kuids_to_variable_names = self.report_styles.get('kuid_names', {})
for (index, row) in enumerate(self.content.get('survey', [])):
if '$kuid' not in row:
if 'name' in row:
row['$kuid'] = json_hash([self.uid, row['name']])
else:
row['$kuid'] = json_hash([self.uid, index, row])
_identifier = row.get('name', row['$kuid'])
kuids_to_variable_names[_identifier] = row['$kuid']
if _identifier not in specifieds:
specifieds[_identifier] = {}
self.report_styles = {
DEFAULT_REPORTS_KEY: default,
SPECIFIC_REPORTS_KEY: specifieds,
'kuid_names': kuids_to_variable_names,
}
def _populate_summary(self):
if self.content is None:
self.content = {}
self.summary = {}
return
analyzer = AssetContentAnalyzer(**self.content)
self.summary = analyzer.summary
@transaction.atomic
def _snapshot(self, regenerate=True):
asset_version = self.latest_version
try:
snapshot = AssetSnapshot.objects.get(asset=self,
asset_version=asset_version)
if regenerate:
snapshot.delete()
snapshot = False
except AssetSnapshot.MultipleObjectsReturned:
# how did multiple snapshots get here?
snaps = AssetSnapshot.objects.filter(asset=self,
asset_version=asset_version)
snaps.delete()
snapshot = False
except AssetSnapshot.DoesNotExist:
snapshot = False
if not snapshot:
if self.name != '':
form_title = self.name
else:
_settings = self.content.get('settings', {})
form_title = _settings.get('id_string', 'Untitled')
self._append(self.content, settings={
'form_title': form_title,
})
snapshot = AssetSnapshot.objects.create(
asset=self, asset_version=asset_version, source=self.content)
return snapshot
def _update_partial_permissions(self,
user_id,
perm,
remove=False,
partial_perms=None):
"""
Updates partial permissions relation table according to `perm`.
If `perm` == `PERM_PARTIAL_SUBMISSIONS`, then
If `partial_perms` is not `None`, it should be a dict with filters mapped to
their corresponding permission.
Each filter is used to narrow down results when querying Mongo.
e.g.:
```
{
'view_submissions': [{
'_submitted_by': {
'$in': [
'someuser',
'anotheruser'
]
}
}],
}
```
Even if we can only restrict an user to view another's submissions so far,
this code wants to be future-proof and supports other permissions such as:
- `change_submissions`
- `validate_submissions`
`partial_perms` could be passed as:
```
{
'change_submissions': [{
'_submitted_by': {
'$in': [
'someuser',
'anotheruser'
]
}
}]
'validate_submissions': [{
'_submitted_by': 'someuser'
}],
}
```
:param user_id: int.
:param perm: str. see Asset.ASSIGNABLE_PERMISSIONS
:param remove: boolean. Default is false.
:param partial_perms: dict. Default is None.
:return:
"""
def clean_up_table():
# Because of the unique constraint, there should be only
# one record that matches this query.
# We don't look for record existence to avoid extra query.
self.asset_partial_permissions.filter(user_id=user_id).delete()
if perm == PERM_PARTIAL_SUBMISSIONS:
if remove:
clean_up_table()
return
if user_id == self.owner.pk:
raise BadPermissionsException(
_("Can not assign '{}' permission to owner".format(perm)))
if not partial_perms:
raise BadPermissionsException(
_("Can not assign '{}' permission. "
"Partial permissions are missing.".format(perm)))
new_partial_perms = {}
for partial_perm, filters in partial_perms.items():
implied_perms = [
implied_perm
for implied_perm in self.get_implied_perms(partial_perm)
if implied_perm.endswith(SUFFIX_SUBMISSIONS_PERMS)
]
implied_perms.append(partial_perm)
for implied_perm in implied_perms:
if implied_perm not in new_partial_perms:
new_partial_perms[implied_perm] = []
new_partial_perms[implied_perm] += filters
AssetUserPartialPermission.objects.update_or_create(
asset_id=self.pk,
user_id=user_id,
defaults={'permissions': new_partial_perms})
elif perm in self.CONTRADICTORY_PERMISSIONS.get(
PERM_PARTIAL_SUBMISSIONS):
clean_up_table()
class ImportExportTask(models.Model):
"""
A common base model for asynchronous import and exports. Must be
subclassed to be useful. Subclasses must implement the `_run_task()` method
"""
class Meta:
abstract = True
CREATED = 'created'
PROCESSING = 'processing'
COMPLETE = 'complete'
ERROR = 'error'
STATUS_CHOICES = (
(CREATED, CREATED),
(PROCESSING, PROCESSING),
(ERROR, ERROR),
(COMPLETE, COMPLETE),
)
user = models.ForeignKey('auth.User', on_delete=models.CASCADE)
data = JSONBField()
messages = JSONBField(default=dict)
status = models.CharField(choices=STATUS_CHOICES,
max_length=32,
default=CREATED)
date_created = models.DateTimeField(auto_now_add=True)
# date_expired = models.DateTimeField(null=True)
def run(self):
"""
Starts the import/export job by calling the subclass' `_run_task()`
method. Catches all exceptions! Suitable to be called by an
asynchronous task runner (Celery)
"""
with transaction.atomic():
_refetched_self = self._meta.model.objects.get(pk=self.pk)
self.status = _refetched_self.status
del _refetched_self
if self.status == self.COMPLETE:
return
elif self.status != self.CREATED:
# possibly a concurrent task?
raise Exception(
'only recently created {}s can be executed'.format(
self._meta.model_name))
self.status = self.PROCESSING
self.save(update_fields=['status'])
msgs = defaultdict(list)
try:
# This method must be implemented by a subclass
self._run_task(msgs)
self.status = self.COMPLETE
except Exception as err:
msgs['error_type'] = type(err).__name__
msgs['error'] = str(err)
self.status = self.ERROR
logging.error('Failed to run %s: %s' %
(self._meta.model_name, repr(err)),
exc_info=True)
self.messages.update(msgs)
# Record the processing time for diagnostic purposes
self.data['processing_time_seconds'] = (
datetime.datetime.now(self.date_created.tzinfo) -
self.date_created).total_seconds()
try:
self.save(update_fields=['status', 'messages', 'data'])
except TypeError as e:
self.status = self.ERROR
logging.error('Failed to save %s: %s' %
(self._meta.model_name, repr(e)),
exc_info=True)
self.save(update_fields=['status'])
class Asset(ObjectPermissionMixin, DeployableMixin, XlsExportableMixin,
FormpackXLSFormUtilsMixin, models.Model):
name = models.CharField(max_length=255, blank=True, default='')
date_created = models.DateTimeField(auto_now_add=True)
date_modified = models.DateTimeField(auto_now=True)
content = JSONBField(default=dict)
summary = JSONBField(default=dict)
report_styles = JSONBField(default=dict)
report_custom = JSONBField(default=dict)
map_styles = LazyDefaultJSONBField(default=dict)
map_custom = LazyDefaultJSONBField(default=dict)
asset_type = models.CharField(choices=ASSET_TYPES,
max_length=20,
default=ASSET_TYPE_SURVEY)
parent = models.ForeignKey('Asset',
related_name='children',
null=True,
blank=True,
on_delete=models.CASCADE)
owner = models.ForeignKey('auth.User',
related_name='assets',
null=True,
on_delete=models.CASCADE)
uid = KpiUidField(uid_prefix='a')
tags = TaggableManager(manager=KpiTaggableManager)
settings = JSONBField(default=dict)
# `_deployment_data` must **NOT** be touched directly by anything except
# the `deployment` property provided by `DeployableMixin`.
# ToDo Move the field to another table with one-to-one relationship
_deployment_data = JSONBField(default=dict)
# JSON with subset of fields to share
# {
# 'enable': True,
# 'fields': [] # shares all when empty
# }
data_sharing = LazyDefaultJSONBField(default=dict)
# JSON with source assets' information
# {
# <source_uid>: {
# 'fields': [] # includes all fields shared by source when empty
# 'paired_data_uid': 'pdxxxxxxx' # auto-generated read-only
# 'filename: 'xxxxx.xml'
# },
# ...
# <source_uid>: {
# 'fields': []
# 'paired_data_uid': 'pdxxxxxxx'
# 'filename: 'xxxxx.xml'
# }
# }
paired_data = LazyDefaultJSONBField(default=dict)
objects = AssetManager()
@property
def kind(self):
return 'asset'
class Meta:
# Example in Django documentation represents `ordering` as a list
# (even if it can be a list or a tuple). We enforce the type to `list`
# because `rest_framework.filters.OrderingFilter` work with lists.
# `AssetOrderingFilter` inherits from this class and it is used `
# in `AssetViewSet to sort the result.
# It avoids back and forth between types and/or coercing where
# ordering is needed
ordering = [
'-date_modified',
]
permissions = (
# change_, add_, and delete_asset are provided automatically
# by Django
(PERM_VIEW_ASSET, t('Can view asset')),
(PERM_DISCOVER_ASSET, t('Can discover asset in public lists')),
(PERM_MANAGE_ASSET, t('Can manage all aspects of asset')),
# Permissions for collected data, i.e. submissions
(PERM_ADD_SUBMISSIONS, t('Can submit data to asset')),
(PERM_VIEW_SUBMISSIONS, t('Can view submitted data for asset')),
(PERM_PARTIAL_SUBMISSIONS,
t('Can make partial actions on '
'submitted data for asset '
'for specific users')),
(PERM_CHANGE_SUBMISSIONS,
t('Can modify submitted data for asset')),
(PERM_DELETE_SUBMISSIONS,
t('Can delete submitted data for asset')),
(PERM_VALIDATE_SUBMISSIONS,
t("Can validate submitted data asset")),
# TEMPORARY Issue #1161: A flag to indicate that permissions came
# solely from `sync_kobocat_xforms` and not from any user
# interaction with KPI
(PERM_FROM_KC_ONLY, 'INTERNAL USE ONLY; DO NOT ASSIGN'))
# Since Django 2.1, 4 permissions are added for each registered model:
# - add
# - change
# - delete
# - view
# See https://docs.djangoproject.com/en/2.2/topics/auth/default/#default-permissions
# for more detail.
# `view_asset` clashes with newly built-in one.
# The simplest way to fix this is to keep old behaviour
default_permissions = ('add', 'change', 'delete')
# Labels for each `asset_type` as they should be presented to users. Can be
# strings or callables if special logic is needed. Callables receive the
# codename of the permission for which a label is being created
ASSET_TYPE_LABELS_FOR_PERMISSIONS = {
ASSET_TYPE_SURVEY: (lambda p: t('project')
if p == PERM_MANAGE_ASSET else t('form')),
ASSET_TYPE_TEMPLATE:
t('template'),
ASSET_TYPE_BLOCK:
t('block'),
ASSET_TYPE_QUESTION:
t('question'),
ASSET_TYPE_TEXT:
t('text'), # unused?
ASSET_TYPE_EMPTY:
t('empty'), # unused?
ASSET_TYPE_COLLECTION:
t('collection'),
}
# Assignable permissions that are stored in the database.
# The labels are templates used by `get_label_for_permission()`, which you
# should call instead of accessing this dictionary directly
ASSIGNABLE_PERMISSIONS_WITH_LABELS = {
PERM_VIEW_ASSET: t('View ##asset_type_label##'),
PERM_CHANGE_ASSET: t('Edit ##asset_type_label##'),
PERM_DISCOVER_ASSET: t('Discover ##asset_type_label##'),
PERM_MANAGE_ASSET: t('Manage ##asset_type_label##'),
PERM_ADD_SUBMISSIONS: t('Add submissions'),
PERM_VIEW_SUBMISSIONS: t('View submissions'),
PERM_PARTIAL_SUBMISSIONS: {
'default':
t('Act on submissions only from specific users'),
PERM_VIEW_SUBMISSIONS:
t('View submissions only from specific users'),
PERM_CHANGE_SUBMISSIONS:
t('Edit submissions only from specific users'),
PERM_DELETE_SUBMISSIONS:
t('Delete submissions only from specific users'),
PERM_VALIDATE_SUBMISSIONS:
t('Validate submissions only from specific users'),
},
PERM_CHANGE_SUBMISSIONS: t('Edit submissions'),
PERM_DELETE_SUBMISSIONS: t('Delete submissions'),
PERM_VALIDATE_SUBMISSIONS: t('Validate submissions'),
}
ASSIGNABLE_PERMISSIONS = tuple(ASSIGNABLE_PERMISSIONS_WITH_LABELS.keys())
# Depending on our `asset_type`, only some permissions might be applicable
ASSIGNABLE_PERMISSIONS_BY_TYPE = {
ASSET_TYPE_SURVEY:
tuple((p for p in ASSIGNABLE_PERMISSIONS if p != PERM_DISCOVER_ASSET)),
ASSET_TYPE_TEMPLATE: (
PERM_VIEW_ASSET,
PERM_CHANGE_ASSET,
PERM_MANAGE_ASSET,
),
ASSET_TYPE_BLOCK: (
PERM_VIEW_ASSET,
PERM_CHANGE_ASSET,
PERM_MANAGE_ASSET,
),
ASSET_TYPE_QUESTION: (
PERM_VIEW_ASSET,
PERM_CHANGE_ASSET,
PERM_MANAGE_ASSET,
),
ASSET_TYPE_TEXT: (), # unused?
ASSET_TYPE_EMPTY: (
PERM_VIEW_ASSET,
PERM_CHANGE_ASSET,
PERM_MANAGE_ASSET,
),
ASSET_TYPE_COLLECTION: (
PERM_VIEW_ASSET,
PERM_CHANGE_ASSET,
PERM_DISCOVER_ASSET,
PERM_MANAGE_ASSET,
),
}
# Calculated permissions that are neither directly assignable nor stored
# in the database, but instead implied by assignable permissions
CALCULATED_PERMISSIONS = (PERM_DELETE_ASSET, )
# Only certain permissions can be inherited
HERITABLE_PERMISSIONS = {
# parent permission: child permission
PERM_VIEW_ASSET: PERM_VIEW_ASSET,
PERM_CHANGE_ASSET: PERM_CHANGE_ASSET
}
# Granting some permissions implies also granting other permissions
IMPLIED_PERMISSIONS = {
# Format: explicit: (implied, implied, ...)
PERM_CHANGE_ASSET: (PERM_VIEW_ASSET, ),
PERM_DISCOVER_ASSET: (PERM_VIEW_ASSET, ),
PERM_MANAGE_ASSET:
tuple((p for p in ASSIGNABLE_PERMISSIONS
if p not in (PERM_MANAGE_ASSET, PERM_PARTIAL_SUBMISSIONS))),
PERM_ADD_SUBMISSIONS: (PERM_VIEW_ASSET, ),
PERM_VIEW_SUBMISSIONS: (PERM_VIEW_ASSET, ),
PERM_PARTIAL_SUBMISSIONS: (PERM_VIEW_ASSET, ),
PERM_CHANGE_SUBMISSIONS: (
PERM_VIEW_SUBMISSIONS,
PERM_ADD_SUBMISSIONS,
),
PERM_DELETE_SUBMISSIONS: (PERM_VIEW_SUBMISSIONS, ),
PERM_VALIDATE_SUBMISSIONS: (PERM_VIEW_SUBMISSIONS, ),
}
CONTRADICTORY_PERMISSIONS = {
PERM_PARTIAL_SUBMISSIONS: (
PERM_VIEW_SUBMISSIONS,
PERM_CHANGE_SUBMISSIONS,
PERM_DELETE_SUBMISSIONS,
PERM_VALIDATE_SUBMISSIONS,
PERM_MANAGE_ASSET,
),
PERM_VIEW_SUBMISSIONS: (PERM_PARTIAL_SUBMISSIONS, ),
PERM_CHANGE_SUBMISSIONS: (PERM_PARTIAL_SUBMISSIONS, ),
PERM_DELETE_SUBMISSIONS: (PERM_PARTIAL_SUBMISSIONS, ),
PERM_VALIDATE_SUBMISSIONS: (PERM_PARTIAL_SUBMISSIONS, ),
}
# Some permissions must be copied to KC
KC_PERMISSIONS_MAP = { # keys are KPI's codenames, values are KC's
PERM_CHANGE_SUBMISSIONS: 'change_xform', # "Can Edit" in KC UI
PERM_VIEW_SUBMISSIONS: 'view_xform', # "Can View" in KC UI
PERM_ADD_SUBMISSIONS: 'report_xform', # "Can submit to" in KC UI
PERM_DELETE_SUBMISSIONS:
'delete_data_xform', # "Can Delete Data" in KC UI
PERM_VALIDATE_SUBMISSIONS: 'validate_xform', # "Can Validate" in KC UI
}
KC_CONTENT_TYPE_KWARGS = {'app_label': 'logger', 'model': 'xform'}
# KC records anonymous access as flags on the `XForm`
KC_ANONYMOUS_PERMISSIONS_XFORM_FLAGS = {
PERM_VIEW_SUBMISSIONS: {
'shared': True,
'shared_data': True
}
}
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# The two fields below are needed to keep a trace of the object state
# before any alteration. See `__self.__copy_hidden_fields()` for details
# They must be set with an invalid value for their counterparts to
# be the comparison is accurate.
self.__parent_id_copy = -1
self.__deployment_data_copy = None
self.__copy_hidden_fields()
def __str__(self):
return '{} ({})'.format(self.name, self.uid)
def adjust_content_on_save(self):
"""
This is called on save by default if content exists.
Can be disabled / skipped by calling with parameter:
asset.save(adjust_content=False)
"""
self._standardize(self.content)
self._make_default_translation_first(self.content)
self._strip_empty_rows(self.content)
self._assign_kuids(self.content)
self._autoname(self.content)
self._unlink_list_items(self.content)
self._remove_empty_expressions(self.content)
settings = self.content['settings']
_title = settings.pop('form_title', None)
id_string = settings.get('id_string')
filename = self.summary.pop('filename', None)
if filename:
# if we have filename available, set the id_string
# and/or form_title from the filename.
if not id_string:
id_string = sluggify_label(filename)
settings['id_string'] = id_string
if not _title:
_title = filename
if self.asset_type not in [ASSET_TYPE_SURVEY, ASSET_TYPE_TEMPLATE]:
# instead of deleting the settings, simply clear them out
self.content['settings'] = {}
strip_kobo_locking_profile(self.content)
if _title is not None:
self.name = _title
def clone(self, version_uid=None):
# not currently used, but this is how "to_clone_dict" should work
return Asset.objects.create(**self.to_clone_dict(version=version_uid))
def create_version(self) -> [AssetVersion, None]:
"""
Create a version of current asset.
Asset has to belong to `ASSET_TYPE_WITH_CONTENT` otherwise no version
is created and `None` is returned.
"""
if self.asset_type not in ASSET_TYPES_WITH_CONTENT:
return
return self.asset_versions.create(
name=self.name,
version_content=self.content,
_deployment_data=self._deployment_data,
# Any new version starts out as not-deployed,
# even if the asset itself is already deployed.
# Note: `asset_version.deployed` is set in the
# serializer `DeploymentSerializer`
deployed=False,
)
@property
def deployed_versions(self):
return self.asset_versions.filter(
deployed=True).order_by('-date_modified')
@property
def discoverable_when_public(self):
# This property is only needed when `self` is a collection.
# We want to make a distinction between a collection which is not
# discoverable and an asset which is not a collection
# (which implies cannot be discoverable)
if self.asset_type != ASSET_TYPE_COLLECTION:
return None
return self.permissions.filter(
permission__codename=PERM_DISCOVER_ASSET,
user_id=settings.ANONYMOUS_USER_ID).exists()
def get_filters_for_partial_perm(
self,
user_id: int,
perm: str = PERM_VIEW_SUBMISSIONS) -> Union[list, None]:
"""
Returns the list of filters for a specific permission `perm`
and this specific asset.
`perm` can only one of the submission permissions.
"""
if (not perm.endswith(SUFFIX_SUBMISSIONS_PERMS)
or perm == PERM_PARTIAL_SUBMISSIONS):
raise BadPermissionsException(
t('Only partial permissions for '
'submissions are supported'))
perms = self.get_partial_perms(user_id, with_filters=True)
if perms:
try:
return perms[perm]
except KeyError:
# User has some partial permissions but not the good one.
# Return a false condition to avoid showing any results.
return [{'_id': -1}]
return None
def get_label_for_permission(
self, permission_or_codename: Union[Permission, str]) -> str:
"""
Get the correct label for a permission (object or codename) based on
the type of this asset
"""
try:
codename = permission_or_codename.codename
permission = permission_or_codename
except AttributeError:
codename = permission_or_codename
permission = None
try:
label = self.ASSIGNABLE_PERMISSIONS_WITH_LABELS[codename]
except KeyError:
if permission:
label = permission.name
else:
cached_code_names = get_cached_code_names()
label = cached_code_names[codename]['name']
asset_type_label = self.ASSET_TYPE_LABELS_FOR_PERMISSIONS[
self.asset_type]
try:
# Some labels may be callables
asset_type_label = asset_type_label(codename)
except TypeError:
# Others are just strings
pass
# For partial permissions, label is a dict.
# There is no replacements to do in the nested labels, but these lines
# are there to support in case we need it one day
if isinstance(label, dict):
labels = copy.deepcopy(label)
for key_ in labels.keys():
labels[key_] = labels[key_].replace(
'##asset_type_label##',
# Raises TypeError if not coerced explicitly due to
# ugettext_lazy()
str(asset_type_label))
return labels
else:
return label.replace(
'##asset_type_label##',
# Raises TypeError if not coerced explicitly due to
# ugettext_lazy()
str(asset_type_label))
def get_partial_perms(
self,
user_id: int,
with_filters: bool = False) -> Union[list, dict, None]:
"""
Returns the list of permissions the user is restricted to,
for this specific asset.
If `with_filters` is `True`, it returns a dict of permissions (as keys)
and the filters (as values) to apply on query to narrow down
the results.
For example:
`get_partial_perms(user1_obj.id)` would return
```
['view_submissions',]
```
`get_partial_perms(user1_obj.id, with_filters=True)` would return
```
{
'view_submissions: [
{'_submitted_by': {'$in': ['user1', 'user2']}},
{'_submitted_by': 'user3'}
],
}
```
If user doesn't have any partial permissions, it returns `None`.
"""
perms = self.asset_partial_permissions.filter(user_id=user_id)\
.values_list("permissions", flat=True).first()
if perms:
if with_filters:
return perms
else:
return list(perms)
return None
@property
def has_active_hooks(self):
"""
Returns if asset has active hooks.
Useful to update `kc.XForm.has_kpi_hooks` field.
:return: {boolean}
"""
return self.hooks.filter(active=True).exists()
def has_subscribed_user(self, user_id):
# This property is only needed when `self` is a collection.
# We want to make a distinction between a collection which does not have
# the subscribed user and an asset which is not a collection
# (which implies cannot have subscriptions)
if self.asset_type != ASSET_TYPE_COLLECTION:
return None
# ToDo: See if using a loop can reduce the number of SQL queries.
return self.userassetsubscription_set.filter(user_id=user_id).exists()
@property
def latest_deployed_version(self):
return self.deployed_versions.first()
@property
def latest_version(self):
versions = None
try:
versions = self.prefetched_latest_versions
except AttributeError:
versions = self.asset_versions.order_by('-date_modified')
try:
return versions[0]
except IndexError:
return None
@staticmethod
def optimize_queryset_for_list(queryset):
""" Used by serializers to improve performance when listing assets """
queryset = queryset.defer(
# Avoid pulling these from the database because they are often huge
# and we don't need them for list views.
'content',
'report_styles'
).select_related(
# We only need `username`, but `select_related('owner__username')`
# actually pulled in the entire `auth_user` table under Django 1.8.
# In Django 1.9+, "select_related() prohibits non-relational fields
# for nested relations."
'owner',
).prefetch_related(
'permissions__permission',
'permissions__user',
# `Prefetch(..., to_attr='prefetched_list')` stores the prefetched
# related objects in a list (`prefetched_list`) that we can use in
# other methods to avoid additional queries; see:
# https://docs.djangoproject.com/en/1.8/ref/models/querysets/#prefetch-objects
Prefetch('tags', to_attr='prefetched_tags'),
Prefetch(
'asset_versions',
queryset=AssetVersion.objects.order_by('-date_modified').only(
'uid', 'asset', 'date_modified', 'deployed'),
to_attr='prefetched_latest_versions',
),
)
return queryset
def refresh_from_db(self, using=None, fields=None):
super().refresh_from_db(using=using, fields=fields)
# Refresh hidden fields too
self.__copy_hidden_fields(fields)
def rename_translation(self, _from, _to):
if not self._has_translations(self.content, 2):
raise ValueError('no translations available')
self._rename_translation(self.content, _from, _to)
# todo: test and implement this method
# todo 2019-04-25: Still needed, `revert_to_version` does the same?
# def restore_version(self, uid):
# _version_to_restore = self.asset_versions.get(uid=uid)
# self.content = _version_to_restore.version_content
# self.name = _version_to_restore.name
def revert_to_version(self, version_uid):
av = self.asset_versions.get(uid=version_uid)
self.content = av.version_content
self.save()
def save(self,
force_insert=False,
force_update=False,
update_fields=None,
adjust_content=True,
create_version=True,
update_parent_languages=True,
*args,
**kwargs):
is_new = self.pk is None
if self.asset_type not in ASSET_TYPES_WITH_CONTENT:
# so long as all of the operations in this overridden `save()`
# method pertain to content, bail out if it's impossible for this
# asset to have content in the first place
super().save(force_insert=force_insert,
force_update=force_update,
update_fields=update_fields,
*args,
**kwargs)
return
if self.content is None:
self.content = {}
# in certain circumstances, we don't want content to
# be altered on save. (e.g. on asset.deploy())
if adjust_content:
self.adjust_content_on_save()
# populate summary
self._populate_summary()
# infer asset_type only between question and block
if self.asset_type in [ASSET_TYPE_QUESTION, ASSET_TYPE_BLOCK]:
try:
row_count = int(self.summary.get('row_count'))
except TypeError:
pass
else:
if row_count == 1:
self.asset_type = ASSET_TYPE_QUESTION
elif row_count > 1:
self.asset_type = ASSET_TYPE_BLOCK
self._populate_report_styles()
# Ensure `_deployment_data` is not saved directly
try:
stored_data_key = self._deployment_data['_stored_data_key']
except KeyError:
if self._deployment_data != self.__deployment_data_copy:
raise DeploymentDataException
else:
if stored_data_key != self.deployment.stored_data_key:
raise DeploymentDataException
else:
self._deployment_data.pop('_stored_data_key', None)
self.__copy_hidden_fields()
super().save(force_insert=force_insert,
force_update=force_update,
update_fields=update_fields,
*args,
**kwargs)
# Update languages for parent and previous parent.
# e.g. if a survey has been moved from one collection to another,
# we want both collections to be updated.
if self.parent is not None and update_parent_languages:
if (self.parent_id != self.__parent_id_copy
and self.__parent_id_copy is not None):
try:
previous_parent = Asset.objects.get(
pk=self.__parent_id_copy)
previous_parent.update_languages()
self.__parent_id_copy = self.parent_id
except Asset.DoesNotExist:
pass
# If object is new, we can add its languages to its parent without
# worrying about removing its old values. It avoids an extra query.
if is_new:
self.parent.update_languages([self])
else:
# Otherwise, because we cannot know which languages are from
# this object, update will be performed with all parent's
# children.
self.parent.update_languages()
if self.has_deployment:
self.deployment.sync_media_files(AssetFile.PAIRED_DATA)
if create_version:
self.create_version()
@property
def snapshot(self):
return self._snapshot(regenerate=False)
@property
def tag_string(self):
try:
tag_list = self.prefetched_tags
except AttributeError:
tag_names = self.tags.values_list('name', flat=True)
else:
tag_names = [t.name for t in tag_list]
return ','.join(tag_names)
@tag_string.setter
def tag_string(self, value):
intended_tags = value.split(',')
self.tags.set(*intended_tags)
def to_clone_dict(self, version: Union[str, AssetVersion] = None) -> dict:
"""
Returns a dictionary of the asset based on its version.
:param version: Optional. It can be an object or its unique id
:return dict
"""
if not isinstance(version, AssetVersion):
if version:
version = self.asset_versions.get(uid=version)
else:
version = self.asset_versions.first()
if not version:
version = self.create_version()
return {
'name': version.name,
'content': version.version_content,
'asset_type': self.asset_type,
'tag_string': self.tag_string,
}
def to_ss_structure(self):
return flatten_content(self.content, in_place=False)
def update_languages(self, children=None):
"""
Updates object's languages by aggregating all its children's languages
Args:
children (list<Asset>): Optional. When specified, `children`'s languages
are merged with `self`'s languages. Otherwise, when it's `None`,
DB is fetched to build the list according to `self.children`
"""
# If object is not a collection, it should not have any children.
# No need to go further.
if self.asset_type != ASSET_TYPE_COLLECTION:
return
obj_languages = self.summary.get('languages', [])
languages = set()
if children:
languages = set(obj_languages)
children_languages = [
child.summary.get('languages') for child in children
if child.summary.get('languages')
]
else:
children_languages = list(
self.children.values_list(
'summary__languages', flat=True).exclude(
Q(summary__languages=[])
| Q(summary__languages=[None])).order_by())
if children_languages:
# Flatten `children_languages` to 1-dimension list.
languages.update(reduce(add, children_languages))
languages.discard(None)
# Object of type set is not JSON serializable
languages = list(languages)
# If languages are still the same, no needs to update the object
if sorted(obj_languages) == sorted(languages):
return
self.summary['languages'] = languages
self.save(update_fields=['summary'])
@property
def version__content_hash(self):
# Avoid reading the property `self.latest_version` more than once, since
# it may execute a database query each time it's read
latest_version = self.latest_version
if latest_version:
return latest_version.content_hash
@property
def version_id(self):
# Avoid reading the property `self.latest_version` more than once, since
# it may execute a database query each time it's read
latest_version = self.latest_version
if latest_version:
return latest_version.uid
@property
def version_number_and_date(self) -> str:
# Returns the count of all deployed versions (plus one for the current
# version if it is not deployed) and the date the asset was last
# modified
count = self.deployed_versions.count()
if not self.latest_version.deployed:
count = count + 1
return f'{count} {self.date_modified:(%Y-%m-%d %H:%M:%S)}'
# TODO: take leading underscore off of `_snapshot()` and call it directly?
# we would also have to remove or rename the `snapshot` property
def versioned_snapshot(
self,
version_uid: str,
root_node_name: Optional[str] = None) -> AssetSnapshot:
return self._snapshot(
regenerate=False,
version_uid=version_uid,
root_node_name=root_node_name,
)
def _populate_report_styles(self):
default = self.report_styles.get(DEFAULT_REPORTS_KEY, {})
specifieds = self.report_styles.get(SPECIFIC_REPORTS_KEY, {})
kuids_to_variable_names = self.report_styles.get('kuid_names', {})
for (index, row) in enumerate(self.content.get('survey', [])):
if '$kuid' not in row:
if 'name' in row:
row['$kuid'] = json_hash([self.uid, row['name']])
else:
row['$kuid'] = json_hash([self.uid, index, row])
_identifier = row.get('name', row['$kuid'])
kuids_to_variable_names[_identifier] = row['$kuid']
if _identifier not in specifieds:
specifieds[_identifier] = {}
self.report_styles = {
DEFAULT_REPORTS_KEY: default,
SPECIFIC_REPORTS_KEY: specifieds,
'kuid_names': kuids_to_variable_names,
}
def _populate_summary(self):
if self.content is None:
self.content = {}
self.summary = {}
return
analyzer = AssetContentAnalyzer(**self.content)
self.summary = analyzer.summary
@transaction.atomic
def _snapshot(
self,
regenerate: bool = True,
version_uid: Optional[str] = None,
root_node_name: Optional[str] = None,
) -> AssetSnapshot:
if version_uid:
asset_version = self.asset_versions.get(uid=version_uid)
else:
asset_version = self.latest_version
try:
snapshot = AssetSnapshot.objects.get(asset=self,
asset_version=asset_version)
if regenerate:
snapshot.delete()
snapshot = False
except AssetSnapshot.MultipleObjectsReturned:
# how did multiple snapshots get here?
snaps = AssetSnapshot.objects.filter(asset=self,
asset_version=asset_version)
snaps.delete()
snapshot = False
except AssetSnapshot.DoesNotExist:
snapshot = False
if not snapshot:
try:
form_title = asset_version.form_title
content = asset_version.version_content
except AttributeError:
form_title = self.form_title
content = self.content
settings_ = {'form_title': form_title}
if root_node_name:
# `name` may not sound like the right setting to control the
# XML root node name, but it is, according to the XLSForm
# specification:
# https://xlsform.org/en/#specify-xforms-root-node-name
settings_['name'] = root_node_name
settings_['id_string'] = root_node_name
self._append(content, settings=settings_)
snapshot = AssetSnapshot.objects.create(
asset=self, asset_version=asset_version, source=content)
return snapshot
def _update_partial_permissions(
self,
user: '******',
perm: str,
remove: bool = False,
partial_perms: Optional[dict] = None,
):
"""
Stores, updates, and removes permissions that apply only to a subset of
submissions in a project (also called row-level permissions or partial
permissions).
If `perm = PERM_PARTIAL_SUBMISSIONS`, it must be accompanied by
`partial_perms`, which is a dictionary of permissions mapped to MongoDB
filters. Each key of that dictionary is a permission string (codename),
and each value is a list of MongoDB queries that specify which
submissions the permission affects. A submission is affected if it
matches *ANY* of the queries in the list.
For example, to allow `user` to edit submissions made by 'alice' or
'bob', and to allow `user` also to validate only submissions made by
'bob', the following `partial_perms` could be used:
```
{
'change_submissions': [{
'_submitted_by': {
'$in': [
'alice',
'bob'
]
}
}],
'validate_submissions': [{
'_submitted_by': 'bob'
}],
}
```
If `perm` is something other than `PERM_PARTIAL_SUBMISSIONS`, and that
permission contradicts `PERM_PARTIAL_SUBMISSIONS`, *all* partial
permission assignments for `user` on this asset are removed from the
database. If the permission does not conflict, no action is taken.
`remove = True` deletes all partial permissions assignments for `user`
on this asset.
"""
def clean_up_table():
# Because of the unique constraint, there should be only
# one record that matches this query.
# We don't look for record existence to avoid extra query.
self.asset_partial_permissions.filter(user_id=user.pk).delete()
if perm == PERM_PARTIAL_SUBMISSIONS:
if remove:
clean_up_table()
return
if user.pk == self.owner.pk:
raise BadPermissionsException(
t("Can not assign '{}' permission to owner".format(perm)))
if not partial_perms:
raise BadPermissionsException(
t("Can not assign '{}' permission. "
"Partial permissions are missing.".format(perm)))
new_partial_perms = AssetUserPartialPermission\
.update_partial_perms_to_include_implied(
self,
partial_perms
)
AssetUserPartialPermission.objects.update_or_create(
asset_id=self.pk,
user_id=user.pk,
defaults={'permissions': new_partial_perms})
# There are no real partial permissions for 'add_submissions' but
# 'change_submissions' implies it. So if 'add_submissions' is in the
# partial permissions list, it must be assigned to the user to the
# user as well to let them perform edit actions on their subset of
# data. Otherwise, KC will reject some actions.
if PERM_ADD_SUBMISSIONS in new_partial_perms:
self.assign_perm(user_obj=user,
perm=PERM_ADD_SUBMISSIONS,
defer_recalc=True)
elif perm in self.CONTRADICTORY_PERMISSIONS.get(
PERM_PARTIAL_SUBMISSIONS):
clean_up_table()
def __copy_hidden_fields(self, fields: Optional[list] = None):
"""
Save a copy of `parent_id` and `_deployment_data` for these purposes
`save()` respectively.
- `self.__parent_id_copy` is used to detect whether asset is linked a
different parent
- `self.__deployment_data_copy` is used to detect whether
`_deployment_data` has been altered directly
"""
# When fields are deferred, Django instantiates another copy
# of the current Asset object to retrieve the value of the
# requested field. Because we need to get a copy at the very
# first beginning of the life of the object, this method is
# called in the object constructor. Thus, trying to copy
# deferred fields would create an infinite loop.
# If `fields` is provided, fields are no longer deferred and should be
# copied right away.
if (fields is None and 'parent_id' not in self.get_deferred_fields()
or fields and 'parent_id' in fields):
self.__parent_id_copy = self.parent_id
if (fields is None
and '_deployment_data' not in self.get_deferred_fields()
or fields and '_deployment_data' in fields):
self.__deployment_data_copy = copy.deepcopy(self._deployment_data)
class ModelWithJSONBFieldTest(DirtyFieldsMixin, models.Model):
jsonb_field = JSONBField()
class Hook(models.Model):
# Export types
XML = "xml"
JSON = "json"
# Authentication levels
NO_AUTH = "no_auth"
BASIC_AUTH = "basic_auth"
# Export types list
EXPORT_TYPE_CHOICES = ((XML, XML), (JSON, JSON))
# Authentication levels list
AUTHENTICATION_LEVEL_CHOICES = ((NO_AUTH, NO_AUTH), (BASIC_AUTH,
BASIC_AUTH))
asset = models.ForeignKey("kpi.Asset",
related_name="hooks",
on_delete=models.CASCADE)
uid = KpiUidField(uid_prefix="h")
name = models.CharField(max_length=255, blank=False)
endpoint = models.CharField(max_length=500, blank=False)
active = models.BooleanField(default=True)
export_type = models.CharField(choices=EXPORT_TYPE_CHOICES,
default=JSON,
max_length=10)
auth_level = models.CharField(choices=AUTHENTICATION_LEVEL_CHOICES,
default=NO_AUTH,
max_length=10)
settings = JSONBField(default=dict)
date_created = models.DateTimeField(default=timezone.now)
date_modified = models.DateTimeField(default=timezone.now)
email_notification = models.BooleanField(default=True)
subset_fields = ArrayField(
models.CharField(max_length=500),
blank=True,
default=list,
)
payload_template = models.TextField(null=True, blank=True)
class Meta:
ordering = ["name"]
def __init__(self, *args, **kwargs):
self.__totals = {}
super().__init__(*args, **kwargs)
def save(self, *args, **kwargs):
# Update date_modified each time object is saved
self.date_modified = timezone.now()
super().save(*args, **kwargs)
def __str__(self):
return "%s:%s - %s" % (self.asset, self.name, self.endpoint)
def get_service_definition(self):
mod = import_module("kobo.apps.hook.services.service_{}".format(
self.export_type))
return getattr(mod, "ServiceDefinition")
@property
def success_count(self):
if not self.__totals:
self._get_totals()
return self.__totals.get(HOOK_LOG_SUCCESS)
@property
def failed_count(self):
if not self.__totals:
self._get_totals()
return self.__totals.get(HOOK_LOG_FAILED)
@property
def pending_count(self):
if not self.__totals:
self._get_totals()
return self.__totals.get(HOOK_LOG_PENDING)
def _get_totals(self):
# TODO add some cache
queryset = self.logs.values("status").annotate(
values_count=models.Count("status"))
queryset.query.clear_ordering(True)
# Initialize totals
self.__totals = {
HOOK_LOG_SUCCESS: 0,
HOOK_LOG_FAILED: 0,
HOOK_LOG_PENDING: 0
}
for record in queryset:
self.__totals[record.get("status")] = record.get("values_count")
def reset_totals(self):
# TODO remove cache when it's enabled
self.__totals = {}
class AssetUserPartialPermission(models.Model):
"""
Many-to-Many table which provides users' permissions
on other users' submissions
For example,
- Asset:
- uid: aAAAAAA
- id: 1
- User:
- username: someuser
- id: 1
We want someuser to be able to view otheruser's submissions
Records should be
`permissions` is dict formatted as is:
asset_id | user_id | permissions
1 | 1 | {"someuser": ["view_submissions"]}
Using a list per user for permissions, gives the opportunity to add other permissions
such as `change_submissions`, `delete_submissions` for later purpose
"""
class Meta:
unique_together = [['asset', 'user']]
asset = models.ForeignKey('Asset', related_name='asset_partial_permissions',
on_delete=models.CASCADE)
user = models.ForeignKey('auth.User', related_name='user_partial_permissions',
on_delete=models.CASCADE)
permissions = JSONBField(default=dict)
date_created = models.DateTimeField(default=timezone.now)
date_modified = models.DateTimeField(default=timezone.now)
def save(self, *args, **kwargs):
if self.pk is not None:
self.date_modified = timezone.now()
super().save(*args, **kwargs)
@staticmethod
def update_partial_perms_to_include_implied(
asset: 'kpi.models.Asset', partial_perms: dict
) -> dict:
new_partial_perms = defaultdict(list)
in_op = MongoHelper.IN_OPERATOR
for partial_perm, filters in partial_perms.items():
if partial_perm not in new_partial_perms:
new_partial_perms[partial_perm] = filters
# TODO: omit `add_submissions`? It's required at the asset
# level for any kind of editing (e.g. partial
# `change_submissions` requires asset-wide `add_submissions`),
# but it doesn't make sense to restrict adding submissions
# "only to those submissions that match some criteria"
implied_perms = [
implied_perm
for implied_perm in asset.get_implied_perms(
partial_perm, for_instance=asset
)
if implied_perm.endswith(SUFFIX_SUBMISSIONS_PERMS)
]
for implied_perm in implied_perms:
if (
implied_perm not in new_partial_perms
and implied_perm in partial_perms
):
new_partial_perms[implied_perm] = partial_perms[
implied_perm
]
new_partial_perm = new_partial_perms[implied_perm]
# Trivial case, i.e.: permissions are built with front end.
# All permissions have only one filter and the same filter
# Example:
# ```
# partial_perms = {
# 'view_submissions' : [
# {'_submitted_by': {'$in': ['johndoe']}}
# ],
# 'delete_submissions': [
# {'_submitted_by': {'$in': ['quidam']}}
# ]
# }
# ```
# should give
# ```
# new_partial_perms = {
# 'view_submissions' : [
# {'_submitted_by': {'$in': ['johndoe', 'quidam']}}
# ],
# 'delete_submissions': [
# {'_submitted_by': {'$in': ['quidam']}}
# ]
# }
if (
len(filters) == 1
and len(new_partial_perm) == 1
and isinstance(new_partial_perm, list)
):
current_filters = new_partial_perms[implied_perm][0]
filter_ = filters[0]
# Front end only supports `_submitted_by`, but if users
# use the API, it could be something else.
filter_key = list(filter_)[0]
try:
new_value = filter_[filter_key][in_op]
current_values = current_filters[filter_key][in_op]
except (KeyError, TypeError):
pass
else:
new_partial_perm[0][filter_key][in_op] = list(
set(current_values + new_value)
)
continue
# As said earlier, front end only supports `'_submitted_by'`
# filter, but many different and more complex filters could
# be used.
# If we reach these lines, it means conditions cannot be
# merged, so we concatenate then with an `OR` operator.
# Example:
# ```
# partial_perms = {
# 'view_submissions' : [{'_submitted_by': 'johndoe'}],
# 'delete_submissions': [
# {'_submission_date': {'$lte': '2021-01-01'},
# {'_submission_date': {'$gte': '2020-01-01'}
# ]
# }
# ```
# should give
# ```
# new_partial_perms = {
# 'view_submissions' : [
# [{'_submitted_by': 'johndoe'}],
# [
# {'_submission_date': {'$lte': '2021-01-01'},
# {'_submission_date': {'$gte': '2020-01-01'}
# ]
# },
# 'delete_submissions': [
# {'_submission_date': {'$lte': '2021-01-01'},
# {'_submission_date': {'$gte': '2020-01-01'}
# ]
# }
# To avoid more complexity (and different syntax than
# trivial case), we delegate to MongoHelper the task to join
# lists with the `$or` operator.
try:
new_partial_perm = new_partial_perms[implied_perm][0]
except IndexError:
# If we get an IndexError, implied permission does not
# belong to current assignment. Let's copy the filters
#
new_partial_perms[implied_perm] = filters
else:
if not isinstance(new_partial_perm, list):
new_partial_perms[implied_perm] = [
filters,
new_partial_perms[implied_perm],
]
else:
new_partial_perms[implied_perm].append(filters)
return new_partial_perms
class AssetFile(OpenRosaManifestInterface, models.Model):
# More to come!
MAP_LAYER = 'map_layer'
FORM_MEDIA = 'form_media'
TYPE_CHOICES = (
(MAP_LAYER, MAP_LAYER),
(FORM_MEDIA, FORM_MEDIA),
)
ALLOWED_MIME_TYPES = {
FORM_MEDIA: ('image', 'audio', 'video', 'text/csv', 'application/xml'),
MAP_LAYER: (
'text/csv',
'application/vnd.google-earth.kml+xml',
'application/vnd.google-earth.kmz',
'application/wkt',
'application/geo+json',
'application/json',
),
}
uid = KpiUidField(uid_prefix='af')
asset = models.ForeignKey('Asset',
related_name='asset_files',
on_delete=models.CASCADE)
# Keep track of the uploading user, who could be anyone with `change_asset`
# rights, not just the asset owner
user = models.ForeignKey('auth.User',
related_name='asset_files',
on_delete=models.CASCADE)
file_type = models.CharField(choices=TYPE_CHOICES, max_length=32)
description = models.CharField(max_length=255)
date_created = models.DateTimeField(default=timezone.now)
content = PrivateFileField(upload_to=upload_to, max_length=380, null=True)
metadata = JSONBField(default=dict)
date_deleted = models.DateTimeField(null=True, default=None)
def delete(self, using=None, keep_parents=False, force=False):
# Delete object and files on storage if `force` is True or file type
# is anything else than 'form_media'
if force or self.file_type != self.FORM_MEDIA:
if not self.is_remote_url:
self.content.delete(save=False)
return super().delete(using=using, keep_parents=keep_parents)
# Otherwise, just flag the file as deleted.
self.date_deleted = timezone.now()
self.save(update_fields=['date_deleted'])
@property
def filename(self):
"""
Implements `OpenRosaManifestInterface.filename()`
"""
if hasattr(self, '__filename'):
return self.__filename
self.set_filename()
self.__filename = self.metadata['filename']
return self.__filename
def get_download_url(self, request):
"""
Implements `OpenRosaManifestInterface.get_download_url()`
"""
return reverse('asset-file-content',
args=(self.asset.uid, self.uid),
request=request)
@staticmethod
def get_path(asset, file_type, filename):
return posixpath.join(asset.owner.username, 'asset_files', asset.uid,
file_type, filename)
@property
def hash(self):
"""
Implements `OpenRosaManifestInterface.hash()`
"""
if hasattr(self, '__hash'):
return self.__hash
self.set_hash()
self.__hash = self.metadata['hash']
return self.__hash
@property
def is_remote_url(self):
try:
self.metadata['redirect_url']
except KeyError:
return False
return True
def save(self,
force_insert=False,
force_update=False,
using=None,
update_fields=None):
if self.pk is None:
self.set_filename()
self.set_hash()
self.set_mimetype()
return super().save(force_insert, force_update, using, update_fields)
def set_filename(self):
if not self.metadata.get('filename'):
self.metadata['filename'] = self.content.name
def set_hash(self):
if not self.metadata.get('hash'):
if self.is_remote_url:
md5_hash = get_hash(self.metadata['redirect_url'])
else:
md5_hash = get_hash(self.content.file.read())
self.metadata['hash'] = f'md5:{md5_hash}'
def set_mimetype(self):
mimetype, _ = guess_type(self.metadata['filename'])
self.metadata['mimetype'] = mimetype