class AssetVersion(models.Model): uid = KpiUidField(uid_prefix='v') asset = models.ForeignKey('Asset', related_name='asset_versions', on_delete=models.CASCADE) name = models.CharField(null=True, max_length=255) date_modified = models.DateTimeField(default=timezone.now) # preserving _reversion_version in case we don't save all that we # need to in the first migration from reversion to AssetVersion _reversion_version = models.OneToOneField( Version, null=True, on_delete=models.SET_NULL, ) version_content = JSONBField() uid_aliases = JSONBField(null=True) deployed_content = JSONBField(null=True) _deployment_data = JSONBField(default=dict) deployed = models.BooleanField(default=False) class Meta: ordering = ['-date_modified'] def _deployed_content(self): if self.deployed_content is not None: return self.deployed_content legacy_names = self._reversion_version is not None if legacy_names: return to_xlsform_structure(self.version_content, deprecated_autoname=True) else: return to_xlsform_structure(self.version_content, move_autonames=True) def to_formpack_schema(self): return { 'content': expand_content(self._deployed_content()), 'version': self.uid, 'version_id_key': '__version__', } @property def content_hash(self): # used to determine changes in the content from version to version # not saved, only compared with other asset_versions _json_string = json.dumps(self.version_content, sort_keys=True) return hashlib.sha1(hashable_str(_json_string)).hexdigest() def __str__(self): return '{}@{} T{}{}'.format( self.asset.uid, self.uid, self.date_modified.strftime('%Y-%m-%d %H:%M'), ' (deployed)' if self.deployed else '')
class KobocatUserProfile(ShadowModel): """ From onadata/apps/main/models/user_profile.py Not read-only because we need write access to `require_auth` """ class Meta(ShadowModel.Meta): db_table = 'main_userprofile' verbose_name = 'user profile' verbose_name_plural = 'user profiles' # This field is required. user = models.OneToOneField(KobocatUser, related_name='profile', on_delete=models.CASCADE) # Other fields here name = models.CharField(max_length=255, blank=True) city = models.CharField(max_length=255, blank=True) country = models.CharField(max_length=2, blank=True) organization = models.CharField(max_length=255, blank=True) home_page = models.CharField(max_length=255, blank=True) twitter = models.CharField(max_length=255, blank=True) description = models.CharField(max_length=255, blank=True) require_auth = models.BooleanField( default=False, verbose_name="Require authentication to see forms and submit data") address = models.CharField(max_length=255, blank=True) phonenumber = models.CharField(max_length=30, blank=True) created_by = models.ForeignKey(User, null=True, blank=True, on_delete=models.CASCADE) num_of_submissions = models.IntegerField(default=0) metadata = JSONBField(default=dict, blank=True)
class ExtraUserDetail(models.Model): user = models.OneToOneField(settings.AUTH_USER_MODEL, related_name='extra_details', on_delete=models.CASCADE) data = JSONBField(default=dict) def __str__(self): return '{}\'s data: {}'.format(self.user.__str__(), repr(self.data))
class Migration(migrations.Migration): dependencies = [ ('hub', '0006_remove_formbuilder_preference_table'), ] operations = [ migrations.AlterField( model_name='extrauserdetail', name='data', field=JSONBField(default=dict), ), migrations.AlterField( model_name='perusersetting', name='user_queries', field=JSONBField( help_text= 'A JSON representation of a *list* of Django queries, e.g. `[{"email__iendswith": "@kobotoolbox.org"}, {"email__iendswith": "@kbtdev.org"}]`. A matching user is one who would be returned by ANY of the queries in the list.' ), ), ]
class PerUserSetting(models.Model): """ A configuration setting that has different values depending on whether not a user matches certain criteria """ user_queries = JSONBField( help_text='A JSON representation of a *list* of Django queries, ' 'e.g. `[{"email__iendswith": "@kobotoolbox.org"}, ' '{"email__iendswith": "@kbtdev.org"}]`. ' 'A matching user is one who would be returned by ANY of ' 'the queries in the list.' ) name = models.CharField(max_length=255, unique=True, default='INTERCOM_APP_ID') # The only one for now! value_when_matched = models.CharField(max_length=2048, blank=True) value_when_not_matched = models.CharField(max_length=2048, blank=True) def user_matches(self, user, ignore_invalid_queries=True): if user.is_anonymous: user = get_anonymous_user() manager = user._meta.model.objects queryset = manager.none() for user_query in self.user_queries: try: queryset |= manager.filter(**user_query) except (FieldError, TypeError): if ignore_invalid_queries: return False else: raise return queryset.filter(pk=user.pk).exists() def get_for_user(self, user): if self.user_matches(user): return self.value_when_matched else: return self.value_when_not_matched def clean(self): user = User.objects.first() if not user: return try: self.user_matches(user, ignore_invalid_queries=False) except FieldError as e: raise ValidationError({'user_queries': e.message}) except TypeError: raise ValidationError( {'user_queries': 'JSON structure is incorrect.'}) def __str__(self): return self.name
class Migration(migrations.Migration): dependencies = [ ('kpi', '0016_asset_settings'), ] operations = [ migrations.AddField( model_name='assetversion', name='uid_aliases', field=JSONBField(null=True), ), ]
class Migration(migrations.Migration): dependencies = [ ('kpi', '0015_assetversion'), ] operations = [ migrations.AddField( model_name='asset', name='settings', field=JSONBField(default=dict), ), ]
class Migration(migrations.Migration): dependencies = [ ('kpi', '0018_export_task'), ] operations = [ migrations.AddField( model_name='asset', name='report_custom', field=JSONBField(default=dict), ), ]
class KobocatUserProfile(ShadowModel): """ From onadata/apps/main/models/user_profile.py Not read-only because we need write access to `require_auth` """ class Meta(ShadowModel.Meta): db_table = 'main_userprofile' verbose_name = 'user profile' verbose_name_plural = 'user profiles' # This field is required. user = models.OneToOneField(KobocatUser, related_name='profile', on_delete=models.CASCADE) # Other fields here name = models.CharField(max_length=255, blank=True) city = models.CharField(max_length=255, blank=True) country = models.CharField(max_length=2, blank=True) organization = models.CharField(max_length=255, blank=True) home_page = models.CharField(max_length=255, blank=True) twitter = models.CharField(max_length=255, blank=True) description = models.CharField(max_length=255, blank=True) require_auth = models.BooleanField( default=False, verbose_name="Require authentication to see forms and submit data" ) address = models.CharField(max_length=255, blank=True) phonenumber = models.CharField(max_length=30, blank=True) created_by = models.ForeignKey(KobocatUser, null=True, blank=True, on_delete=models.CASCADE) num_of_submissions = models.IntegerField(default=0) metadata = JSONBField(default=dict, blank=True) # We need to cast `is_active` to an (positive small) integer because KoBoCAT # is using `LazyBooleanField` which is an integer behind the scene. # We do not want to port this class to KPI only for one line of code. is_mfa_active = models.PositiveSmallIntegerField(default=False) @classmethod def set_mfa_status(cls, user_id: int, is_active: bool): try: user_profile, created = cls.objects.get_or_create(user_id=user_id) except cls.DoesNotExist: pass else: user_profile.is_mfa_active = int(is_active) user_profile.save(update_fields=['is_mfa_active'])
class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), ('kpi', '0021_map-custom-styles'), ] operations = [ migrations.CreateModel( name='AssetFile', fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('uid', kpi.fields.KpiUidField(uid_prefix=b'af')), ('file_type', models.CharField(max_length=32, choices=[(b'map_layer', b'map_layer')])), ('name', models.CharField(max_length=255)), ('date_created', models.DateTimeField(default=django.utils.timezone.now)), ('content', private_storage.fields.PrivateFileField( storage=private_storage.storage.files. PrivateFileSystemStorage(), max_length=380, upload_to=kpi.models.asset_file.upload_to)), ('metadata', JSONBField(default=dict)), ], ), # Why did `manage.py makemigrations` create these as separate operations? migrations.AddField( model_name='assetfile', name='asset', field=models.ForeignKey(related_name='asset_files', to='kpi.Asset', on_delete=models.CASCADE), ), migrations.AddField( model_name='assetfile', name='user', field=models.ForeignKey(related_name='asset_files', to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE), ), ]
class InAppMessageUserInteractions(models.Model): message = models.ForeignKey(InAppMessage, on_delete=models.CASCADE) user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) interactions = JSONBField(default=dict) class Meta: unique_together = ('message', 'user') def __str__(self): return '{} with {} ({}): {}'.format( self.user.username, self.message.title, self.message.uid, self.interactions, )
class Migration(migrations.Migration): dependencies = [ ('kpi', '0022_assetfile'), ] operations = [ migrations.CreateModel( name='Hook', fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('uid', kpi.fields.KpiUidField(uid_prefix=b'h')), ('name', models.CharField(max_length=255)), ('endpoint', models.CharField(max_length=500)), ('active', models.BooleanField(default=True)), ('export_type', models.CharField(default=b'json', max_length=10, choices=[(b'xml', b'xml'), (b'json', b'json')])), ('auth_level', models.CharField(default=b'no_auth', max_length=10, choices=[(b'no_auth', b'no_auth'), (b'basic_auth', b'basic_auth')])), ('settings', JSONBField(default=dict)), ('date_created', models.DateTimeField(default=django.utils.timezone.now)), ('date_modified', models.DateTimeField(default=django.utils.timezone.now)), ('asset', models.ForeignKey(related_name='hooks', to='kpi.Asset', on_delete=models.CASCADE)), ], options={ 'ordering': ['name'], }, ), migrations.CreateModel( name='HookLog', fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('uid', kpi.fields.KpiUidField(uid_prefix=b'hl')), ('instance_id', models.IntegerField(default=0, db_index=True)), ('tries', models.PositiveSmallIntegerField(default=0)), ('status', models.PositiveSmallIntegerField(default=1)), ('status_code', models.IntegerField(default=None, null=True, blank=True)), ('message', models.TextField(default=b'')), ('date_created', models.DateTimeField(auto_now_add=True)), ('date_modified', models.DateTimeField(auto_now_add=True)), ('hook', models.ForeignKey(related_name='logs', to='hook.Hook', on_delete=models.CASCADE)), ], options={ 'ordering': ['-date_created'], }, ), ]
class Migration(migrations.Migration): dependencies = [ ('hub', '0004_configurationfile'), ] operations = [ migrations.CreateModel( name='PerUserSetting', fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('user_queries', JSONBField(help_text='A JSON representation of a *list* of Django queries, e.g. `[{"email__endswith": "@kobotoolbox.org"}, {"email__endswith": "@kbtdev.org"}]`. A matching user is one who would be returned by ANY of the queries in the list.')), ('name', models.CharField(unique=True, max_length=255)), ('value_when_matched', models.CharField(max_length=2048, blank=True)), ('value_when_not_matched', models.CharField(max_length=2048, blank=True)), ], ), ]
class AssetExportSettings(models.Model): uid = KpiUidField(uid_prefix='es') asset = models.ForeignKey('Asset', related_name='asset_export_settings', on_delete=models.CASCADE) date_modified = models.DateTimeField() name = models.CharField(max_length=255, blank=True, default='') export_settings = JSONBField(default=dict) def save(self, *args, **kwargs): self.date_modified = timezone.now() super().save(*args, **kwargs) class Meta: ordering = ['-date_modified'] unique_together = ('asset', 'name') def __str__(self): return f'{self.name} {self.uid}'
class AssetFile(models.Model): # More to come! MAP_LAYER = 'map_layer' TYPE_CHOICES = ((MAP_LAYER, MAP_LAYER), ) uid = KpiUidField(uid_prefix='af') asset = models.ForeignKey('Asset', related_name='asset_files', on_delete=models.CASCADE) # Keep track of the uploading user, who could be anyone with `change_asset` # rights, not just the asset owner user = models.ForeignKey('auth.User', related_name='asset_files', on_delete=models.CASCADE) file_type = models.CharField(choices=TYPE_CHOICES, max_length=32) name = models.CharField(max_length=255) date_created = models.DateTimeField(default=timezone.now) # TODO: Handle deletion! The file won't be deleted automatically when the # object is removed from the database content = PrivateFileField(upload_to=upload_to, max_length=380) metadata = JSONBField(default=dict)
class AssetUserPartialPermission(models.Model): """ Many-to-Many table which provides users' permissions on other users' submissions For example, - Asset: - uid: aAAAAAA - id: 1 - User: - username: someuser - id: 1 We want someuser to be able to view otheruser's submissions Records should be `permissions` is dict formatted as is: asset_id | user_id | permissions 1 | 1 | {"someuser": ["view_submissions"]} Using a list per user for permissions, gives the opportunity to add other permissions such as `change_submissions`, `delete_submissions` for later purpose """ class Meta: unique_together = [['asset', 'user']] asset = models.ForeignKey('Asset', related_name='asset_partial_permissions', on_delete=models.CASCADE) user = models.ForeignKey('auth.User', related_name='user_partial_permissions', on_delete=models.CASCADE) permissions = JSONBField(default=dict) date_created = models.DateTimeField(default=timezone.now) date_modified = models.DateTimeField(default=timezone.now) def save(self, *args, **kwargs): if self.pk is not None: self.date_modified = timezone.now() super().save(*args, **kwargs)
class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ migrations.CreateModel( name='InAppMessage', fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('uid', kpi.fields.KpiUidField(uid_prefix='iam')), ('title', models.CharField(max_length=255)), ('snippet', markdownx.models.MarkdownxField()), ('body', markdownx.models.MarkdownxField()), ('published', models.BooleanField( default=False, help_text= 'When published, this message appears to all users. It otherwise appears only to the last editor' )), ('valid_from', models.DateTimeField( default=datetime.datetime(1970, 1, 1, 0, 0))), ('valid_until', models.DateTimeField( default=datetime.datetime(1970, 1, 1, 0, 0))), ('last_editor', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), ], ), migrations.CreateModel( name='InAppMessageFile', fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('content', private_storage.fields.PrivateFileField( storage=private_storage.storage.files. PrivateFileSystemStorage(), upload_to='__in_app_message/%Y/%m/%d/')), ], ), migrations.CreateModel( name='InAppMessageUserInteractions', fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('interactions', JSONBField(default=dict)), ('message', models.ForeignKey(to='help.InAppMessage', on_delete=models.CASCADE)), ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), ], ), migrations.AlterUniqueTogether( name='inappmessageuserinteractions', unique_together=set([('message', 'user')]), ), ]
class AssetSnapshot( models.Model, AbstractFormList, XlsExportableMixin, FormpackXLSFormUtilsMixin, ): """ This model serves as a cache of the XML that was exported by the installed version of pyxform. TODO: come up with a policy to clear this cache out. DO NOT: depend on these snapshots existing for more than a day until a policy is set. Done with https://github.com/kobotoolbox/kpi/pull/2434. Remove above lines when PR is merged """ xml = models.TextField() source = JSONBField(default=dict) details = JSONBField(default=dict) owner = models.ForeignKey('auth.User', related_name='asset_snapshots', null=True, on_delete=models.CASCADE) asset = models.ForeignKey('Asset', null=True, on_delete=models.CASCADE) _reversion_version_id = models.IntegerField(null=True) asset_version = models.OneToOneField('AssetVersion', on_delete=models.CASCADE, null=True) date_created = models.DateTimeField(auto_now_add=True) uid = KpiUidField(uid_prefix='s') @property def content(self): return self.source @property def description(self): """ Implements `OpenRosaFormListInterface.description` """ return self.asset.settings.get('description', '') @property def form_id(self): """ Implements `OpenRosaFormListInterface.form_id()` """ return self.uid def get_download_url(self, request): """ Implements `OpenRosaFormListInterface.get_download_url()` """ return reverse( viewname='assetsnapshot-detail', format='xml', kwargs={'uid': self.uid}, request=request ) def get_manifest_url(self, request): """ Implements `OpenRosaFormListInterface.get_manifest_url()` """ return reverse( viewname='assetsnapshot-manifest', format='xml', kwargs={'uid': self.uid}, request=request ) @property def md5_hash(self): """ Implements `OpenRosaFormListInterface.md5_hash()` """ return f'{calculate_hash(self.xml, prefix=True)}' @property def name(self): """ Implements `OpenRosaFormListInterface.name()` """ return self.asset.name def save(self, *args, **kwargs): if self.asset is not None: # Previously, `self.source` was a nullable field. It must now # either contain valid content or be an empty dictionary. assert self.asset is not None if not self.source: if self.asset_version is None: self.asset_version = self.asset.latest_version self.source = self.asset_version.version_content if self.owner is None: self.owner = self.asset.owner _note = self.details.pop('note', None) _source = copy.deepcopy(self.source) self._standardize(_source) self._make_default_translation_first(_source) self._strip_empty_rows(_source) self._autoname(_source) self._remove_empty_expressions(_source) # TODO: move these inside `generate_xml_from_source()`? _settings = _source.get('settings', {}) form_title = _settings.get('form_title') id_string = _settings.get('id_string') root_node_name = _settings.get('name') self.xml, self.details = self.generate_xml_from_source( _source, include_note=_note, root_node_name=root_node_name, form_title=form_title, id_string=id_string, ) self.source = _source return super().save(*args, **kwargs) def generate_xml_from_source(self, source, include_note=False, root_node_name=None, form_title=None, id_string=None): if not root_node_name: if self.asset and self.asset.uid: root_node_name = self.asset.uid else: root_node_name = 'snapshot_xml' if not form_title: if self.asset and self.asset.name: form_title = self.asset.name else: form_title = 'Snapshot XML' if id_string is None: id_string = root_node_name if include_note and 'survey' in source: _translations = source.get('translations', []) _label = include_note if len(_translations) > 0: _label = [_label for t in _translations] source['survey'].append({'type': 'note', 'name': 'prepended_note', 'label': _label}) source_copy = copy.deepcopy(source) self._expand_kobo_qs(source_copy) self._populate_fields_with_autofields(source_copy) self._strip_kuids(source_copy) allow_choice_duplicates(source_copy) warnings = [] details = {} try: xml = FormPack({'content': source_copy}, root_node_name=root_node_name, id_string=id_string, title=form_title)[0].to_xml(warnings=warnings) details.update({ 'status': 'success', 'warnings': warnings, }) except Exception as err: err_message = str(err) logging.error('Failed to generate xform for asset', extra={ 'src': source, 'id_string': id_string, 'uid': self.uid, '_msg': err_message, 'warnings': warnings, }) xml = '' details.update({ 'status': 'failure', 'error_type': type(err).__name__, 'error': err_message, 'warnings': warnings, }) return xml, details
class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), ('kpi', '0022_assetfile'), ] operations = [ migrations.CreateModel( name='AssetUserPartialPermission', fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('permissions', JSONBField(default=dict)), ('date_created', models.DateTimeField(default=django.utils.timezone.now)), ('date_modified', models.DateTimeField(default=django.utils.timezone.now)), ], ), migrations.AlterModelOptions( name='asset', options={ 'ordering': ('-date_modified', ), 'permissions': (('view_asset', 'Can view asset'), ('share_asset', "Can change asset's sharing settings"), ('add_submissions', 'Can submit data to asset'), ('view_submissions', 'Can view submitted data for asset'), ('partial_submissions', 'Can make partial actions on submitted data for asset for specific users' ), ('change_submissions', 'Can modify submitted data for asset'), ('delete_submissions', 'Can delete submitted data for asset'), ('share_submissions', "Can change sharing settings for asset's submitted data"), ('validate_submissions', 'Can validate submitted data asset'), ('from_kc_only', 'INTERNAL USE ONLY; DO NOT ASSIGN')) }, ), migrations.AddField( model_name='assetuserpartialpermission', name='asset', field=models.ForeignKey(related_name='asset_partial_permissions', to='kpi.Asset', on_delete=models.CASCADE), ), migrations.AddField( model_name='assetuserpartialpermission', name='user', field=models.ForeignKey(related_name='user_partial_permissions', to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE), ), migrations.AlterUniqueTogether( name='assetuserpartialpermission', unique_together=set([('asset', 'user')]), ), ]
class AssetFile(models.Model, AbstractFormMedia): # More to come! MAP_LAYER = 'map_layer' FORM_MEDIA = 'form_media' PAIRED_DATA = 'paired_data' TYPE_CHOICES = ( (MAP_LAYER, MAP_LAYER), (FORM_MEDIA, FORM_MEDIA), (PAIRED_DATA, PAIRED_DATA), ) ALLOWED_MIME_TYPES = { FORM_MEDIA: ( 'image', 'audio', 'video', 'text/csv', 'application/xml', 'application/zip', ), PAIRED_DATA: ('application/xml', ), MAP_LAYER: ( 'text/csv', 'application/vnd.google-earth.kml+xml', 'application/vnd.google-earth.kmz', 'application/wkt', 'application/geo+json', 'application/json', ), } uid = KpiUidField(uid_prefix='af') asset = models.ForeignKey('Asset', related_name='asset_files', on_delete=models.CASCADE) # Keep track of the uploading user, who could be anyone with `change_asset` # rights, not just the asset owner user = models.ForeignKey('auth.User', related_name='asset_files', on_delete=models.CASCADE) file_type = models.CharField(choices=TYPE_CHOICES, max_length=32) description = models.CharField(max_length=255) date_created = models.DateTimeField(default=timezone.now) content = PrivateFileField(upload_to=upload_to, max_length=380, null=True) metadata = JSONBField(default=dict) date_deleted = models.DateTimeField(null=True, default=None) date_modified = models.DateTimeField(default=timezone.now) synced_with_backend = models.BooleanField(default=False) @property def backend_media_id(self): """ Implements `SyncBackendMediaInterface.backend_media_id()` """ return (self.metadata['redirect_url'] if self.is_remote_url else self.filename) def delete(self, using=None, keep_parents=False, force=False): # Delete object and files on storage if `force` is True or file type # is anything else than 'form_media' if force or self.file_type != self.FORM_MEDIA: if not self.is_remote_url: self.content.delete(save=False) return super().delete(using=using, keep_parents=keep_parents) # Otherwise, just flag the file as deleted. self.date_deleted = timezone.now() self.synced_with_backend = False self.save(update_fields=['date_deleted', 'synced_with_backend']) @property def deleted_at(self): """ Implements: - `SyncBackendMediaInterface.deleted_at()` """ return self.date_deleted @property def filename(self): """ Implements: - `OpenRosaManifestInterface.filename()` - `SyncBackendMediaInterface.filename()` """ self.set_filename() return self.metadata['filename'] def get_download_url(self, request): """ Implements `OpenRosaManifestInterface.get_download_url()` """ return reverse('asset-file-content', args=(self.asset.uid, self.uid), request=request) @staticmethod def get_path(asset, file_type, filename): return posixpath.join(asset.owner.username, 'asset_files', asset.uid, file_type, filename) @property def md5_hash(self): """ Implements: - `OpenRosaManifestInterface.md5_hash()` - `SyncBackendMediaInterface.md5_hash()` """ if not self.metadata.get('hash'): self.set_md5_hash() return self.metadata['hash'] @property def is_remote_url(self): """ Implements `SyncBackendMediaInterface.is_remote_url()` """ try: self.metadata['redirect_url'] except KeyError: return False return True @property def mimetype(self): """ Implements `SyncBackendMediaInterface.mimetype()` """ self.set_mimetype() return self.metadata['mimetype'] def save(self, force_insert=False, force_update=False, using=None, update_fields=None): if self.pk is None: self.set_filename() self.set_md5_hash() self.set_mimetype() else: self.date_modified = timezone.now() return super().save(force_insert, force_update, using, update_fields) def set_filename(self): if not self.metadata.get('filename'): self.metadata['filename'] = self.content.name def set_md5_hash(self, md5_hash: Optional[str] = None): """ Calculate md5 hash and store it in `metadata` field if it does not exist or empty. Value can be also set with the optional `md5_hash` parameter. If `md5_hash` is an empty string, the hash is recalculated. """ if md5_hash is not None: self.metadata['hash'] = md5_hash if not self.metadata.get('hash'): if self.is_remote_url: md5_hash = calculate_hash(self.metadata['redirect_url'], prefix=True) else: try: md5_hash = calculate_hash(self.content.file.read(), prefix=True) except ValueError: md5_hash = None self.metadata['hash'] = md5_hash def set_mimetype(self): if not self.metadata.get('mimetype'): mimetype, _ = guess_type(self.filename) self.metadata['mimetype'] = mimetype
class Migration(migrations.Migration): dependencies = [ ('reversion', '0002_auto_20141216_1509'), ('kpi', '0014_discoverable_subscribable_collections'), ] operations = [ migrations.CreateModel( name='AssetVersion', fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('uid', kpi.fields.KpiUidField(uid_prefix='v')), ('name', models.CharField(max_length=255, null=True)), ('date_modified', models.DateTimeField(default=timezone.now)), ('version_content', JSONBField()), ('deployed_content', JSONBField(null=True)), ('_deployment_data', JSONBField(default=False)), ('deployed', models.BooleanField(default=False)), ('_reversion_version', models.OneToOneField(null=True, on_delete=models.SET_NULL, to='reversion.Version')), ('asset', models.ForeignKey(related_name='asset_versions', to='kpi.Asset', on_delete=models.CASCADE)), ], options={ 'ordering': ['-date_modified'], }, ), migrations.AlterField( model_name='asset', name='summary', field=JSONField(default=dict, null=True), ), migrations.AddField( model_name='asset', name='report_styles', field=JSONBField(default=dict), ), migrations.RenameField( model_name='assetsnapshot', old_name='asset_version_id', new_name='_reversion_version_id', ), migrations.AddField( model_name='assetsnapshot', name='asset_version', field=models.OneToOneField(null=True, on_delete=models.CASCADE, to='kpi.AssetVersion'), ), migrations.RunPython( copy_reversion_to_assetversion, noop, ), ]
class Asset(ObjectPermissionMixin, TagStringMixin, DeployableMixin, XlsExportable, FormpackXLSFormUtils, models.Model): name = models.CharField(max_length=255, blank=True, default='') date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) content = JSONBField(default=dict) summary = JSONBField(default=dict) report_styles = JSONBField(default=dict) report_custom = JSONBField(default=dict) map_styles = LazyDefaultJSONBField(default=dict) map_custom = LazyDefaultJSONBField(default=dict) asset_type = models.CharField(choices=ASSET_TYPES, max_length=20, default=ASSET_TYPE_SURVEY) parent = models.ForeignKey('Collection', related_name='assets', null=True, blank=True, on_delete=models.CASCADE) owner = models.ForeignKey('auth.User', related_name='assets', null=True, on_delete=models.CASCADE) # TODO: remove this flag; support for it has been removed from # ObjectPermissionMixin editors_can_change_permissions = models.BooleanField(default=False) uid = KpiUidField(uid_prefix='a') tags = TaggableManager(manager=KpiTaggableManager) settings = JSONBField(default=dict) # _deployment_data should be accessed through the `deployment` property # provided by `DeployableMixin` _deployment_data = JSONBField(default=dict) permissions = GenericRelation(ObjectPermission) objects = AssetManager() @property def kind(self): return 'asset' class Meta: ordering = ('-date_modified', ) permissions = ( # change_, add_, and delete_asset are provided automatically # by Django (PERM_VIEW_ASSET, _('Can view asset')), (PERM_SHARE_ASSET, _("Can change asset's sharing settings")), # Permissions for collected data, i.e. submissions (PERM_ADD_SUBMISSIONS, _('Can submit data to asset')), (PERM_VIEW_SUBMISSIONS, _('Can view submitted data for asset')), (PERM_PARTIAL_SUBMISSIONS, _('Can make partial actions on ' 'submitted data for asset ' 'for specific users')), (PERM_CHANGE_SUBMISSIONS, _('Can modify submitted data for asset')), (PERM_DELETE_SUBMISSIONS, _('Can delete submitted data for asset')), (PERM_SHARE_SUBMISSIONS, _("Can change sharing settings for " "asset's submitted data")), (PERM_VALIDATE_SUBMISSIONS, _("Can validate submitted data asset")), # TEMPORARY Issue #1161: A flag to indicate that permissions came # solely from `sync_kobocat_xforms` and not from any user # interaction with KPI (PERM_FROM_KC_ONLY, 'INTERNAL USE ONLY; DO NOT ASSIGN')) # Since Django 2.1, 4 permissions are added for each registered model: # - add # - change # - delete # - view # See https://docs.djangoproject.com/en/2.2/topics/auth/default/#default-permissions # for more detail. # `view_asset` clashes with newly built-in one. # The simplest way to fix this is to keep old behaviour default_permissions = ('add', 'change', 'delete') # Labels for each `asset_type` as they should be presented to users ASSET_TYPE_LABELS = { ASSET_TYPE_SURVEY: _('form'), ASSET_TYPE_TEMPLATE: _('template'), ASSET_TYPE_BLOCK: _('block'), ASSET_TYPE_QUESTION: _('question'), ASSET_TYPE_TEXT: _('text'), # unused? ASSET_TYPE_EMPTY: _('empty'), # unused? #ASSET_TYPE_COLLECTION: _('collection'), } # Assignable permissions that are stored in the database. # The labels are templates used by `get_label_for_permission()`, which you # should call instead of accessing this dictionary directly ASSIGNABLE_PERMISSIONS_WITH_LABELS = { PERM_VIEW_ASSET: _('View ##asset_type_label##'), PERM_CHANGE_ASSET: _('Edit ##asset_type_label##'), PERM_ADD_SUBMISSIONS: _('Add submissions'), PERM_VIEW_SUBMISSIONS: _('View submissions'), PERM_PARTIAL_SUBMISSIONS: _('View submissions only from specific users'), PERM_CHANGE_SUBMISSIONS: _('Edit submissions'), PERM_DELETE_SUBMISSIONS: _('Delete submissions'), PERM_VALIDATE_SUBMISSIONS: _('Validate submissions'), } ASSIGNABLE_PERMISSIONS = tuple(ASSIGNABLE_PERMISSIONS_WITH_LABELS.keys()) # Depending on our `asset_type`, only some permissions might be applicable ASSIGNABLE_PERMISSIONS_BY_TYPE = { ASSET_TYPE_SURVEY: ASSIGNABLE_PERMISSIONS, # all of them ASSET_TYPE_TEMPLATE: (PERM_VIEW_ASSET, PERM_CHANGE_ASSET), ASSET_TYPE_BLOCK: (PERM_VIEW_ASSET, PERM_CHANGE_ASSET), ASSET_TYPE_QUESTION: (PERM_VIEW_ASSET, PERM_CHANGE_ASSET), ASSET_TYPE_TEXT: (), # unused? ASSET_TYPE_EMPTY: (), # unused? #ASSET_TYPE_COLLECTION: # tbd } # Calculated permissions that are neither directly assignable nor stored # in the database, but instead implied by assignable permissions CALCULATED_PERMISSIONS = (PERM_SHARE_ASSET, PERM_DELETE_ASSET, PERM_SHARE_SUBMISSIONS) # Certain Collection permissions carry over to Asset MAPPED_PARENT_PERMISSIONS = { PERM_VIEW_COLLECTION: PERM_VIEW_ASSET, PERM_CHANGE_COLLECTION: PERM_CHANGE_ASSET } # Granting some permissions implies also granting other permissions IMPLIED_PERMISSIONS = { # Format: explicit: (implied, implied, ...) PERM_CHANGE_ASSET: (PERM_VIEW_ASSET, ), PERM_ADD_SUBMISSIONS: (PERM_VIEW_ASSET, ), PERM_VIEW_SUBMISSIONS: (PERM_VIEW_ASSET, ), PERM_PARTIAL_SUBMISSIONS: (PERM_VIEW_ASSET, ), PERM_CHANGE_SUBMISSIONS: (PERM_VIEW_SUBMISSIONS, ), PERM_DELETE_SUBMISSIONS: (PERM_VIEW_SUBMISSIONS, ), PERM_VALIDATE_SUBMISSIONS: (PERM_VIEW_SUBMISSIONS, ) } CONTRADICTORY_PERMISSIONS = { PERM_PARTIAL_SUBMISSIONS: ( PERM_VIEW_SUBMISSIONS, PERM_CHANGE_SUBMISSIONS, PERM_DELETE_SUBMISSIONS, PERM_VALIDATE_SUBMISSIONS, ), PERM_VIEW_SUBMISSIONS: (PERM_PARTIAL_SUBMISSIONS, ), PERM_CHANGE_SUBMISSIONS: (PERM_PARTIAL_SUBMISSIONS, ), PERM_DELETE_SUBMISSIONS: (PERM_PARTIAL_SUBMISSIONS, ), PERM_VALIDATE_SUBMISSIONS: (PERM_PARTIAL_SUBMISSIONS, ), } # Some permissions must be copied to KC KC_PERMISSIONS_MAP = { # keys are KPI's codenames, values are KC's PERM_CHANGE_SUBMISSIONS: 'change_xform', # "Can Edit" in KC UI PERM_VIEW_SUBMISSIONS: 'view_xform', # "Can View" in KC UI PERM_ADD_SUBMISSIONS: 'report_xform', # "Can submit to" in KC UI PERM_DELETE_SUBMISSIONS: 'delete_data_xform', # "Can Delete Data" in KC UI PERM_VALIDATE_SUBMISSIONS: 'validate_xform', # "Can Validate" in KC UI } KC_CONTENT_TYPE_KWARGS = {'app_label': 'logger', 'model': 'xform'} # KC records anonymous access as flags on the `XForm` KC_ANONYMOUS_PERMISSIONS_XFORM_FLAGS = { PERM_VIEW_SUBMISSIONS: { 'shared': True, 'shared_data': True } } def __str__(self): return '{} ({})'.format(self.name, self.uid) def adjust_content_on_save(self): """ This is called on save by default if content exists. Can be disabled / skipped by calling with parameter: asset.save(adjust_content=False) """ self._standardize(self.content) self._make_default_translation_first(self.content) self._strip_empty_rows(self.content) self._assign_kuids(self.content) self._autoname(self.content) self._unlink_list_items(self.content) self._remove_empty_expressions(self.content) settings = self.content['settings'] _title = settings.pop('form_title', None) id_string = settings.get('id_string') filename = self.summary.pop('filename', None) if filename: # if we have filename available, set the id_string # and/or form_title from the filename. if not id_string: id_string = sluggify_label(filename) settings['id_string'] = id_string if not _title: _title = filename if self.asset_type not in [ASSET_TYPE_SURVEY, ASSET_TYPE_TEMPLATE]: # instead of deleting the settings, simply clear them out self.content['settings'] = {} if _title is not None: self.name = _title def clone(self, version_uid=None): # not currently used, but this is how "to_clone_dict" should work return Asset.objects.create(**self.to_clone_dict(version_uid)) @property def deployed_versions(self): return self.asset_versions.filter( deployed=True).order_by('-date_modified') def get_ancestors_or_none(self): # ancestors are ordered from farthest to nearest if self.parent is not None: return self.parent.get_ancestors(include_self=True) else: return None def get_filters_for_partial_perm(self, user_id, perm=PERM_VIEW_SUBMISSIONS): """ Returns the list of filters for a specific permission `perm` and this specific asset. :param user_id: :param perm: see `constants.*_SUBMISSIONS` :return: """ if not perm.endswith( SUFFIX_SUBMISSIONS_PERMS) or perm == PERM_PARTIAL_SUBMISSIONS: raise BadPermissionsException( _('Only partial permissions for ' 'submissions are supported')) perms = self.get_partial_perms(user_id, with_filters=True) if perms: return perms.get(perm) return None def get_label_for_permission(self, permission_or_codename): try: codename = permission_or_codename.codename permission = permission_or_codename except AttributeError: codename = permission_or_codename permission = None try: label = self.ASSIGNABLE_PERMISSIONS_WITH_LABELS[codename] except KeyError: if not permission: # Seems expensive. Cache it? permission = Permission.objects.get( # `content_type` and `codename` are `unique_together` # https://github.com/django/django/blob/e893c0ad8b0b5b0a1e5be3345c287044868effc4/django/contrib/auth/models.py#L69 content_type=ContentType.objects.get_for_model(self), codename=codename) label = permission.name label = label.replace( '##asset_type_label##', # Raises TypeError if not coerced explicitly str(self.ASSET_TYPE_LABELS[self.asset_type])) return label def get_partial_perms(self, user_id, with_filters=False): """ Returns the list of permissions the user is restricted to, for this specific asset. If `with_filters` is `True`, it returns a dict of permissions (as keys) and the filters (as values) to apply on query to narrow down the results. For example: `get_partial_perms(user1_obj.id)` would return ``` ['view_submissions',] ``` `get_partial_perms(user1_obj.id, with_filters=True)` would return ``` { 'view_submissions: [ {'_submitted_by': {'$in': ['user1', 'user2']}}, {'_submitted_by': 'user3'} ], } ``` If user doesn't have any partial permissions, it returns `None`. :param user_obj: auth.User :param with_filters: boolean. Optional :return: list|dict|None """ perms = self.asset_partial_permissions.filter(user_id=user_id)\ .values_list("permissions", flat=True).first() if perms: if with_filters: return perms else: return list(perms) return None @property def has_active_hooks(self): """ Returns if asset has active hooks. Useful to update `kc.XForm.has_kpi_hooks` field. :return: {boolean} """ return self.hooks.filter(active=True).exists() @property def latest_deployed_version(self): return self.deployed_versions.first() @property def latest_version(self): versions = None try: versions = self.prefetched_latest_versions except AttributeError: versions = self.asset_versions.order_by('-date_modified') try: return versions[0] except IndexError: return None @staticmethod def optimize_queryset_for_list(queryset): """ Used by serializers to improve performance when listing assets """ queryset = queryset.defer( # Avoid pulling these from the database because they are often huge # and we don't need them for list views. 'content', 'report_styles' ).select_related( # We only need `username`, but `select_related('owner__username')` # actually pulled in the entire `auth_user` table under Django 1.8. # In Django 1.9+, "select_related() prohibits non-relational fields # for nested relations." 'owner', ).prefetch_related( # We previously prefetched `permissions__content_object`, but that # actually pulled the entirety of each permission's linked asset # from the database! For now, the solution is to remove # `content_object` here *and* from # `ObjectPermissionNestedSerializer`. 'permissions__permission', 'permissions__user', # `Prefetch(..., to_attr='prefetched_list')` stores the prefetched # related objects in a list (`prefetched_list`) that we can use in # other methods to avoid additional queries; see: # https://docs.djangoproject.com/en/1.8/ref/models/querysets/#prefetch-objects Prefetch('tags', to_attr='prefetched_tags'), Prefetch( 'asset_versions', queryset=AssetVersion.objects.order_by('-date_modified').only( 'uid', 'asset', 'date_modified', 'deployed'), to_attr='prefetched_latest_versions', ), ) return queryset def rename_translation(self, _from, _to): if not self._has_translations(self.content, 2): raise ValueError('no translations available') self._rename_translation(self.content, _from, _to) # todo: test and implement this method # todo 2019-04-25: Still needed, `revert_to_version` does the same? # def restore_version(self, uid): # _version_to_restore = self.asset_versions.get(uid=uid) # self.content = _version_to_restore.version_content # self.name = _version_to_restore.name def revert_to_version(self, version_uid): av = self.asset_versions.get(uid=version_uid) self.content = av.version_content self.save() def save(self, *args, **kwargs): if self.content is None: self.content = {} # in certain circumstances, we don't want content to # be altered on save. (e.g. on asset.deploy()) if kwargs.pop('adjust_content', True): self.adjust_content_on_save() # populate summary self._populate_summary() # infer asset_type only between question and block if self.asset_type in [ASSET_TYPE_QUESTION, ASSET_TYPE_BLOCK]: try: row_count = int(self.summary.get('row_count')) except TypeError: pass else: if row_count == 1: self.asset_type = ASSET_TYPE_QUESTION elif row_count > 1: self.asset_type = ASSET_TYPE_BLOCK self._populate_report_styles() _create_version = kwargs.pop('create_version', True) super().save(*args, **kwargs) if _create_version: self.asset_versions.create( name=self.name, version_content=self.content, _deployment_data=self._deployment_data, # asset_version.deployed is set in the # DeploymentSerializer deployed=False, ) @property def snapshot(self): return self._snapshot(regenerate=False) def to_clone_dict(self, version_uid=None, version=None): """ Returns a dictionary of the asset based on version_uid or version. If `version` is specified, there are no needs to provide `version_uid` and make another request to DB. :param version_uid: string :param version: AssetVersion :return: dict """ if not isinstance(version, AssetVersion): if version_uid: version = self.asset_versions.get(uid=version_uid) else: version = self.asset_versions.first() return { 'name': version.name, 'content': version.version_content, 'asset_type': self.asset_type, 'tag_string': self.tag_string, } def to_ss_structure(self): return flatten_content(self.content, in_place=False) @property def version__content_hash(self): # Avoid reading the propery `self.latest_version` more than once, since # it may execute a database query each time it's read latest_version = self.latest_version if latest_version: return latest_version.content_hash @property def version_id(self): # Avoid reading the propery `self.latest_version` more than once, since # it may execute a database query each time it's read latest_version = self.latest_version if latest_version: return latest_version.uid def _populate_report_styles(self): default = self.report_styles.get(DEFAULT_REPORTS_KEY, {}) specifieds = self.report_styles.get(SPECIFIC_REPORTS_KEY, {}) kuids_to_variable_names = self.report_styles.get('kuid_names', {}) for (index, row) in enumerate(self.content.get('survey', [])): if '$kuid' not in row: if 'name' in row: row['$kuid'] = json_hash([self.uid, row['name']]) else: row['$kuid'] = json_hash([self.uid, index, row]) _identifier = row.get('name', row['$kuid']) kuids_to_variable_names[_identifier] = row['$kuid'] if _identifier not in specifieds: specifieds[_identifier] = {} self.report_styles = { DEFAULT_REPORTS_KEY: default, SPECIFIC_REPORTS_KEY: specifieds, 'kuid_names': kuids_to_variable_names, } def _populate_summary(self): if self.content is None: self.content = {} self.summary = {} return analyzer = AssetContentAnalyzer(**self.content) self.summary = analyzer.summary @transaction.atomic def _snapshot(self, regenerate=True): asset_version = self.latest_version try: snapshot = AssetSnapshot.objects.get(asset=self, asset_version=asset_version) if regenerate: snapshot.delete() snapshot = False except AssetSnapshot.MultipleObjectsReturned: # how did multiple snapshots get here? snaps = AssetSnapshot.objects.filter(asset=self, asset_version=asset_version) snaps.delete() snapshot = False except AssetSnapshot.DoesNotExist: snapshot = False if not snapshot: if self.name != '': form_title = self.name else: _settings = self.content.get('settings', {}) form_title = _settings.get('id_string', 'Untitled') self._append(self.content, settings={ 'form_title': form_title, }) snapshot = AssetSnapshot.objects.create( asset=self, asset_version=asset_version, source=self.content) return snapshot def _update_partial_permissions(self, user_id, perm, remove=False, partial_perms=None): """ Updates partial permissions relation table according to `perm`. If `perm` == `PERM_PARTIAL_SUBMISSIONS`, then If `partial_perms` is not `None`, it should be a dict with filters mapped to their corresponding permission. Each filter is used to narrow down results when querying Mongo. e.g.: ``` { 'view_submissions': [{ '_submitted_by': { '$in': [ 'someuser', 'anotheruser' ] } }], } ``` Even if we can only restrict an user to view another's submissions so far, this code wants to be future-proof and supports other permissions such as: - `change_submissions` - `validate_submissions` `partial_perms` could be passed as: ``` { 'change_submissions': [{ '_submitted_by': { '$in': [ 'someuser', 'anotheruser' ] } }] 'validate_submissions': [{ '_submitted_by': 'someuser' }], } ``` :param user_id: int. :param perm: str. see Asset.ASSIGNABLE_PERMISSIONS :param remove: boolean. Default is false. :param partial_perms: dict. Default is None. :return: """ def clean_up_table(): # Because of the unique constraint, there should be only # one record that matches this query. # We don't look for record existence to avoid extra query. self.asset_partial_permissions.filter(user_id=user_id).delete() if perm == PERM_PARTIAL_SUBMISSIONS: if remove: clean_up_table() return if user_id == self.owner.pk: raise BadPermissionsException( _("Can not assign '{}' permission to owner".format(perm))) if not partial_perms: raise BadPermissionsException( _("Can not assign '{}' permission. " "Partial permissions are missing.".format(perm))) new_partial_perms = {} for partial_perm, filters in partial_perms.items(): implied_perms = [ implied_perm for implied_perm in self.get_implied_perms(partial_perm) if implied_perm.endswith(SUFFIX_SUBMISSIONS_PERMS) ] implied_perms.append(partial_perm) for implied_perm in implied_perms: if implied_perm not in new_partial_perms: new_partial_perms[implied_perm] = [] new_partial_perms[implied_perm] += filters AssetUserPartialPermission.objects.update_or_create( asset_id=self.pk, user_id=user_id, defaults={'permissions': new_partial_perms}) elif perm in self.CONTRADICTORY_PERMISSIONS.get( PERM_PARTIAL_SUBMISSIONS): clean_up_table()
class ImportExportTask(models.Model): """ A common base model for asynchronous import and exports. Must be subclassed to be useful. Subclasses must implement the `_run_task()` method """ class Meta: abstract = True CREATED = 'created' PROCESSING = 'processing' COMPLETE = 'complete' ERROR = 'error' STATUS_CHOICES = ( (CREATED, CREATED), (PROCESSING, PROCESSING), (ERROR, ERROR), (COMPLETE, COMPLETE), ) user = models.ForeignKey('auth.User', on_delete=models.CASCADE) data = JSONBField() messages = JSONBField(default=dict) status = models.CharField(choices=STATUS_CHOICES, max_length=32, default=CREATED) date_created = models.DateTimeField(auto_now_add=True) # date_expired = models.DateTimeField(null=True) def run(self): """ Starts the import/export job by calling the subclass' `_run_task()` method. Catches all exceptions! Suitable to be called by an asynchronous task runner (Celery) """ with transaction.atomic(): _refetched_self = self._meta.model.objects.get(pk=self.pk) self.status = _refetched_self.status del _refetched_self if self.status == self.COMPLETE: return elif self.status != self.CREATED: # possibly a concurrent task? raise Exception( 'only recently created {}s can be executed'.format( self._meta.model_name)) self.status = self.PROCESSING self.save(update_fields=['status']) msgs = defaultdict(list) try: # This method must be implemented by a subclass self._run_task(msgs) self.status = self.COMPLETE except Exception as err: msgs['error_type'] = type(err).__name__ msgs['error'] = str(err) self.status = self.ERROR logging.error('Failed to run %s: %s' % (self._meta.model_name, repr(err)), exc_info=True) self.messages.update(msgs) # Record the processing time for diagnostic purposes self.data['processing_time_seconds'] = ( datetime.datetime.now(self.date_created.tzinfo) - self.date_created).total_seconds() try: self.save(update_fields=['status', 'messages', 'data']) except TypeError as e: self.status = self.ERROR logging.error('Failed to save %s: %s' % (self._meta.model_name, repr(e)), exc_info=True) self.save(update_fields=['status'])
class Asset(ObjectPermissionMixin, DeployableMixin, XlsExportableMixin, FormpackXLSFormUtilsMixin, models.Model): name = models.CharField(max_length=255, blank=True, default='') date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) content = JSONBField(default=dict) summary = JSONBField(default=dict) report_styles = JSONBField(default=dict) report_custom = JSONBField(default=dict) map_styles = LazyDefaultJSONBField(default=dict) map_custom = LazyDefaultJSONBField(default=dict) asset_type = models.CharField(choices=ASSET_TYPES, max_length=20, default=ASSET_TYPE_SURVEY) parent = models.ForeignKey('Asset', related_name='children', null=True, blank=True, on_delete=models.CASCADE) owner = models.ForeignKey('auth.User', related_name='assets', null=True, on_delete=models.CASCADE) uid = KpiUidField(uid_prefix='a') tags = TaggableManager(manager=KpiTaggableManager) settings = JSONBField(default=dict) # `_deployment_data` must **NOT** be touched directly by anything except # the `deployment` property provided by `DeployableMixin`. # ToDo Move the field to another table with one-to-one relationship _deployment_data = JSONBField(default=dict) # JSON with subset of fields to share # { # 'enable': True, # 'fields': [] # shares all when empty # } data_sharing = LazyDefaultJSONBField(default=dict) # JSON with source assets' information # { # <source_uid>: { # 'fields': [] # includes all fields shared by source when empty # 'paired_data_uid': 'pdxxxxxxx' # auto-generated read-only # 'filename: 'xxxxx.xml' # }, # ... # <source_uid>: { # 'fields': [] # 'paired_data_uid': 'pdxxxxxxx' # 'filename: 'xxxxx.xml' # } # } paired_data = LazyDefaultJSONBField(default=dict) objects = AssetManager() @property def kind(self): return 'asset' class Meta: # Example in Django documentation represents `ordering` as a list # (even if it can be a list or a tuple). We enforce the type to `list` # because `rest_framework.filters.OrderingFilter` work with lists. # `AssetOrderingFilter` inherits from this class and it is used ` # in `AssetViewSet to sort the result. # It avoids back and forth between types and/or coercing where # ordering is needed ordering = [ '-date_modified', ] permissions = ( # change_, add_, and delete_asset are provided automatically # by Django (PERM_VIEW_ASSET, t('Can view asset')), (PERM_DISCOVER_ASSET, t('Can discover asset in public lists')), (PERM_MANAGE_ASSET, t('Can manage all aspects of asset')), # Permissions for collected data, i.e. submissions (PERM_ADD_SUBMISSIONS, t('Can submit data to asset')), (PERM_VIEW_SUBMISSIONS, t('Can view submitted data for asset')), (PERM_PARTIAL_SUBMISSIONS, t('Can make partial actions on ' 'submitted data for asset ' 'for specific users')), (PERM_CHANGE_SUBMISSIONS, t('Can modify submitted data for asset')), (PERM_DELETE_SUBMISSIONS, t('Can delete submitted data for asset')), (PERM_VALIDATE_SUBMISSIONS, t("Can validate submitted data asset")), # TEMPORARY Issue #1161: A flag to indicate that permissions came # solely from `sync_kobocat_xforms` and not from any user # interaction with KPI (PERM_FROM_KC_ONLY, 'INTERNAL USE ONLY; DO NOT ASSIGN')) # Since Django 2.1, 4 permissions are added for each registered model: # - add # - change # - delete # - view # See https://docs.djangoproject.com/en/2.2/topics/auth/default/#default-permissions # for more detail. # `view_asset` clashes with newly built-in one. # The simplest way to fix this is to keep old behaviour default_permissions = ('add', 'change', 'delete') # Labels for each `asset_type` as they should be presented to users. Can be # strings or callables if special logic is needed. Callables receive the # codename of the permission for which a label is being created ASSET_TYPE_LABELS_FOR_PERMISSIONS = { ASSET_TYPE_SURVEY: (lambda p: t('project') if p == PERM_MANAGE_ASSET else t('form')), ASSET_TYPE_TEMPLATE: t('template'), ASSET_TYPE_BLOCK: t('block'), ASSET_TYPE_QUESTION: t('question'), ASSET_TYPE_TEXT: t('text'), # unused? ASSET_TYPE_EMPTY: t('empty'), # unused? ASSET_TYPE_COLLECTION: t('collection'), } # Assignable permissions that are stored in the database. # The labels are templates used by `get_label_for_permission()`, which you # should call instead of accessing this dictionary directly ASSIGNABLE_PERMISSIONS_WITH_LABELS = { PERM_VIEW_ASSET: t('View ##asset_type_label##'), PERM_CHANGE_ASSET: t('Edit ##asset_type_label##'), PERM_DISCOVER_ASSET: t('Discover ##asset_type_label##'), PERM_MANAGE_ASSET: t('Manage ##asset_type_label##'), PERM_ADD_SUBMISSIONS: t('Add submissions'), PERM_VIEW_SUBMISSIONS: t('View submissions'), PERM_PARTIAL_SUBMISSIONS: { 'default': t('Act on submissions only from specific users'), PERM_VIEW_SUBMISSIONS: t('View submissions only from specific users'), PERM_CHANGE_SUBMISSIONS: t('Edit submissions only from specific users'), PERM_DELETE_SUBMISSIONS: t('Delete submissions only from specific users'), PERM_VALIDATE_SUBMISSIONS: t('Validate submissions only from specific users'), }, PERM_CHANGE_SUBMISSIONS: t('Edit submissions'), PERM_DELETE_SUBMISSIONS: t('Delete submissions'), PERM_VALIDATE_SUBMISSIONS: t('Validate submissions'), } ASSIGNABLE_PERMISSIONS = tuple(ASSIGNABLE_PERMISSIONS_WITH_LABELS.keys()) # Depending on our `asset_type`, only some permissions might be applicable ASSIGNABLE_PERMISSIONS_BY_TYPE = { ASSET_TYPE_SURVEY: tuple((p for p in ASSIGNABLE_PERMISSIONS if p != PERM_DISCOVER_ASSET)), ASSET_TYPE_TEMPLATE: ( PERM_VIEW_ASSET, PERM_CHANGE_ASSET, PERM_MANAGE_ASSET, ), ASSET_TYPE_BLOCK: ( PERM_VIEW_ASSET, PERM_CHANGE_ASSET, PERM_MANAGE_ASSET, ), ASSET_TYPE_QUESTION: ( PERM_VIEW_ASSET, PERM_CHANGE_ASSET, PERM_MANAGE_ASSET, ), ASSET_TYPE_TEXT: (), # unused? ASSET_TYPE_EMPTY: ( PERM_VIEW_ASSET, PERM_CHANGE_ASSET, PERM_MANAGE_ASSET, ), ASSET_TYPE_COLLECTION: ( PERM_VIEW_ASSET, PERM_CHANGE_ASSET, PERM_DISCOVER_ASSET, PERM_MANAGE_ASSET, ), } # Calculated permissions that are neither directly assignable nor stored # in the database, but instead implied by assignable permissions CALCULATED_PERMISSIONS = (PERM_DELETE_ASSET, ) # Only certain permissions can be inherited HERITABLE_PERMISSIONS = { # parent permission: child permission PERM_VIEW_ASSET: PERM_VIEW_ASSET, PERM_CHANGE_ASSET: PERM_CHANGE_ASSET } # Granting some permissions implies also granting other permissions IMPLIED_PERMISSIONS = { # Format: explicit: (implied, implied, ...) PERM_CHANGE_ASSET: (PERM_VIEW_ASSET, ), PERM_DISCOVER_ASSET: (PERM_VIEW_ASSET, ), PERM_MANAGE_ASSET: tuple((p for p in ASSIGNABLE_PERMISSIONS if p not in (PERM_MANAGE_ASSET, PERM_PARTIAL_SUBMISSIONS))), PERM_ADD_SUBMISSIONS: (PERM_VIEW_ASSET, ), PERM_VIEW_SUBMISSIONS: (PERM_VIEW_ASSET, ), PERM_PARTIAL_SUBMISSIONS: (PERM_VIEW_ASSET, ), PERM_CHANGE_SUBMISSIONS: ( PERM_VIEW_SUBMISSIONS, PERM_ADD_SUBMISSIONS, ), PERM_DELETE_SUBMISSIONS: (PERM_VIEW_SUBMISSIONS, ), PERM_VALIDATE_SUBMISSIONS: (PERM_VIEW_SUBMISSIONS, ), } CONTRADICTORY_PERMISSIONS = { PERM_PARTIAL_SUBMISSIONS: ( PERM_VIEW_SUBMISSIONS, PERM_CHANGE_SUBMISSIONS, PERM_DELETE_SUBMISSIONS, PERM_VALIDATE_SUBMISSIONS, PERM_MANAGE_ASSET, ), PERM_VIEW_SUBMISSIONS: (PERM_PARTIAL_SUBMISSIONS, ), PERM_CHANGE_SUBMISSIONS: (PERM_PARTIAL_SUBMISSIONS, ), PERM_DELETE_SUBMISSIONS: (PERM_PARTIAL_SUBMISSIONS, ), PERM_VALIDATE_SUBMISSIONS: (PERM_PARTIAL_SUBMISSIONS, ), } # Some permissions must be copied to KC KC_PERMISSIONS_MAP = { # keys are KPI's codenames, values are KC's PERM_CHANGE_SUBMISSIONS: 'change_xform', # "Can Edit" in KC UI PERM_VIEW_SUBMISSIONS: 'view_xform', # "Can View" in KC UI PERM_ADD_SUBMISSIONS: 'report_xform', # "Can submit to" in KC UI PERM_DELETE_SUBMISSIONS: 'delete_data_xform', # "Can Delete Data" in KC UI PERM_VALIDATE_SUBMISSIONS: 'validate_xform', # "Can Validate" in KC UI } KC_CONTENT_TYPE_KWARGS = {'app_label': 'logger', 'model': 'xform'} # KC records anonymous access as flags on the `XForm` KC_ANONYMOUS_PERMISSIONS_XFORM_FLAGS = { PERM_VIEW_SUBMISSIONS: { 'shared': True, 'shared_data': True } } def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # The two fields below are needed to keep a trace of the object state # before any alteration. See `__self.__copy_hidden_fields()` for details # They must be set with an invalid value for their counterparts to # be the comparison is accurate. self.__parent_id_copy = -1 self.__deployment_data_copy = None self.__copy_hidden_fields() def __str__(self): return '{} ({})'.format(self.name, self.uid) def adjust_content_on_save(self): """ This is called on save by default if content exists. Can be disabled / skipped by calling with parameter: asset.save(adjust_content=False) """ self._standardize(self.content) self._make_default_translation_first(self.content) self._strip_empty_rows(self.content) self._assign_kuids(self.content) self._autoname(self.content) self._unlink_list_items(self.content) self._remove_empty_expressions(self.content) settings = self.content['settings'] _title = settings.pop('form_title', None) id_string = settings.get('id_string') filename = self.summary.pop('filename', None) if filename: # if we have filename available, set the id_string # and/or form_title from the filename. if not id_string: id_string = sluggify_label(filename) settings['id_string'] = id_string if not _title: _title = filename if self.asset_type not in [ASSET_TYPE_SURVEY, ASSET_TYPE_TEMPLATE]: # instead of deleting the settings, simply clear them out self.content['settings'] = {} strip_kobo_locking_profile(self.content) if _title is not None: self.name = _title def clone(self, version_uid=None): # not currently used, but this is how "to_clone_dict" should work return Asset.objects.create(**self.to_clone_dict(version=version_uid)) def create_version(self) -> [AssetVersion, None]: """ Create a version of current asset. Asset has to belong to `ASSET_TYPE_WITH_CONTENT` otherwise no version is created and `None` is returned. """ if self.asset_type not in ASSET_TYPES_WITH_CONTENT: return return self.asset_versions.create( name=self.name, version_content=self.content, _deployment_data=self._deployment_data, # Any new version starts out as not-deployed, # even if the asset itself is already deployed. # Note: `asset_version.deployed` is set in the # serializer `DeploymentSerializer` deployed=False, ) @property def deployed_versions(self): return self.asset_versions.filter( deployed=True).order_by('-date_modified') @property def discoverable_when_public(self): # This property is only needed when `self` is a collection. # We want to make a distinction between a collection which is not # discoverable and an asset which is not a collection # (which implies cannot be discoverable) if self.asset_type != ASSET_TYPE_COLLECTION: return None return self.permissions.filter( permission__codename=PERM_DISCOVER_ASSET, user_id=settings.ANONYMOUS_USER_ID).exists() def get_filters_for_partial_perm( self, user_id: int, perm: str = PERM_VIEW_SUBMISSIONS) -> Union[list, None]: """ Returns the list of filters for a specific permission `perm` and this specific asset. `perm` can only one of the submission permissions. """ if (not perm.endswith(SUFFIX_SUBMISSIONS_PERMS) or perm == PERM_PARTIAL_SUBMISSIONS): raise BadPermissionsException( t('Only partial permissions for ' 'submissions are supported')) perms = self.get_partial_perms(user_id, with_filters=True) if perms: try: return perms[perm] except KeyError: # User has some partial permissions but not the good one. # Return a false condition to avoid showing any results. return [{'_id': -1}] return None def get_label_for_permission( self, permission_or_codename: Union[Permission, str]) -> str: """ Get the correct label for a permission (object or codename) based on the type of this asset """ try: codename = permission_or_codename.codename permission = permission_or_codename except AttributeError: codename = permission_or_codename permission = None try: label = self.ASSIGNABLE_PERMISSIONS_WITH_LABELS[codename] except KeyError: if permission: label = permission.name else: cached_code_names = get_cached_code_names() label = cached_code_names[codename]['name'] asset_type_label = self.ASSET_TYPE_LABELS_FOR_PERMISSIONS[ self.asset_type] try: # Some labels may be callables asset_type_label = asset_type_label(codename) except TypeError: # Others are just strings pass # For partial permissions, label is a dict. # There is no replacements to do in the nested labels, but these lines # are there to support in case we need it one day if isinstance(label, dict): labels = copy.deepcopy(label) for key_ in labels.keys(): labels[key_] = labels[key_].replace( '##asset_type_label##', # Raises TypeError if not coerced explicitly due to # ugettext_lazy() str(asset_type_label)) return labels else: return label.replace( '##asset_type_label##', # Raises TypeError if not coerced explicitly due to # ugettext_lazy() str(asset_type_label)) def get_partial_perms( self, user_id: int, with_filters: bool = False) -> Union[list, dict, None]: """ Returns the list of permissions the user is restricted to, for this specific asset. If `with_filters` is `True`, it returns a dict of permissions (as keys) and the filters (as values) to apply on query to narrow down the results. For example: `get_partial_perms(user1_obj.id)` would return ``` ['view_submissions',] ``` `get_partial_perms(user1_obj.id, with_filters=True)` would return ``` { 'view_submissions: [ {'_submitted_by': {'$in': ['user1', 'user2']}}, {'_submitted_by': 'user3'} ], } ``` If user doesn't have any partial permissions, it returns `None`. """ perms = self.asset_partial_permissions.filter(user_id=user_id)\ .values_list("permissions", flat=True).first() if perms: if with_filters: return perms else: return list(perms) return None @property def has_active_hooks(self): """ Returns if asset has active hooks. Useful to update `kc.XForm.has_kpi_hooks` field. :return: {boolean} """ return self.hooks.filter(active=True).exists() def has_subscribed_user(self, user_id): # This property is only needed when `self` is a collection. # We want to make a distinction between a collection which does not have # the subscribed user and an asset which is not a collection # (which implies cannot have subscriptions) if self.asset_type != ASSET_TYPE_COLLECTION: return None # ToDo: See if using a loop can reduce the number of SQL queries. return self.userassetsubscription_set.filter(user_id=user_id).exists() @property def latest_deployed_version(self): return self.deployed_versions.first() @property def latest_version(self): versions = None try: versions = self.prefetched_latest_versions except AttributeError: versions = self.asset_versions.order_by('-date_modified') try: return versions[0] except IndexError: return None @staticmethod def optimize_queryset_for_list(queryset): """ Used by serializers to improve performance when listing assets """ queryset = queryset.defer( # Avoid pulling these from the database because they are often huge # and we don't need them for list views. 'content', 'report_styles' ).select_related( # We only need `username`, but `select_related('owner__username')` # actually pulled in the entire `auth_user` table under Django 1.8. # In Django 1.9+, "select_related() prohibits non-relational fields # for nested relations." 'owner', ).prefetch_related( 'permissions__permission', 'permissions__user', # `Prefetch(..., to_attr='prefetched_list')` stores the prefetched # related objects in a list (`prefetched_list`) that we can use in # other methods to avoid additional queries; see: # https://docs.djangoproject.com/en/1.8/ref/models/querysets/#prefetch-objects Prefetch('tags', to_attr='prefetched_tags'), Prefetch( 'asset_versions', queryset=AssetVersion.objects.order_by('-date_modified').only( 'uid', 'asset', 'date_modified', 'deployed'), to_attr='prefetched_latest_versions', ), ) return queryset def refresh_from_db(self, using=None, fields=None): super().refresh_from_db(using=using, fields=fields) # Refresh hidden fields too self.__copy_hidden_fields(fields) def rename_translation(self, _from, _to): if not self._has_translations(self.content, 2): raise ValueError('no translations available') self._rename_translation(self.content, _from, _to) # todo: test and implement this method # todo 2019-04-25: Still needed, `revert_to_version` does the same? # def restore_version(self, uid): # _version_to_restore = self.asset_versions.get(uid=uid) # self.content = _version_to_restore.version_content # self.name = _version_to_restore.name def revert_to_version(self, version_uid): av = self.asset_versions.get(uid=version_uid) self.content = av.version_content self.save() def save(self, force_insert=False, force_update=False, update_fields=None, adjust_content=True, create_version=True, update_parent_languages=True, *args, **kwargs): is_new = self.pk is None if self.asset_type not in ASSET_TYPES_WITH_CONTENT: # so long as all of the operations in this overridden `save()` # method pertain to content, bail out if it's impossible for this # asset to have content in the first place super().save(force_insert=force_insert, force_update=force_update, update_fields=update_fields, *args, **kwargs) return if self.content is None: self.content = {} # in certain circumstances, we don't want content to # be altered on save. (e.g. on asset.deploy()) if adjust_content: self.adjust_content_on_save() # populate summary self._populate_summary() # infer asset_type only between question and block if self.asset_type in [ASSET_TYPE_QUESTION, ASSET_TYPE_BLOCK]: try: row_count = int(self.summary.get('row_count')) except TypeError: pass else: if row_count == 1: self.asset_type = ASSET_TYPE_QUESTION elif row_count > 1: self.asset_type = ASSET_TYPE_BLOCK self._populate_report_styles() # Ensure `_deployment_data` is not saved directly try: stored_data_key = self._deployment_data['_stored_data_key'] except KeyError: if self._deployment_data != self.__deployment_data_copy: raise DeploymentDataException else: if stored_data_key != self.deployment.stored_data_key: raise DeploymentDataException else: self._deployment_data.pop('_stored_data_key', None) self.__copy_hidden_fields() super().save(force_insert=force_insert, force_update=force_update, update_fields=update_fields, *args, **kwargs) # Update languages for parent and previous parent. # e.g. if a survey has been moved from one collection to another, # we want both collections to be updated. if self.parent is not None and update_parent_languages: if (self.parent_id != self.__parent_id_copy and self.__parent_id_copy is not None): try: previous_parent = Asset.objects.get( pk=self.__parent_id_copy) previous_parent.update_languages() self.__parent_id_copy = self.parent_id except Asset.DoesNotExist: pass # If object is new, we can add its languages to its parent without # worrying about removing its old values. It avoids an extra query. if is_new: self.parent.update_languages([self]) else: # Otherwise, because we cannot know which languages are from # this object, update will be performed with all parent's # children. self.parent.update_languages() if self.has_deployment: self.deployment.sync_media_files(AssetFile.PAIRED_DATA) if create_version: self.create_version() @property def snapshot(self): return self._snapshot(regenerate=False) @property def tag_string(self): try: tag_list = self.prefetched_tags except AttributeError: tag_names = self.tags.values_list('name', flat=True) else: tag_names = [t.name for t in tag_list] return ','.join(tag_names) @tag_string.setter def tag_string(self, value): intended_tags = value.split(',') self.tags.set(*intended_tags) def to_clone_dict(self, version: Union[str, AssetVersion] = None) -> dict: """ Returns a dictionary of the asset based on its version. :param version: Optional. It can be an object or its unique id :return dict """ if not isinstance(version, AssetVersion): if version: version = self.asset_versions.get(uid=version) else: version = self.asset_versions.first() if not version: version = self.create_version() return { 'name': version.name, 'content': version.version_content, 'asset_type': self.asset_type, 'tag_string': self.tag_string, } def to_ss_structure(self): return flatten_content(self.content, in_place=False) def update_languages(self, children=None): """ Updates object's languages by aggregating all its children's languages Args: children (list<Asset>): Optional. When specified, `children`'s languages are merged with `self`'s languages. Otherwise, when it's `None`, DB is fetched to build the list according to `self.children` """ # If object is not a collection, it should not have any children. # No need to go further. if self.asset_type != ASSET_TYPE_COLLECTION: return obj_languages = self.summary.get('languages', []) languages = set() if children: languages = set(obj_languages) children_languages = [ child.summary.get('languages') for child in children if child.summary.get('languages') ] else: children_languages = list( self.children.values_list( 'summary__languages', flat=True).exclude( Q(summary__languages=[]) | Q(summary__languages=[None])).order_by()) if children_languages: # Flatten `children_languages` to 1-dimension list. languages.update(reduce(add, children_languages)) languages.discard(None) # Object of type set is not JSON serializable languages = list(languages) # If languages are still the same, no needs to update the object if sorted(obj_languages) == sorted(languages): return self.summary['languages'] = languages self.save(update_fields=['summary']) @property def version__content_hash(self): # Avoid reading the property `self.latest_version` more than once, since # it may execute a database query each time it's read latest_version = self.latest_version if latest_version: return latest_version.content_hash @property def version_id(self): # Avoid reading the property `self.latest_version` more than once, since # it may execute a database query each time it's read latest_version = self.latest_version if latest_version: return latest_version.uid @property def version_number_and_date(self) -> str: # Returns the count of all deployed versions (plus one for the current # version if it is not deployed) and the date the asset was last # modified count = self.deployed_versions.count() if not self.latest_version.deployed: count = count + 1 return f'{count} {self.date_modified:(%Y-%m-%d %H:%M:%S)}' # TODO: take leading underscore off of `_snapshot()` and call it directly? # we would also have to remove or rename the `snapshot` property def versioned_snapshot( self, version_uid: str, root_node_name: Optional[str] = None) -> AssetSnapshot: return self._snapshot( regenerate=False, version_uid=version_uid, root_node_name=root_node_name, ) def _populate_report_styles(self): default = self.report_styles.get(DEFAULT_REPORTS_KEY, {}) specifieds = self.report_styles.get(SPECIFIC_REPORTS_KEY, {}) kuids_to_variable_names = self.report_styles.get('kuid_names', {}) for (index, row) in enumerate(self.content.get('survey', [])): if '$kuid' not in row: if 'name' in row: row['$kuid'] = json_hash([self.uid, row['name']]) else: row['$kuid'] = json_hash([self.uid, index, row]) _identifier = row.get('name', row['$kuid']) kuids_to_variable_names[_identifier] = row['$kuid'] if _identifier not in specifieds: specifieds[_identifier] = {} self.report_styles = { DEFAULT_REPORTS_KEY: default, SPECIFIC_REPORTS_KEY: specifieds, 'kuid_names': kuids_to_variable_names, } def _populate_summary(self): if self.content is None: self.content = {} self.summary = {} return analyzer = AssetContentAnalyzer(**self.content) self.summary = analyzer.summary @transaction.atomic def _snapshot( self, regenerate: bool = True, version_uid: Optional[str] = None, root_node_name: Optional[str] = None, ) -> AssetSnapshot: if version_uid: asset_version = self.asset_versions.get(uid=version_uid) else: asset_version = self.latest_version try: snapshot = AssetSnapshot.objects.get(asset=self, asset_version=asset_version) if regenerate: snapshot.delete() snapshot = False except AssetSnapshot.MultipleObjectsReturned: # how did multiple snapshots get here? snaps = AssetSnapshot.objects.filter(asset=self, asset_version=asset_version) snaps.delete() snapshot = False except AssetSnapshot.DoesNotExist: snapshot = False if not snapshot: try: form_title = asset_version.form_title content = asset_version.version_content except AttributeError: form_title = self.form_title content = self.content settings_ = {'form_title': form_title} if root_node_name: # `name` may not sound like the right setting to control the # XML root node name, but it is, according to the XLSForm # specification: # https://xlsform.org/en/#specify-xforms-root-node-name settings_['name'] = root_node_name settings_['id_string'] = root_node_name self._append(content, settings=settings_) snapshot = AssetSnapshot.objects.create( asset=self, asset_version=asset_version, source=content) return snapshot def _update_partial_permissions( self, user: '******', perm: str, remove: bool = False, partial_perms: Optional[dict] = None, ): """ Stores, updates, and removes permissions that apply only to a subset of submissions in a project (also called row-level permissions or partial permissions). If `perm = PERM_PARTIAL_SUBMISSIONS`, it must be accompanied by `partial_perms`, which is a dictionary of permissions mapped to MongoDB filters. Each key of that dictionary is a permission string (codename), and each value is a list of MongoDB queries that specify which submissions the permission affects. A submission is affected if it matches *ANY* of the queries in the list. For example, to allow `user` to edit submissions made by 'alice' or 'bob', and to allow `user` also to validate only submissions made by 'bob', the following `partial_perms` could be used: ``` { 'change_submissions': [{ '_submitted_by': { '$in': [ 'alice', 'bob' ] } }], 'validate_submissions': [{ '_submitted_by': 'bob' }], } ``` If `perm` is something other than `PERM_PARTIAL_SUBMISSIONS`, and that permission contradicts `PERM_PARTIAL_SUBMISSIONS`, *all* partial permission assignments for `user` on this asset are removed from the database. If the permission does not conflict, no action is taken. `remove = True` deletes all partial permissions assignments for `user` on this asset. """ def clean_up_table(): # Because of the unique constraint, there should be only # one record that matches this query. # We don't look for record existence to avoid extra query. self.asset_partial_permissions.filter(user_id=user.pk).delete() if perm == PERM_PARTIAL_SUBMISSIONS: if remove: clean_up_table() return if user.pk == self.owner.pk: raise BadPermissionsException( t("Can not assign '{}' permission to owner".format(perm))) if not partial_perms: raise BadPermissionsException( t("Can not assign '{}' permission. " "Partial permissions are missing.".format(perm))) new_partial_perms = AssetUserPartialPermission\ .update_partial_perms_to_include_implied( self, partial_perms ) AssetUserPartialPermission.objects.update_or_create( asset_id=self.pk, user_id=user.pk, defaults={'permissions': new_partial_perms}) # There are no real partial permissions for 'add_submissions' but # 'change_submissions' implies it. So if 'add_submissions' is in the # partial permissions list, it must be assigned to the user to the # user as well to let them perform edit actions on their subset of # data. Otherwise, KC will reject some actions. if PERM_ADD_SUBMISSIONS in new_partial_perms: self.assign_perm(user_obj=user, perm=PERM_ADD_SUBMISSIONS, defer_recalc=True) elif perm in self.CONTRADICTORY_PERMISSIONS.get( PERM_PARTIAL_SUBMISSIONS): clean_up_table() def __copy_hidden_fields(self, fields: Optional[list] = None): """ Save a copy of `parent_id` and `_deployment_data` for these purposes `save()` respectively. - `self.__parent_id_copy` is used to detect whether asset is linked a different parent - `self.__deployment_data_copy` is used to detect whether `_deployment_data` has been altered directly """ # When fields are deferred, Django instantiates another copy # of the current Asset object to retrieve the value of the # requested field. Because we need to get a copy at the very # first beginning of the life of the object, this method is # called in the object constructor. Thus, trying to copy # deferred fields would create an infinite loop. # If `fields` is provided, fields are no longer deferred and should be # copied right away. if (fields is None and 'parent_id' not in self.get_deferred_fields() or fields and 'parent_id' in fields): self.__parent_id_copy = self.parent_id if (fields is None and '_deployment_data' not in self.get_deferred_fields() or fields and '_deployment_data' in fields): self.__deployment_data_copy = copy.deepcopy(self._deployment_data)
class ModelWithJSONBFieldTest(DirtyFieldsMixin, models.Model): jsonb_field = JSONBField()
class Hook(models.Model): # Export types XML = "xml" JSON = "json" # Authentication levels NO_AUTH = "no_auth" BASIC_AUTH = "basic_auth" # Export types list EXPORT_TYPE_CHOICES = ((XML, XML), (JSON, JSON)) # Authentication levels list AUTHENTICATION_LEVEL_CHOICES = ((NO_AUTH, NO_AUTH), (BASIC_AUTH, BASIC_AUTH)) asset = models.ForeignKey("kpi.Asset", related_name="hooks", on_delete=models.CASCADE) uid = KpiUidField(uid_prefix="h") name = models.CharField(max_length=255, blank=False) endpoint = models.CharField(max_length=500, blank=False) active = models.BooleanField(default=True) export_type = models.CharField(choices=EXPORT_TYPE_CHOICES, default=JSON, max_length=10) auth_level = models.CharField(choices=AUTHENTICATION_LEVEL_CHOICES, default=NO_AUTH, max_length=10) settings = JSONBField(default=dict) date_created = models.DateTimeField(default=timezone.now) date_modified = models.DateTimeField(default=timezone.now) email_notification = models.BooleanField(default=True) subset_fields = ArrayField( models.CharField(max_length=500), blank=True, default=list, ) payload_template = models.TextField(null=True, blank=True) class Meta: ordering = ["name"] def __init__(self, *args, **kwargs): self.__totals = {} super().__init__(*args, **kwargs) def save(self, *args, **kwargs): # Update date_modified each time object is saved self.date_modified = timezone.now() super().save(*args, **kwargs) def __str__(self): return "%s:%s - %s" % (self.asset, self.name, self.endpoint) def get_service_definition(self): mod = import_module("kobo.apps.hook.services.service_{}".format( self.export_type)) return getattr(mod, "ServiceDefinition") @property def success_count(self): if not self.__totals: self._get_totals() return self.__totals.get(HOOK_LOG_SUCCESS) @property def failed_count(self): if not self.__totals: self._get_totals() return self.__totals.get(HOOK_LOG_FAILED) @property def pending_count(self): if not self.__totals: self._get_totals() return self.__totals.get(HOOK_LOG_PENDING) def _get_totals(self): # TODO add some cache queryset = self.logs.values("status").annotate( values_count=models.Count("status")) queryset.query.clear_ordering(True) # Initialize totals self.__totals = { HOOK_LOG_SUCCESS: 0, HOOK_LOG_FAILED: 0, HOOK_LOG_PENDING: 0 } for record in queryset: self.__totals[record.get("status")] = record.get("values_count") def reset_totals(self): # TODO remove cache when it's enabled self.__totals = {}
class AssetUserPartialPermission(models.Model): """ Many-to-Many table which provides users' permissions on other users' submissions For example, - Asset: - uid: aAAAAAA - id: 1 - User: - username: someuser - id: 1 We want someuser to be able to view otheruser's submissions Records should be `permissions` is dict formatted as is: asset_id | user_id | permissions 1 | 1 | {"someuser": ["view_submissions"]} Using a list per user for permissions, gives the opportunity to add other permissions such as `change_submissions`, `delete_submissions` for later purpose """ class Meta: unique_together = [['asset', 'user']] asset = models.ForeignKey('Asset', related_name='asset_partial_permissions', on_delete=models.CASCADE) user = models.ForeignKey('auth.User', related_name='user_partial_permissions', on_delete=models.CASCADE) permissions = JSONBField(default=dict) date_created = models.DateTimeField(default=timezone.now) date_modified = models.DateTimeField(default=timezone.now) def save(self, *args, **kwargs): if self.pk is not None: self.date_modified = timezone.now() super().save(*args, **kwargs) @staticmethod def update_partial_perms_to_include_implied( asset: 'kpi.models.Asset', partial_perms: dict ) -> dict: new_partial_perms = defaultdict(list) in_op = MongoHelper.IN_OPERATOR for partial_perm, filters in partial_perms.items(): if partial_perm not in new_partial_perms: new_partial_perms[partial_perm] = filters # TODO: omit `add_submissions`? It's required at the asset # level for any kind of editing (e.g. partial # `change_submissions` requires asset-wide `add_submissions`), # but it doesn't make sense to restrict adding submissions # "only to those submissions that match some criteria" implied_perms = [ implied_perm for implied_perm in asset.get_implied_perms( partial_perm, for_instance=asset ) if implied_perm.endswith(SUFFIX_SUBMISSIONS_PERMS) ] for implied_perm in implied_perms: if ( implied_perm not in new_partial_perms and implied_perm in partial_perms ): new_partial_perms[implied_perm] = partial_perms[ implied_perm ] new_partial_perm = new_partial_perms[implied_perm] # Trivial case, i.e.: permissions are built with front end. # All permissions have only one filter and the same filter # Example: # ``` # partial_perms = { # 'view_submissions' : [ # {'_submitted_by': {'$in': ['johndoe']}} # ], # 'delete_submissions': [ # {'_submitted_by': {'$in': ['quidam']}} # ] # } # ``` # should give # ``` # new_partial_perms = { # 'view_submissions' : [ # {'_submitted_by': {'$in': ['johndoe', 'quidam']}} # ], # 'delete_submissions': [ # {'_submitted_by': {'$in': ['quidam']}} # ] # } if ( len(filters) == 1 and len(new_partial_perm) == 1 and isinstance(new_partial_perm, list) ): current_filters = new_partial_perms[implied_perm][0] filter_ = filters[0] # Front end only supports `_submitted_by`, but if users # use the API, it could be something else. filter_key = list(filter_)[0] try: new_value = filter_[filter_key][in_op] current_values = current_filters[filter_key][in_op] except (KeyError, TypeError): pass else: new_partial_perm[0][filter_key][in_op] = list( set(current_values + new_value) ) continue # As said earlier, front end only supports `'_submitted_by'` # filter, but many different and more complex filters could # be used. # If we reach these lines, it means conditions cannot be # merged, so we concatenate then with an `OR` operator. # Example: # ``` # partial_perms = { # 'view_submissions' : [{'_submitted_by': 'johndoe'}], # 'delete_submissions': [ # {'_submission_date': {'$lte': '2021-01-01'}, # {'_submission_date': {'$gte': '2020-01-01'} # ] # } # ``` # should give # ``` # new_partial_perms = { # 'view_submissions' : [ # [{'_submitted_by': 'johndoe'}], # [ # {'_submission_date': {'$lte': '2021-01-01'}, # {'_submission_date': {'$gte': '2020-01-01'} # ] # }, # 'delete_submissions': [ # {'_submission_date': {'$lte': '2021-01-01'}, # {'_submission_date': {'$gte': '2020-01-01'} # ] # } # To avoid more complexity (and different syntax than # trivial case), we delegate to MongoHelper the task to join # lists with the `$or` operator. try: new_partial_perm = new_partial_perms[implied_perm][0] except IndexError: # If we get an IndexError, implied permission does not # belong to current assignment. Let's copy the filters # new_partial_perms[implied_perm] = filters else: if not isinstance(new_partial_perm, list): new_partial_perms[implied_perm] = [ filters, new_partial_perms[implied_perm], ] else: new_partial_perms[implied_perm].append(filters) return new_partial_perms
class AssetFile(OpenRosaManifestInterface, models.Model): # More to come! MAP_LAYER = 'map_layer' FORM_MEDIA = 'form_media' TYPE_CHOICES = ( (MAP_LAYER, MAP_LAYER), (FORM_MEDIA, FORM_MEDIA), ) ALLOWED_MIME_TYPES = { FORM_MEDIA: ('image', 'audio', 'video', 'text/csv', 'application/xml'), MAP_LAYER: ( 'text/csv', 'application/vnd.google-earth.kml+xml', 'application/vnd.google-earth.kmz', 'application/wkt', 'application/geo+json', 'application/json', ), } uid = KpiUidField(uid_prefix='af') asset = models.ForeignKey('Asset', related_name='asset_files', on_delete=models.CASCADE) # Keep track of the uploading user, who could be anyone with `change_asset` # rights, not just the asset owner user = models.ForeignKey('auth.User', related_name='asset_files', on_delete=models.CASCADE) file_type = models.CharField(choices=TYPE_CHOICES, max_length=32) description = models.CharField(max_length=255) date_created = models.DateTimeField(default=timezone.now) content = PrivateFileField(upload_to=upload_to, max_length=380, null=True) metadata = JSONBField(default=dict) date_deleted = models.DateTimeField(null=True, default=None) def delete(self, using=None, keep_parents=False, force=False): # Delete object and files on storage if `force` is True or file type # is anything else than 'form_media' if force or self.file_type != self.FORM_MEDIA: if not self.is_remote_url: self.content.delete(save=False) return super().delete(using=using, keep_parents=keep_parents) # Otherwise, just flag the file as deleted. self.date_deleted = timezone.now() self.save(update_fields=['date_deleted']) @property def filename(self): """ Implements `OpenRosaManifestInterface.filename()` """ if hasattr(self, '__filename'): return self.__filename self.set_filename() self.__filename = self.metadata['filename'] return self.__filename def get_download_url(self, request): """ Implements `OpenRosaManifestInterface.get_download_url()` """ return reverse('asset-file-content', args=(self.asset.uid, self.uid), request=request) @staticmethod def get_path(asset, file_type, filename): return posixpath.join(asset.owner.username, 'asset_files', asset.uid, file_type, filename) @property def hash(self): """ Implements `OpenRosaManifestInterface.hash()` """ if hasattr(self, '__hash'): return self.__hash self.set_hash() self.__hash = self.metadata['hash'] return self.__hash @property def is_remote_url(self): try: self.metadata['redirect_url'] except KeyError: return False return True def save(self, force_insert=False, force_update=False, using=None, update_fields=None): if self.pk is None: self.set_filename() self.set_hash() self.set_mimetype() return super().save(force_insert, force_update, using, update_fields) def set_filename(self): if not self.metadata.get('filename'): self.metadata['filename'] = self.content.name def set_hash(self): if not self.metadata.get('hash'): if self.is_remote_url: md5_hash = get_hash(self.metadata['redirect_url']) else: md5_hash = get_hash(self.content.file.read()) self.metadata['hash'] = f'md5:{md5_hash}' def set_mimetype(self): mimetype, _ = guess_type(self.metadata['filename']) self.metadata['mimetype'] = mimetype