def test_kuid_persists(): initial_kuid_1 = 'aaaa1111' initial_kuid_2 = 'bbbb2222' asset = Asset( content={ 'survey': [ { 'type': 'text', 'name': 'abc', '$kuid': initial_kuid_1 }, { 'type': 'text', 'name': 'def', '$kuid': initial_kuid_2 }, ], }) # kobo_specific_types=True avoids calling _strip_kuids # so, can we assume that kuids are supposed to remain? content = asset.ordered_xlsform_content(kobo_specific_types=True) # kuids are stripped in "kobo_to_xlsform.to_xlsform_structure(...)" assert '$kuid' in content['survey'][0] assert content['survey'][0].get('$kuid') == initial_kuid_1 assert '$kuid' in content['survey'][1] assert content['survey'][1].get('$kuid') == initial_kuid_2
def _create_cloned_asset(self): asset = Asset() asset.owner = self.asset.owner asset.content = self.asset.content asset.save() asset.deploy(backend='mock', active=True) asset.save() return asset
def test_remove_empty_expressions(): a1 = Asset(asset_type='survey', content={}) c1 = {'survey': [{'relevant': ''}]} a1._remove_empty_expressions(c1) assert _r1(c1) == {} c1 = {'survey': [{'bind': None}]} a1._remove_empty_expressions(c1) assert _r1(c1) == {}
def test_expand_twice(): a1 = Asset(asset_type='survey', content={'survey': [{'type': 'note', 'label::English': 'english', 'hint::English': 'hint', }]}) a1.adjust_content_on_save() assert 'translations' in a1.content assert len(a1.content['translations']) > 0 assert 'translated' in a1.content assert len(a1.content['translated']) > 0 assert sorted(a1.content['translated']) == ['hint', 'label']
def test_save_transformations(): a1 = Asset(asset_type='survey', content={}) content = color_picker_asset_content() a1._standardize(content) a1._strip_empty_rows(content) a1._assign_kuids(content) form_title = a1.pop_setting(content, 'form_title') a1._autoname(content) assert 'schema' in content assert content['translations'] == [None] assert form_title == 'color picker' assert content['settings'] == {'id_string': 'colorpik'}
def test_anonymous_get_only_owner_s_assignments(self): self.client.logout() permission_list_response = self.client.get(self.asset_permissions_list_url, format='json') self.assertEqual(permission_list_response.status_code, status.HTTP_200_OK) admin_perms = self.asset.get_perms(self.admin) results = permission_list_response.data # Get admin permissions. expected_perms = [] for admin_perm in admin_perms: if admin_perm in Asset.get_assignable_permissions(): expected_perms.append((self.admin.username, admin_perm)) expected_perms = sorted(expected_perms, key=lambda element: (element[0], element[1])) obj_perms = [] for assignment in results: object_permission = self.url_to_obj(assignment.get('url')) obj_perms.append((object_permission.user.username, object_permission.permission.codename)) obj_perms = sorted(obj_perms, key=lambda element: (element[0], element[1])) self.assertEqual(expected_perms, obj_perms)
def test_editors_see_only_self_anon_and_owner_assignments(self): self.client.login(username='******', password='******') permission_list_response = self.client.get( self.get_asset_perm_assignment_list_url(self.asset), format='json') self.assertEqual(permission_list_response.status_code, status.HTTP_200_OK) results = permission_list_response.data assignable_perms = Asset.get_assignable_permissions() expected_perms = [] for user in [ self.admin, self.someuser, # Permissions assigned to self.anotheruser must not appear get_anonymous_user(), ]: user_perms = self.asset.get_perms(user) expected_perms.extend( (user.username, perm) for perm in set(user_perms).intersection(assignable_perms)) expected_perms = sorted(expected_perms, key=lambda element: (element[0], element[1])) obj_perms = [] for assignment in results: object_permission = self.url_to_obj(assignment.get('url')) obj_perms.append(( object_permission.user.username, object_permission.permission.codename, )) obj_perms = sorted(obj_perms, key=lambda element: (element[0], element[1])) self.assertEqual(expected_perms, obj_perms)
def validate_parent(self, parent: Asset) -> Asset: request = self.context['request'] user = request.user if user.is_anonymous: user = get_anonymous_user() # Validate first if user can update the current parent if self.instance and self.instance.parent is not None: if not self.instance.parent.has_perm(user, PERM_CHANGE_ASSET): raise serializers.ValidationError( _('User cannot update current parent collection')) # Target collection is `None`, no need to check permissions if parent is None: return parent # `user` must have write access to target parent before being able to # move the asset. parent_perms = parent.get_perms(user) if PERM_VIEW_ASSET not in parent_perms: raise serializers.ValidationError(_('Target collection not found')) if PERM_CHANGE_ASSET not in parent_perms: raise serializers.ValidationError( _('User cannot update target parent collection')) return parent
def _import_asset(asset, parent_collection=None, asset_type='survey'): survey_dict = _csv_to_dict(asset.body) obj = { 'name': asset.name, 'date_created': asset.date_created, 'date_modified': asset.date_modified, 'asset_type': asset_type, 'owner': user, } if parent_collection is not None: obj['parent'] = parent_collection del obj['name'] new_asset = Asset(**obj) _set_auto_field_update(Asset, "date_created", False) _set_auto_field_update(Asset, "date_modified", False) new_asset.content = survey_dict new_asset.date_created = obj['date_created'] new_asset.date_modified = obj['date_modified'] new_asset.save() _set_auto_field_update(Asset, "date_created", True) _set_auto_field_update(Asset, "date_modified", True) # Note on the old draft the uid of the new asset asset.kpi_asset_uid = new_asset.uid asset.save() return new_asset
def test_bulk_assign_permissions(self): # TODO Improve this test permission_list_response = self.client.get(self.asset_permissions_list_url, format='json') self.assertEqual(permission_list_response.status_code, status.HTTP_200_OK) total = len(permission_list_response.data) # Add number of permissions added with 'view_asset' total += len(Asset.get_implied_perms(PERM_VIEW_ASSET)) + 1 # Add number of permissions added with 'change_asset' total += len(Asset.get_implied_perms(PERM_CHANGE_ASSET)) + 1 response = self._logged_user_gives_permissions([ ('someuser', PERM_VIEW_ASSET), ('someuser', PERM_VIEW_ASSET), # Add a duplicate which should not count ('anotheruser', PERM_CHANGE_ASSET) ]) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(len(response.data), total)
def setUp(self): self.client.login(username="******", password="******") self.someuser = User.objects.get(username="******") self.asset = a = Asset() a.name = 'Two points and one text' a.owner = self.someuser a.asset_type = 'survey' a.content = { 'survey': [ { 'name': 'geo1', 'type': 'geopoint', 'label': 'Where were you?' }, { 'name': 'geo2', 'type': 'geopoint', 'label': 'Where are you?' }, { 'name': 'text', 'type': 'text', 'label': 'How are you?' }, ] } a.save() a.deploy(backend='mock', active=True) a.save() v_uid = a.latest_deployed_version.uid self.submissions = [ { '__version__': v_uid, 'geo1': '10.11 10.12 10.13 10.14', 'geo2': '10.21 10.22 10.23 10.24', 'text': 'Tired', }, { '__version__': v_uid, 'geo1': '20.11 20.12 20.13 20.14', 'geo2': '20.21 20.22 20.23 20.24', 'text': 'Relieved', }, { '__version__': v_uid, 'geo1': '30.11 30.12 30.13 30.14', 'geo2': '30.21 30.22 30.23 30.24', 'text': 'Excited', }, ] a.deployment.mock_submissions(self.submissions) a.deployment.set_namespace(self.URL_NAMESPACE) self.submission_list_url = a.deployment.submission_list_url
def test_autoname_handles_non_latin_labels_with_kobo_score_and_kobo_rank(): a = Asset(asset_type='survey', content={}) content = { 'survey': [ { 'type': 'score__row', 'label': ['नमस्ते'] }, { 'type': 'rank__level', 'label': ['नमस्ते'] }, ] } a._standardize(content) a._strip_empty_rows(content) a._assign_kuids(content) a._autoname(content) for row in content['survey']: assert row['$autoname'].startswith('select_one')
def test_export_uid_filter(self): assert self.user.username == 'someuser' def _create_export_task(asset): export_task = ExportTask() export_task.user = self.user export_task.data = { 'source': reverse('asset-detail', args=[asset.uid]), 'type': 'csv' } messages = defaultdict(list) export_task._run_task(messages) return export_task matching_export = _create_export_task(self.asset) # Create a clone and generate an export from it excluded_asset = Asset() excluded_asset.owner = self.asset.owner excluded_asset.content = self.asset.content excluded_asset.save() excluded_asset.deploy(backend='mock', active=True) excluded_asset.save() excluded_export = _create_export_task(excluded_asset) # Retrieve all the exports unfiltered self.client.login(username='******', password='******') list_url = reverse(self._get_endpoint('exporttask-list')) response = self.client.get(list_url) assert response.status_code == status.HTTP_200_OK assert response.json()['count'] == 2 # Retrieve the exports filtered by a single asset uid filter_url = f'{list_url}?q=source:{self.asset.uid}' response = self.client.get(filter_url) assert response.status_code == status.HTTP_200_OK response_dict = response.json() assert response_dict['count'] == 1 assert self.asset.uid in response_dict['results'][0]['data']['source']
def setUp(self): self.client.login(username='******', password='******') self.current_username = '******' self.asset = Asset.objects.filter(owner__username='******').first() self.list_url = reverse(self._get_endpoint('asset-file-list'), args=[self.asset.uid]) # TODO: change the fixture so every asset's owner has all expected # permissions? For now, call `save()` to recalculate permissions and # verify the result self.asset.save() self.assertListEqual( sorted(list(self.asset.get_perms(self.asset.owner))), sorted(list(Asset.get_assignable_permissions(False) + Asset.CALCULATED_PERMISSIONS)) )
def nonstandard_asset(): return Asset(asset_type='survey', content={ 'survey': [ {'type': 'select_one abc', 'Label': 'select a letter'}, # todo: handle certain "expand" features after aliases are replaced # {'type': 'select1 abc', 'Label': 'select a letter'}, {}, {'misc_value': 'gets removed by _strip_empty_rows'}, ], 'choices': [ {'list name': 'abc', 'label': letter} for letter in string.ascii_lowercase ] })
def test_import_library_bulk_xls(self): library_sheet_content = [ ['block', 'name', 'type', 'label', 'tag:subject:fungus', 'tag:subject:useless'], ['mushroom', 'cap', 'text', 'Describe the cap', '1', None], ['mushroom', 'gills', 'text', 'Describe the gills', '1', None], ['mushroom', 'spores', 'text', 'Describe the spores', '1', None], [None, 'non_block', 'acknowledge', 'I am not inside a block!', None, '1'], ['mushroom', 'size', 'text', 'Describe the size', '1', None], ['mushroom', 'season', 'select_multiple seasons', 'Found during which seasons?', None, None], [None, 'also_non_block', 'integer', 'I, too, refuse to join a block!', None, '1'], ] choices_sheet_content = [ ['list name', 'name', 'label'], ['seasons', 'spring', 'Spring'], ['seasons', 'summer', 'Summer'], ['seasons', 'fall', 'Fall'], ['seasons', 'winter', 'Winter'], ] content = ( ('library', library_sheet_content), ('choices', choices_sheet_content), ) task_data = self._construct_xls_for_import( content, name='Collection created from bulk library import' ) post_url = reverse('api_v2:importtask-list') response = self.client.post(post_url, task_data) assert response.status_code == status.HTTP_201_CREATED detail_response = self.client.get(response.data['url']) assert detail_response.status_code == status.HTTP_200_OK assert detail_response.data['status'] == 'complete' # Transform the XLS sheets and rows into asset content for comparison # with the results of the import # Discarding the first (`block`) column and any tag columns, combine # the first row (headers) with subsequent 'mushroom' rows headers = [ col for col in library_sheet_content[0][1:] if not col.startswith('tag:') ] mushroom_block_survey = [ # We don't have to remove the tag columns from each row, because # `zip()` stops once it reaches the end of `headers` dict(zip(headers, row[1:])) for row in library_sheet_content[1:] if row[0] == 'mushroom' ] # Transform the choices for the 'mushroom' block into a list of dicts mushroom_block_choices = [ dict(zip(choices_sheet_content[0], row)) for row in choices_sheet_content[1:] ] # Create the in-memory only asset, but adjust its contents as if we # were saving it to the database mushroom_block_asset = Asset( content={ 'survey': mushroom_block_survey, 'choices': mushroom_block_choices, } ) mushroom_block_asset.adjust_content_on_save() # Similarly create the in-memory assets for the simpler, non-block # questions non_block_assets = [] for row in library_sheet_content: if row[0] is not None: continue question_asset = Asset( content={'survey': [dict(zip(headers, row[1:]))]} ) question_asset.adjust_content_on_save() non_block_assets.append(question_asset) # Find the new collection created by the import created_details = detail_response.data['messages']['created'][0] assert created_details['kind'] == 'collection' created_collection = Asset.objects.get(uid=created_details['uid']) assert created_collection.name == task_data['name'] created_children = created_collection.children.order_by('date_created') assert len(created_children) == 3 # Verify the imported block created_block = created_children[0] assert created_block.asset_type == ASSET_TYPE_BLOCK self._assert_assets_contents_equal(created_block, mushroom_block_asset) self._assert_assets_contents_equal( created_block, mushroom_block_asset, sheet='choices' ) # Verify the imported non-block questions created_questions = created_children[1:] for q in created_questions: assert q.asset_type == ASSET_TYPE_QUESTION self._assert_assets_contents_equal( created_questions[0], non_block_assets[0] ) self._assert_assets_contents_equal( created_questions[1], non_block_assets[1] ) # Check that tags were assigned correctly tagged_as_fungus = Asset.objects.filter_by_tag_name( 'subject:fungus' ).filter(parent=created_collection).order_by('date_created') assert tagged_as_fungus.count() == 1 self._assert_assets_contents_equal( tagged_as_fungus[0], mushroom_block_asset ) tagged_as_useless = Asset.objects.filter_by_tag_name( 'subject:useless' ).filter(parent=created_collection).order_by('date_created') assert tagged_as_useless.count() == 2 self._assert_assets_contents_equal( tagged_as_useless[0], non_block_assets[0] ) self._assert_assets_contents_equal( tagged_as_useless[1], non_block_assets[1] )
def _name_to_autoname(rows): s = Asset(asset_type='survey', content={}) rows = [dict({'type': 'text'}, **row) for row in rows] content = {'survey': rows} s._autoname(content) return [r['$autoname'] for r in content.get('survey', [])]
def _compile_asset_content(content): a1 = Asset(asset_type='survey', content={}) a1._standardize(content) a1._strip_empty_rows(content) a1._assign_kuids(content) form_title = a1.pop_setting(content, 'form_title', 'a backup title') a1._autoname(content) assert form_title == 'a backup title' # at this stage, the save is complete a1._expand_kobo_qs(content) a1._autoname(content) a1._assign_kuids(content) return content
def handle(self, *args, **options): if not settings.KOBOCAT_URL or not settings.KOBOCAT_INTERNAL_URL: raise ImproperlyConfigured( 'Both KOBOCAT_URL and KOBOCAT_INTERNAL_URL must be ' 'configured before using this command' ) self._quiet = options.get('quiet') username = options.get('username') populate_xform_kpi_asset_uid = options.get('populate_xform_kpi_asset_uid') users = User.objects.all() # Do a basic query just to make sure the ReadOnlyKobocatXForm model is # loaded if not ReadOnlyKobocatXForm.objects.exists(): return self._print_str('%d total users' % users.count()) # A specific user or everyone? if username: users = User.objects.filter(username=username) self._print_str('%d users selected' % users.count()) # We'll be copying the date fields from KC, so don't auto-update them _set_auto_field_update(Asset, "date_created", False) _set_auto_field_update(Asset, "date_modified", False) for user in users: # Make sure the user has a token for access to KC's API Token.objects.get_or_create(user=user) existing_surveys = user.assets.filter(asset_type='survey') # Each asset that the user has already deployed to KC should have a # form uuid stored in its deployment data xform_uuids_to_asset_pks = {} for existing_survey in existing_surveys: dd = existing_survey._deployment_data try: backend_response = dd['backend_response'] except KeyError: continue xform_uuids_to_asset_pks[backend_response['uuid']] = \ existing_survey.pk xforms = user.xforms.all() for xform in xforms: try: with transaction.atomic(): if xform.uuid not in xform_uuids_to_asset_pks: # This is an orphaned KC form. Build a new asset to # match asset = Asset(asset_type='survey', owner=user) asset.name = _make_name_for_asset(asset, xform) else: asset = Asset.objects.get( pk=xform_uuids_to_asset_pks[xform.uuid]) changes = [] try: content_changed = _sync_form_content( asset, xform, changes) metadata_changed = _sync_form_metadata( asset, xform, changes) except SyncKCXFormsWarning as e: error_information = [ 'WARN', user.username, xform.id_string, e.message ] self._print_tabular(*error_information) continue if content_changed or metadata_changed: # preserve the original "asset.content" asset.save(adjust_content=False) # save a new version with standardized content asset.save() if content_changed: asset._mark_latest_version_as_deployed() self._print_tabular( ','.join(changes), user.username, xform.id_string, asset.uid ) else: self._print_tabular( 'NOOP', user.username, xform.id_string, asset.uid ) except Exception as e: error_information = [ 'FAIL', user.username, xform.id_string, repr(e) ] self._print_tabular(*error_information) logging.exception('sync_kobocat_xforms: {}'.format( ', '.join(error_information))) _set_auto_field_update(Asset, "date_created", True) _set_auto_field_update(Asset, "date_modified", True) if populate_xform_kpi_asset_uid: call_command('populate_kc_xform_kpi_asset_uid', username=username)
def _sync_permissions(asset, xform): """ Returns a list of affected users' usernames """ if not settings.SYNC_KOBOCAT_PERMISSIONS: return [] # Get all applicable KC permissions set for this xform xform_user_perms = KobocatUserObjectPermission.objects.filter( permission_id__in=PERMISSIONS_MAP.keys(), content_type=XFORM_CT, object_pk=xform.pk ).values_list('user', 'permission') if not xform_user_perms and not asset.pk: # Nothing to do return [] if not asset.pk: # Asset must have a primary key before working with its permissions asset.save() # Translate KC permissions to KPI permissions and store as dictionary of # { user: set(perm1, perm2, ...) } translated_kc_perms = defaultdict(set) for user, kc_permission in xform_user_perms: translated_kc_perms[user].add(PERMISSIONS_MAP[kc_permission]) # Note that certain KPI permissions should be granted if corresponding # flags on the KC `XForm` are set for kpi_codename, xform_flags in ( Asset.KC_ANONYMOUS_PERMISSIONS_XFORM_FLAGS.items() ): all_flags_set = True for flag, value_when_set in xform_flags.items(): if getattr(xform, flag) != value_when_set: all_flags_set = False break if not all_flags_set: continue translated_kc_perms[ANONYMOUS_USER.pk].add( KPI_CODENAMES_TO_PKS[kpi_codename] ) # Get existing KPI permissions in same dictionary format current_kpi_perms = defaultdict(set) for user, kpi_permission in ObjectPermission.objects.filter( deny=False, content_type=ASSET_CT, object_id=asset.pk ).values_list('user', 'permission'): current_kpi_perms[user].add(kpi_permission) # Look for users in KPI but not in KC. Their permissions may have come from # KC but were later revoked for user in set(current_kpi_perms.keys()).difference(translated_kc_perms): translated_kc_perms[user] = set() affected_usernames = [] for user, expected_perms in translated_kc_perms.items(): if user == xform.user_id: # No need sync the owner's permissions continue # KC does not assign implied permissions, so we have to do the work of # resolving them implied_perms = set() for p in expected_perms: implied_perms.update( Asset.get_implied_perms(KPI_PKS_TO_CODENAMES[p]) ) # Only consider relevant implied permissions implied_perms.intersection_update(KPI_PKS_TO_CODENAMES.values()) # Convert from permission codenames back to PKs expected_perms.update( [KPI_CODENAMES_TO_PKS[codename] for codename in implied_perms] ) user_obj = User.objects.get(pk=user) all_kpi_perms = current_kpi_perms[user] mapped_kpi_perms = current_kpi_perms[user].intersection( PERMISSIONS_MAP.values()) perms_to_assign = expected_perms.difference(mapped_kpi_perms) perms_to_revoke = mapped_kpi_perms.difference(expected_perms) all_revoked = perms_to_revoke and not bool( mapped_kpi_perms.difference(perms_to_revoke)) if not all_kpi_perms and perms_to_assign: # The user has no existing KPI permissions; assign a special flag # permission noting that their only reason for access is this # synchronization script ObjectPermission.objects.get_or_create( user_id=user, permission=FROM_KC_ONLY_PERMISSION, content_type=ASSET_CT, object_id=asset.pk ) for p in perms_to_assign: asset.assign_perm(user_obj, KPI_PKS_TO_CODENAMES[p], skip_kc=True) for p in perms_to_revoke: asset.remove_perm(user_obj, KPI_PKS_TO_CODENAMES[p], skip_kc=True) if all_revoked and FROM_KC_ONLY_PERMISSION.pk in all_kpi_perms: # This user's KPI access came only from this script, and now all KC # permissions have been removed. Purge all KPI grant permissions, # even the non-mapped ones, in order to clean up prerequisite # permissions (e.g. 'view_asset' is a prerequisite of # 'view_submissions') ObjectPermission.objects.filter( user_id=user, deny=False, content_type=ASSET_CT, object_id=asset.pk ).delete() if perms_to_assign or perms_to_revoke: affected_usernames.append(user_obj.username) return affected_usernames
def _new(): return Asset(asset_type='survey', content=fn()) return _new