def test_kuid_persists():
initial_kuid_1 = 'aaaa1111'
initial_kuid_2 = 'bbbb2222'
asset = Asset(
content={
'survey': [
{
'type': 'text',
'name': 'abc',
'$kuid': initial_kuid_1
},
{
'type': 'text',
'name': 'def',
'$kuid': initial_kuid_2
},
],
})
# kobo_specific_types=True avoids calling _strip_kuids
# so, can we assume that kuids are supposed to remain?
content = asset.ordered_xlsform_content(kobo_specific_types=True)
# kuids are stripped in "kobo_to_xlsform.to_xlsform_structure(...)"
assert '$kuid' in content['survey'][0]
assert content['survey'][0].get('$kuid') == initial_kuid_1
assert '$kuid' in content['survey'][1]
assert content['survey'][1].get('$kuid') == initial_kuid_2
def _import_asset(asset, parent_collection=None, asset_type='survey'):
survey_dict = _csv_to_dict(asset.body)
obj = {
'name': asset.name,
'date_created': asset.date_created,
'date_modified': asset.date_modified,
'asset_type': asset_type,
'owner': user,
}
if parent_collection is not None:
obj['parent'] = parent_collection
del obj['name']
new_asset = Asset(**obj)
_set_auto_field_update(Asset, "date_created", False)
_set_auto_field_update(Asset, "date_modified", False)
new_asset.content = survey_dict
new_asset.date_created = obj['date_created']
new_asset.date_modified = obj['date_modified']
new_asset.save()
_set_auto_field_update(Asset, "date_created", True)
_set_auto_field_update(Asset, "date_modified", True)
# Note on the old draft the uid of the new asset
asset.kpi_asset_uid = new_asset.uid
asset.save()
return new_asset
def _create_cloned_asset(self):
asset = Asset()
asset.owner = self.asset.owner
asset.content = self.asset.content
asset.save()
asset.deploy(backend='mock', active=True)
asset.save()
return asset
def test_remove_empty_expressions():
a1 = Asset(asset_type='survey', content={})
c1 = {'survey': [{'relevant': ''}]}
a1._remove_empty_expressions(c1)
assert _r1(c1) == {}
c1 = {'survey': [{'bind': None}]}
a1._remove_empty_expressions(c1)
assert _r1(c1) == {}
def test_expand_twice():
a1 = Asset(asset_type='survey', content={'survey': [{'type': 'note',
'label::English': 'english',
'hint::English': 'hint',
}]})
a1.adjust_content_on_save()
assert 'translations' in a1.content
assert len(a1.content['translations']) > 0
assert 'translated' in a1.content
assert len(a1.content['translated']) > 0
assert sorted(a1.content['translated']) == ['hint', 'label']
def setUp(self):
self.client.login(username="******", password="******")
self.someuser = User.objects.get(username="******")
self.asset = a = Asset()
a.name = 'Two points and one text'
a.owner = self.someuser
a.asset_type = 'survey'
a.content = {
'survey': [
{
'name': 'geo1',
'type': 'geopoint',
'label': 'Where were you?'
},
{
'name': 'geo2',
'type': 'geopoint',
'label': 'Where are you?'
},
{
'name': 'text',
'type': 'text',
'label': 'How are you?'
},
]
}
a.save()
a.deploy(backend='mock', active=True)
a.save()
v_uid = a.latest_deployed_version.uid
self.submissions = [
{
'__version__': v_uid,
'geo1': '10.11 10.12 10.13 10.14',
'geo2': '10.21 10.22 10.23 10.24',
'text': 'Tired',
},
{
'__version__': v_uid,
'geo1': '20.11 20.12 20.13 20.14',
'geo2': '20.21 20.22 20.23 20.24',
'text': 'Relieved',
},
{
'__version__': v_uid,
'geo1': '30.11 30.12 30.13 30.14',
'geo2': '30.21 30.22 30.23 30.24',
'text': 'Excited',
},
]
a.deployment.mock_submissions(self.submissions)
a.deployment.set_namespace(self.URL_NAMESPACE)
self.submission_list_url = a.deployment.submission_list_url
def test_save_transformations():
a1 = Asset(asset_type='survey', content={})
content = color_picker_asset_content()
a1._standardize(content)
a1._strip_empty_rows(content)
a1._assign_kuids(content)
form_title = a1.pop_setting(content, 'form_title')
a1._autoname(content)
assert 'schema' in content
assert content['translations'] == [None]
assert form_title == 'color picker'
assert content['settings'] == {'id_string': 'colorpik'}
def _compile_asset_content(content):
a1 = Asset(asset_type='survey', content={})
a1._standardize(content)
a1._strip_empty_rows(content)
a1._assign_kuids(content)
form_title = a1.pop_setting(content, 'form_title', 'a backup title')
a1._autoname(content)
assert form_title == 'a backup title'
# at this stage, the save is complete
a1._expand_kobo_qs(content)
a1._autoname(content)
a1._assign_kuids(content)
return content
def nonstandard_asset():
return Asset(asset_type='survey', content={
'survey': [
{'type': 'select_one abc', 'Label': 'select a letter'},
# todo: handle certain "expand" features after aliases are replaced
# {'type': 'select1 abc', 'Label': 'select a letter'},
{},
{'misc_value': 'gets removed by _strip_empty_rows'},
],
'choices': [
{'list name': 'abc', 'label': letter}
for letter in string.ascii_lowercase
]
})
def test_autoname_handles_non_latin_labels_with_kobo_score_and_kobo_rank():
a = Asset(asset_type='survey', content={})
content = {
'survey': [
{
'type': 'score__row',
'label': ['नमस्ते']
},
{
'type': 'rank__level',
'label': ['नमस्ते']
},
]
}
a._standardize(content)
a._strip_empty_rows(content)
a._assign_kuids(content)
a._autoname(content)
for row in content['survey']:
assert row['$autoname'].startswith('select_one')
def test_export_uid_filter(self):
assert self.user.username == 'someuser'
def _create_export_task(asset):
export_task = ExportTask()
export_task.user = self.user
export_task.data = {
'source': reverse('asset-detail', args=[asset.uid]),
'type': 'csv'
}
messages = defaultdict(list)
export_task._run_task(messages)
return export_task
matching_export = _create_export_task(self.asset)
# Create a clone and generate an export from it
excluded_asset = Asset()
excluded_asset.owner = self.asset.owner
excluded_asset.content = self.asset.content
excluded_asset.save()
excluded_asset.deploy(backend='mock', active=True)
excluded_asset.save()
excluded_export = _create_export_task(excluded_asset)
# Retrieve all the exports unfiltered
self.client.login(username='******', password='******')
list_url = reverse(self._get_endpoint('exporttask-list'))
response = self.client.get(list_url)
assert response.status_code == status.HTTP_200_OK
assert response.json()['count'] == 2
# Retrieve the exports filtered by a single asset uid
filter_url = f'{list_url}?q=source:{self.asset.uid}'
response = self.client.get(filter_url)
assert response.status_code == status.HTTP_200_OK
response_dict = response.json()
assert response_dict['count'] == 1
assert self.asset.uid in response_dict['results'][0]['data']['source']
def test_import_library_bulk_xls(self):
library_sheet_content = [
['block', 'name', 'type', 'label', 'tag:subject:fungus', 'tag:subject:useless'],
['mushroom', 'cap', 'text', 'Describe the cap', '1', None],
['mushroom', 'gills', 'text', 'Describe the gills', '1', None],
['mushroom', 'spores', 'text', 'Describe the spores', '1', None],
[None, 'non_block', 'acknowledge', 'I am not inside a block!', None, '1'],
['mushroom', 'size', 'text', 'Describe the size', '1', None],
['mushroom', 'season', 'select_multiple seasons', 'Found during which seasons?', None, None],
[None, 'also_non_block', 'integer', 'I, too, refuse to join a block!', None, '1'],
]
choices_sheet_content = [
['list name', 'name', 'label'],
['seasons', 'spring', 'Spring'],
['seasons', 'summer', 'Summer'],
['seasons', 'fall', 'Fall'],
['seasons', 'winter', 'Winter'],
]
content = (
('library', library_sheet_content),
('choices', choices_sheet_content),
)
task_data = self._construct_xls_for_import(
content, name='Collection created from bulk library import'
)
post_url = reverse('api_v2:importtask-list')
response = self.client.post(post_url, task_data)
assert response.status_code == status.HTTP_201_CREATED
detail_response = self.client.get(response.data['url'])
assert detail_response.status_code == status.HTTP_200_OK
assert detail_response.data['status'] == 'complete'
# Transform the XLS sheets and rows into asset content for comparison
# with the results of the import
# Discarding the first (`block`) column and any tag columns, combine
# the first row (headers) with subsequent 'mushroom' rows
headers = [
col
for col in library_sheet_content[0][1:]
if not col.startswith('tag:')
]
mushroom_block_survey = [
# We don't have to remove the tag columns from each row, because
# `zip()` stops once it reaches the end of `headers`
dict(zip(headers, row[1:])) for row in library_sheet_content[1:]
if row[0] == 'mushroom'
]
# Transform the choices for the 'mushroom' block into a list of dicts
mushroom_block_choices = [
dict(zip(choices_sheet_content[0], row))
for row in choices_sheet_content[1:]
]
# Create the in-memory only asset, but adjust its contents as if we
# were saving it to the database
mushroom_block_asset = Asset(
content={
'survey': mushroom_block_survey,
'choices': mushroom_block_choices,
}
)
mushroom_block_asset.adjust_content_on_save()
# Similarly create the in-memory assets for the simpler, non-block
# questions
non_block_assets = []
for row in library_sheet_content:
if row[0] is not None:
continue
question_asset = Asset(
content={'survey': [dict(zip(headers, row[1:]))]}
)
question_asset.adjust_content_on_save()
non_block_assets.append(question_asset)
# Find the new collection created by the import
created_details = detail_response.data['messages']['created'][0]
assert created_details['kind'] == 'collection'
created_collection = Asset.objects.get(uid=created_details['uid'])
assert created_collection.name == task_data['name']
created_children = created_collection.children.order_by('date_created')
assert len(created_children) == 3
# Verify the imported block
created_block = created_children[0]
assert created_block.asset_type == ASSET_TYPE_BLOCK
self._assert_assets_contents_equal(created_block, mushroom_block_asset)
self._assert_assets_contents_equal(
created_block, mushroom_block_asset, sheet='choices'
)
# Verify the imported non-block questions
created_questions = created_children[1:]
for q in created_questions:
assert q.asset_type == ASSET_TYPE_QUESTION
self._assert_assets_contents_equal(
created_questions[0], non_block_assets[0]
)
self._assert_assets_contents_equal(
created_questions[1], non_block_assets[1]
)
# Check that tags were assigned correctly
tagged_as_fungus = Asset.objects.filter_by_tag_name(
'subject:fungus'
).filter(parent=created_collection).order_by('date_created')
assert tagged_as_fungus.count() == 1
self._assert_assets_contents_equal(
tagged_as_fungus[0], mushroom_block_asset
)
tagged_as_useless = Asset.objects.filter_by_tag_name(
'subject:useless'
).filter(parent=created_collection).order_by('date_created')
assert tagged_as_useless.count() == 2
self._assert_assets_contents_equal(
tagged_as_useless[0], non_block_assets[0]
)
self._assert_assets_contents_equal(
tagged_as_useless[1], non_block_assets[1]
)
def handle(self, *args, **options):
if not settings.KOBOCAT_URL or not settings.KOBOCAT_INTERNAL_URL:
raise ImproperlyConfigured(
'Both KOBOCAT_URL and KOBOCAT_INTERNAL_URL must be '
'configured before using this command'
)
self._quiet = options.get('quiet')
username = options.get('username')
populate_xform_kpi_asset_uid = options.get('populate_xform_kpi_asset_uid')
users = User.objects.all()
# Do a basic query just to make sure the ReadOnlyKobocatXForm model is
# loaded
if not ReadOnlyKobocatXForm.objects.exists():
return
self._print_str('%d total users' % users.count())
# A specific user or everyone?
if username:
users = User.objects.filter(username=username)
self._print_str('%d users selected' % users.count())
# We'll be copying the date fields from KC, so don't auto-update them
_set_auto_field_update(Asset, "date_created", False)
_set_auto_field_update(Asset, "date_modified", False)
for user in users:
# Make sure the user has a token for access to KC's API
Token.objects.get_or_create(user=user)
existing_surveys = user.assets.filter(asset_type='survey')
# Each asset that the user has already deployed to KC should have a
# form uuid stored in its deployment data
xform_uuids_to_asset_pks = {}
for existing_survey in existing_surveys:
dd = existing_survey._deployment_data
try:
backend_response = dd['backend_response']
except KeyError:
continue
xform_uuids_to_asset_pks[backend_response['uuid']] = \
existing_survey.pk
xforms = user.xforms.all()
for xform in xforms:
try:
with transaction.atomic():
if xform.uuid not in xform_uuids_to_asset_pks:
# This is an orphaned KC form. Build a new asset to
# match
asset = Asset(asset_type='survey', owner=user)
asset.name = _make_name_for_asset(asset, xform)
else:
asset = Asset.objects.get(
pk=xform_uuids_to_asset_pks[xform.uuid])
changes = []
try:
content_changed = _sync_form_content(
asset, xform, changes)
metadata_changed = _sync_form_metadata(
asset, xform, changes)
except SyncKCXFormsWarning as e:
error_information = [
'WARN',
user.username,
xform.id_string,
e.message
]
self._print_tabular(*error_information)
continue
if content_changed or metadata_changed:
# preserve the original "asset.content"
asset.save(adjust_content=False)
# save a new version with standardized content
asset.save()
if content_changed:
asset._mark_latest_version_as_deployed()
self._print_tabular(
','.join(changes),
user.username,
xform.id_string,
asset.uid
)
else:
self._print_tabular(
'NOOP',
user.username,
xform.id_string,
asset.uid
)
except Exception as e:
error_information = [
'FAIL',
user.username,
xform.id_string,
repr(e)
]
self._print_tabular(*error_information)
logging.exception('sync_kobocat_xforms: {}'.format(
', '.join(error_information)))
_set_auto_field_update(Asset, "date_created", True)
_set_auto_field_update(Asset, "date_modified", True)
if populate_xform_kpi_asset_uid:
call_command('populate_kc_xform_kpi_asset_uid', username=username)
def _new(): return Asset(asset_type='survey', content=fn()) return _new
def _name_to_autoname(rows):
s = Asset(asset_type='survey', content={})
rows = [dict({'type': 'text'}, **row) for row in rows]
content = {'survey': rows}
s._autoname(content)
return [r['$autoname'] for r in content.get('survey', [])]
