def test_get_case_types(self):
self._send_case_to_es(case_type='t1')
self._send_case_to_es(case_type='t2')
self._send_case_to_es(case_type='t3', closed_on=datetime.utcnow())
self._send_case_to_es(domain='other', case_type='t4')
case_types = get_case_types_for_domain_es(self.domain)
self.assertEqual(case_types, {'t1', 't2', 't3'})
self.assertEqual({'t4'}, get_case_types_for_domain_es('other'))
self.assertEqual(set(), get_case_types_for_domain_es('none'))
def test_get_case_types(self):
self._send_case_to_es(case_type='t1')
self._send_case_to_es(case_type='t2')
self._send_case_to_es(case_type='t3', closed_on=datetime.utcnow())
self._send_case_to_es(domain='other', case_type='t4')
case_types = get_case_types_for_domain_es(self.domain)
self.assertEqual(case_types, {'t1', 't2', 't3'})
self.assertEqual({'t4'}, get_case_types_for_domain_es('other'))
self.assertEqual(set(), get_case_types_for_domain_es('none'))
def test_get_case_types_caching(self):
self._send_case_to_es(case_type='t1')
self.assertEqual({'t1'}, get_case_types_for_domain_es(self.domain))
self._send_case_to_es(case_type='t2')
# cached response
self.assertEqual({'t1'}, get_case_types_for_domain_es(self.domain))
# simulate a save
from casexml.apps.case.signals import case_post_save
case_post_save.send(self, case=CommCareCase(domain=self.domain, type='t2'))
self.assertEqual({'t1', 't2'}, get_case_types_for_domain_es(self.domain))
def test_get_case_types_caching(self):
self._send_case_to_es(case_type='t1')
self.assertEqual({'t1'}, get_case_types_for_domain_es(self.domain))
self._send_case_to_es(case_type='t2')
# cached response
self.assertEqual({'t1'}, get_case_types_for_domain_es(self.domain))
# simulate a save
from casexml.apps.case.signals import case_post_save
case_post_save.send(self, case=CommCareCase(domain=self.domain, type='t2'))
self.assertEqual({'t1', 't2'}, get_case_types_for_domain_es(self.domain))
def case_types(self):
"""
:return: Set of all case types for the domain excluding the CallCenter case type.
"""
case_types = get_case_types_for_domain_es(self.domain)
case_types = case_types - {self.cc_case_type}
return case_types
def case_types(self):
"""
:return: Set of all case types for the domain excluding the CallCenter case type.
"""
case_types = get_case_types_for_domain_es(self.domain)
case_types = case_types - {self.cc_case_type}
return case_types
def set_case_type_choices(self, initial):
case_types = [''] + list(get_case_types_for_domain_es(self.domain))
if initial and initial not in case_types:
# Include the deleted case type in the list of choices so that
# we always allow proper display and edit of rules
case_types.append(initial)
case_types.sort()
self.fields['case_type'].choices = ((case_type, case_type)
for case_type in case_types)
def set_case_type_choices(self, initial):
case_types = [''] + list(get_case_types_for_domain_es(self.domain))
if initial and initial not in case_types:
# Include the deleted case type in the list of choices so that
# we always allow proper display and edit of rules
case_types.append(initial)
case_types.sort()
self.fields['case_type'].choices = (
(case_type, case_type) for case_type in case_types
)
def get_case_type_to_properties(domain):
case_type_to_properties = {}
case_types = get_case_types_for_domain_es(domain)
for case_type in case_types:
case_type_to_properties[case_type] = []
case_export_schema = CaseExportDataSchema.generate_schema_from_builds(domain, None, case_type)
for export_group_schema in case_export_schema.group_schemas[0].items:
cleaned_case_property = export_group_schema.label.replace('_', '')
case_type_to_properties[case_type].append(cleaned_case_property)
return case_type_to_properties
def obj_get_list(self, bundle, **kwargs):
domain = kwargs['domain']
couch_user = CouchUser.from_django_user(bundle.request.user)
if not domain_has_privilege(domain, privileges.ZAPIER_INTEGRATION) or not couch_user.is_member_of(domain):
raise ImmediateHttpResponse(
HttpForbidden('You are not allowed to get list of case types for this domain')
)
case_types = get_case_types_for_domain_es(domain)
results = [CaseType(case_type=case_type) for case_type in case_types]
return results
def obj_get_list(self, bundle, **kwargs):
domain = kwargs['domain']
couch_user = CouchUser.from_django_user(bundle.request.user)
if not domain_has_privilege(domain, privileges.ZAPIER_INTEGRATION) or not couch_user.is_member_of(domain):
raise ImmediateHttpResponse(
HttpForbidden('You are not allowed to get list of case types for this domain')
)
case_types = get_case_types_for_domain_es(domain)
results = [CaseType(case_type=case_type) for case_type in case_types]
return results
def get(self, request, domain):
data = {
'@odata.context':
absolute_reverse('odata_meta', args=[domain]),
'value': [{
'name': case_type,
'kind': 'EntitySet',
'url': case_type,
} for case_type in get_case_types_for_domain_es(domain)]
}
return add_odata_headers(JsonResponse(data))
def _get_case_size_stats(domain, sample_size, use_case_search):
total_bytes = 0
total_cases = 0
index_class = CaseSearchES if use_case_search else CaseES
for case_type in get_case_types_for_domain_es(domain, use_case_search):
query = (index_class().domain(domain).case_type(case_type).size(
sample_size))
num_cases, avg_size = _get_totals_for_query(query)
total_bytes += num_cases * avg_size
total_cases += num_cases
return total_cases, total_bytes
def get(self, request, domain):
data = {
'@odata.context': absolute_reverse('odata_meta', args=[domain]),
'value': [
{
'name': case_type,
'kind': 'EntitySet',
'url': case_type,
}
for case_type in get_case_types_for_domain_es(domain)
]
}
return add_odata_headers(JsonResponse(data))
def get_case_type_to_properties(domain):
case_type_to_properties = defaultdict(list)
case_types = get_case_types_for_domain_es(domain)
for case_type in case_types:
if not case_type:
# TODO - understand why a case can have a blank case type and handle appropriately
continue
case_export_schema = (
get_latest_case_export_schema(domain, case_type)
or CaseExportDataSchema.generate_schema_from_builds(domain, None, case_type)
)
for export_group_schema in case_export_schema.group_schemas[0].items:
cleaned_case_property = export_group_schema.label.replace('_', '')
case_type_to_properties[case_type].append(cleaned_case_property)
return dict(case_type_to_properties)
def get_case_type_to_properties(domain):
case_type_to_properties = defaultdict(list)
case_types = get_case_types_for_domain_es(domain)
for case_type in case_types:
if not case_type:
# TODO - understand why a case can have a blank case type and handle appropriately
continue
case_export_schema = (get_latest_case_export_schema(domain, case_type)
or
CaseExportDataSchema.generate_schema_from_builds(
domain, None, case_type))
for export_group_schema in case_export_schema.group_schemas[0].items:
cleaned_case_property = export_group_schema.label.replace('_', '')
case_type_to_properties[case_type].append(cleaned_case_property)
return dict(case_type_to_properties)
def _get_cases_for_apps(self, apps_by_type, as_dict=True):
used_case_types = set()
case_types_by_app = collections.defaultdict(list)
for app_type, apps in apps_by_type.items():
for app_choice in apps:
if not app_choice.id == self.UNKNOWN_SOURCE:
app = get_app(self.domain, app_choice.id)
case_types = []
if hasattr(app, 'modules'):
# Add regular case types
case_types = set([
module.case_type
for module in app.modules if module.case_type
])
# Add user case if any module uses it
if any([module.uses_usercase() for module in app.modules]):
case_types.add(USERCASE_TYPE)
used_case_types = used_case_types.union(case_types)
case_types = [RMIDataChoice(
id=c,
text=c,
data=app_choice.data
) for c in case_types]
if as_dict:
case_types = [c._asdict() for c in case_types]
case_types_by_app[app_choice.id] = case_types
all_case_types = get_case_types_for_domain_es(self.domain)
unknown_case_types = all_case_types.difference(used_case_types)
unknown_case_types = [RMIDataChoice(
id=c,
text=c,
data={
'unknown': True,
}
) for c in unknown_case_types]
if as_dict:
unknown_case_types = [c._asdict() for c in unknown_case_types]
case_types_by_app[self.UNKNOWN_SOURCE] = unknown_case_types
return case_types_by_app
def _get_cases_for_apps(self, apps_by_type, as_dict=True):
used_case_types = set()
case_types_by_app = collections.defaultdict(list)
for app_type, apps in apps_by_type.items():
for app_choice in apps:
if not app_choice.id == self.UNKNOWN_SOURCE:
app = get_app(self.domain, app_choice.id)
case_types = []
if hasattr(app, 'modules'):
# Add regular case types
case_types = set([
module.case_type
for module in app.modules if module.case_type
])
# Add user case if any module uses it
if any([module.uses_usercase() for module in app.modules]):
case_types.add(USERCASE_TYPE)
used_case_types = used_case_types.union(case_types)
case_types = [RMIDataChoice(
id=c,
text=c,
data=app_choice.data
) for c in case_types]
if as_dict:
case_types = [c._asdict() for c in case_types]
case_types_by_app[app_choice.id] = case_types
all_case_types = get_case_types_for_domain_es(self.domain)
unknown_case_types = all_case_types.difference(used_case_types)
unknown_case_types = [RMIDataChoice(
id=c,
text=c,
data={
'unknown': True,
}
) for c in unknown_case_types]
if as_dict:
unknown_case_types = [c._asdict() for c in unknown_case_types]
case_types_by_app[self.UNKNOWN_SOURCE] = unknown_case_types
return case_types_by_app
def options(self):
case_types = sorted(get_case_types_for_domain_es(self.domain))
return [(case, "%s" % case) for case in case_types
if case != USER_LOCATION_OWNER_MAP_TYPE]
def case_type_choices(self):
return [(t, t) for t in get_case_types_for_domain_es(self.domain)]
def get(self, request, domain):
return JsonResponse({
'existing_case_types':
list(get_case_types_for_domain_es(domain))
})
def _process_file_and_get_upload(uploaded_file_handle,
request,
domain,
max_columns=None):
extension = os.path.splitext(
uploaded_file_handle.name)[1][1:].strip().lower()
# NOTE: We may not always be able to reference files from subsequent
# views if your worker changes, so we have to store it elsewhere
# using the soil framework.
if extension not in valid_extensions:
raise SpreadsheetFileExtError(
'The file you chose could not be processed. '
'Please check that it is saved as a Microsoft '
'Excel file.')
# stash content in the default storage for subsequent views
case_upload = CaseUpload.create(uploaded_file_handle,
filename=uploaded_file_handle.name,
domain=domain)
request.session[EXCEL_SESSION_ID] = case_upload.upload_id
case_upload.check_file()
invalid_column_names = set()
with case_upload.get_spreadsheet() as spreadsheet:
columns = spreadsheet.get_header_columns()
validate_column_names(columns, invalid_column_names)
row_count = spreadsheet.max_row
if invalid_column_names:
error_message = format_html(
_("Column names must be <a target='_blank' href='https://www.w3schools.com/xml/xml_elements.asp'>"
"valid XML elements</a> and cannot start with a number or contain spaces or most special characters."
" Please update the following: {}.").format(
', '.join(invalid_column_names)))
raise ImporterRawError(error_message)
if row_count == 0:
raise ImporterError(
'Your spreadsheet is empty. Please try again with a different spreadsheet.'
)
if max_columns is not None and len(columns) > max_columns:
raise ImporterError('Your spreadsheet has too many columns. '
'A maximum of %(max_columns)s is supported.' %
{'max_columns': MAX_CASE_IMPORTER_COLUMNS})
case_types_from_apps = sorted(get_case_types_from_apps(domain))
unrecognized_case_types = sorted([
t for t in get_case_types_for_domain_es(domain)
if t not in case_types_from_apps
])
if len(case_types_from_apps) == 0 and len(unrecognized_case_types) == 0:
raise ImporterError(
'No cases have been submitted to this domain and there are no '
'applications yet. You cannot import case details from an Excel '
'file until you have existing cases or applications.')
context = {
'columns': columns,
'unrecognized_case_types': unrecognized_case_types,
'case_types_from_apps': case_types_from_apps,
'domain': domain,
'slug': base.ImportCases.slug,
}
return case_upload, context
def options(self):
case_types = get_case_types_for_domain_es(self.domain)
return [(case, "%s" % case) for case in case_types
if case != USER_LOCATION_OWNER_MAP_TYPE]
def case_type_choices(self):
return [(t, t) for t in get_case_types_for_domain_es(self.domain)]
def available_case_types(self):
return sorted(get_case_types_for_domain_es(self.domain))
def test_get_case_types_case_sensitive(self):
self._send_case_to_es(case_type='child')
self._send_case_to_es(case_type='Child')
case_types = get_case_types_for_domain_es(self.domain)
self.assertEqual(case_types, {'child', 'Child'})
def test_get_case_types_case_sensitive(self):
self._send_case_to_es(case_type='child')
self._send_case_to_es(case_type='Child')
case_types = get_case_types_for_domain_es(self.domain)
self.assertEqual(case_types, {'child', 'Child'})
def excel_config(request, domain):
"""
Step one of three.
This is the initial post when the user uploads the Excel file
"""
if request.method != 'POST':
return HttpResponseRedirect(base.ImportCases.get_url(domain=domain))
if not request.FILES:
return render_error(request, domain, 'Please choose an Excel file to import.')
uploaded_file_handle = request.FILES['file']
extension = os.path.splitext(uploaded_file_handle.name)[1][1:].strip().lower()
# NOTE: We may not always be able to reference files from subsequent
# views if your worker changes, so we have to store it elsewhere
# using the soil framework.
if extension not in importer_util.ALLOWED_EXTENSIONS:
return render_error(request, domain, _(
'The file you chose could not be processed. '
'Please check that it is saved as a Microsoft '
'Excel file.'
))
# stash content in the default storage for subsequent views
case_upload = CaseUpload.create(uploaded_file_handle,
filename=uploaded_file_handle.name,
domain=domain)
request.session[EXCEL_SESSION_ID] = case_upload.upload_id
try:
case_upload.check_file()
except ImporterError as e:
return render_error(request, domain, get_importer_error_message(e))
except SpreadsheetFileExtError:
return render_error(request, domain, _("Please upload file with extension .xls or .xlsx"))
invalid_column_names = set()
with case_upload.get_spreadsheet() as spreadsheet:
columns = spreadsheet.get_header_columns()
validate_column_names(columns, invalid_column_names)
row_count = spreadsheet.max_row
if invalid_column_names:
error_message = format_html(
_("Column names must be <a target='_blank' href='https://www.w3schools.com/xml/xml_elements.asp'>"
"valid XML elements</a> and cannot start with a number or contain spaces or most special characters."
" Please update the following: {}.").format(
', '.join(invalid_column_names)))
return render_error(request, domain, error_message)
if row_count == 0:
return render_error(request, domain, _(
'Your spreadsheet is empty. Please try again with a different spreadsheet.'
))
if len(columns) > MAX_CASE_IMPORTER_COLUMNS:
return render_error(request, domain, _(
'Your spreadsheet has too many columns. '
'A maximum of %(max_columns)s is supported.'
) % {'max_columns': MAX_CASE_IMPORTER_COLUMNS})
case_types_from_apps = sorted(get_case_types_from_apps(domain))
unrecognized_case_types = sorted([t for t in get_case_types_for_domain_es(domain)
if t not in case_types_from_apps])
if len(case_types_from_apps) == 0 and len(unrecognized_case_types) == 0:
return render_error(request, domain, _(
'No cases have been submitted to this domain and there are no '
'applications yet. You cannot import case details from an Excel '
'file until you have existing cases or applications.'
))
context = {
'columns': columns,
'unrecognized_case_types': unrecognized_case_types,
'case_types_from_apps': case_types_from_apps,
'domain': domain,
'slug': base.ImportCases.slug,
}
context.update(_case_importer_breadcrumb_context(_('Case Options'), domain))
return render(request, "case_importer/excel_config.html", context)
def get(self, request, domain):
return JsonResponse({
'existing_case_types': list(get_case_types_for_domain_es(domain))
})
def excel_config(request, domain):
"""
Step one of three.
This is the initial post when the user uploads the Excel file
"""
if request.method != 'POST':
return HttpResponseRedirect(base.ImportCases.get_url(domain=domain))
if not request.FILES:
return render_error(request, domain,
'Please choose an Excel file to import.')
uploaded_file_handle = request.FILES['file']
extension = os.path.splitext(
uploaded_file_handle.name)[1][1:].strip().lower()
# NOTE: We may not always be able to reference files from subsequent
# views if your worker changes, so we have to store it elsewhere
# using the soil framework.
if extension not in importer_util.ALLOWED_EXTENSIONS:
return render_error(
request, domain,
_('The file you chose could not be processed. '
'Please check that it is saved as a Microsoft '
'Excel file.'))
# stash content in the default storage for subsequent views
case_upload = CaseUpload.create(uploaded_file_handle,
filename=uploaded_file_handle.name,
domain=domain)
request.session[EXCEL_SESSION_ID] = case_upload.upload_id
try:
case_upload.check_file()
except ImporterError as e:
return render_error(request, domain, get_importer_error_message(e))
except SpreadsheetFileExtError:
return render_error(
request, domain,
_("Please upload file with extension .xls or .xlsx"))
invalid_column_names = set()
with case_upload.get_spreadsheet() as spreadsheet:
columns = spreadsheet.get_header_columns()
validate_column_names(columns, invalid_column_names)
row_count = spreadsheet.max_row
if invalid_column_names:
error_message = format_html(
_("Column names must be <a target='_blank' href='https://www.w3schools.com/xml/xml_elements.asp'>"
"valid XML elements</a> and cannot start with a number or contain spaces or most special characters."
" Please update the following: {}.").format(
', '.join(invalid_column_names)))
return render_error(request, domain, error_message)
if row_count == 0:
return render_error(
request, domain,
_('Your spreadsheet is empty. Please try again with a different spreadsheet.'
))
if len(columns) > MAX_CASE_IMPORTER_COLUMNS:
return render_error(
request, domain,
_('Your spreadsheet has too many columns. '
'A maximum of %(max_columns)s is supported.') %
{'max_columns': MAX_CASE_IMPORTER_COLUMNS})
case_types_from_apps = sorted(get_case_types_from_apps(domain))
unrecognized_case_types = sorted([
t for t in get_case_types_for_domain_es(domain)
if t not in case_types_from_apps
])
if len(case_types_from_apps) == 0 and len(unrecognized_case_types) == 0:
return render_error(
request, domain,
_('No cases have been submitted to this domain and there are no '
'applications yet. You cannot import case details from an Excel '
'file until you have existing cases or applications.'))
context = {
'columns': columns,
'unrecognized_case_types': unrecognized_case_types,
'case_types_from_apps': case_types_from_apps,
'domain': domain,
'slug': base.ImportCases.slug,
}
context.update(_case_importer_breadcrumb_context(_('Case Options'),
domain))
request.use_select2_v4 = True
return render(request, "case_importer/excel_config.html", context)
def obj_get_list(self, bundle, **kwargs):
domain = kwargs['domain']
case_types = get_case_types_for_domain_es(domain)
results = [CaseType(case_type=case_type) for case_type in case_types]
return results
def excel_config(request, domain):
"""
Step one of three.
This is the initial post when the user uploads the excel file
"""
if request.method != 'POST':
return HttpResponseRedirect(base.ImportCases.get_url(domain=domain))
if not request.FILES:
return render_error(request, domain,
'Please choose an Excel file to import.')
uploaded_file_handle = request.FILES['file']
extension = os.path.splitext(
uploaded_file_handle.name)[1][1:].strip().lower()
# NOTE: We may not always be able to reference files from subsequent
# views if your worker changes, so we have to store it elsewhere
# using the soil framework.
if extension not in importer_util.ALLOWED_EXTENSIONS:
return render_error(
request, domain,
_('The file you chose could not be processed. '
'Please check that it is saved as a Microsoft '
'Excel file.'))
# stash content in the default storage for subsequent views
case_upload = CaseUpload.create(uploaded_file_handle,
filename=uploaded_file_handle.name)
request.session[EXCEL_SESSION_ID] = case_upload.upload_id
try:
case_upload.check_file()
except ImporterError as e:
return render_error(request, domain, get_importer_error_message(e))
except SpreadsheetFileExtError:
return render_error(
request, domain,
_("Please upload file with extension .xls or .xlsx"))
with case_upload.get_spreadsheet() as spreadsheet:
columns = spreadsheet.get_header_columns()
row_count = spreadsheet.max_row
if row_count == 0:
return render_error(
request, domain,
_('Your spreadsheet is empty. Please try again with a different spreadsheet.'
))
if len(columns) > MAX_CASE_IMPORTER_COLUMNS:
return render_error(
request, domain,
_('Your spreadsheet has too many columns. '
'A maximum of %(max_columns)s is supported.') %
{'max_columns': MAX_CASE_IMPORTER_COLUMNS})
case_types_from_apps = get_case_types_from_apps(domain)
unrecognized_case_types = [
t for t in get_case_types_for_domain_es(domain)
if t not in case_types_from_apps
]
if len(case_types_from_apps) == 0 and len(unrecognized_case_types) == 0:
return render_error(
request, domain,
_('No cases have been submitted to this domain and there are no '
'applications yet. You cannot import case details from an Excel '
'file until you have existing cases or applications.'))
return render(
request, "case_importer/excel_config.html", {
'columns': columns,
'unrecognized_case_types': unrecognized_case_types,
'case_types_from_apps': case_types_from_apps,
'domain': domain,
'report': {
'name': 'Import: Configuration'
},
'slug': base.ImportCases.slug
})
