def bulk_import_async(import_id, config, domain, excel_id):
excel_ref = DownloadBase.get(excel_id)
spreadsheet = importer_util.get_spreadsheet(excel_ref, config.named_columns)
result = do_import(spreadsheet, config, domain, task=bulk_import_async)
# return compatible with soil
return {"messages": result}
def excel_commit(request, domain):
"""
Step three of three.
This page is submitted with the list of column to
case property mappings for this upload.
The config variable is an ImporterConfig object that
has everything gathered from previous steps, with the
addition of all the field data. See that class for
more information.
"""
config = importer_util.ImporterConfig.from_request(request)
excel_id = request.session.get(EXCEL_SESSION_ID)
excel_ref = DownloadBase.get(excel_id)
try:
importer_util.get_spreadsheet(excel_ref, config.named_columns)
except ImporterError as e:
return render_error(request, domain, _get_importer_error_message(e))
download = DownloadBase()
download.set_task(bulk_import_async.delay(
download.download_id,
config,
domain,
excel_id,
))
try:
del request.session[EXCEL_SESSION_ID]
except KeyError:
pass
return render(
request,
"importer/excel_commit.html", {
'download_id': download.download_id,
'template': 'importer/partials/import_status.html',
'domain': domain,
'report': {
'name': 'Import: Completed'
},
'slug': base.ImportCases.slug
}
)
def excel_commit(request, domain):
"""
Step three of three.
This page is submitted with the list of column to
case property mappings for this upload.
The config variable is an ImporterConfig object that
has everything gathered from previous steps, with the
addition of all the field data. See that class for
more information.
"""
config = importer_util.ImporterConfig.from_request(request)
excel_id = request.session.get(EXCEL_SESSION_ID)
excel_ref = DownloadBase.get(excel_id)
try:
importer_util.get_spreadsheet(excel_ref, config.named_columns)
except ImporterError as e:
return render_error(request, domain, get_importer_error_message(e))
download = DownloadBase()
download.set_task(bulk_import_async.delay(
config,
domain,
excel_id,
))
try:
del request.session[EXCEL_SESSION_ID]
except KeyError:
pass
return render(
request,
"importer/excel_commit.html", {
'download_id': download.download_id,
'template': 'importer/partials/import_status.html',
'domain': domain,
'report': {
'name': 'Import: Completed'
},
'slug': base.ImportCases.slug
}
)
def bulk_import_async(import_id, config, domain, excel_id):
excel_ref = DownloadBase.get(excel_id)
try:
spreadsheet_or_error = importer_util.get_spreadsheet(excel_ref, config.named_columns)
except ImporterError as spreadsheet_or_error:
pass
result = do_import(spreadsheet_or_error, config, domain, task=bulk_import_async)
# return compatible with soil
return {"messages": result}
def excel_commit(request, domain):
"""
Step three of three.
This page is submitted with the list of column to
case property mappings for this upload.
The config variable is an ImporterConfig object that
has everything gathered from previous steps, with the
addition of all the field data. See that class for
more information.
"""
config = importer_util.ImporterConfig.from_request(request)
excel_id = request.session.get(EXCEL_SESSION_ID)
excel_ref = DownloadBase.get(excel_id)
spreadsheet = importer_util.get_spreadsheet(excel_ref, config.named_columns)
if not spreadsheet:
return _spreadsheet_expired(request, domain)
if spreadsheet.has_errors:
messages.error(request, _('The session containing the file you '
'uploaded has expired - please upload '
'a new one.'))
return HttpResponseRedirect(base.ImportCases.get_url(domain=domain) + "?error=cache")
download = DownloadBase()
download.set_task(bulk_import_async.delay(
download.download_id,
config,
domain,
excel_id,
))
try:
del request.session[EXCEL_SESSION_ID]
except KeyError:
pass
return render(
request,
"importer/excel_commit.html", {
'download_id': download.download_id,
'template': 'importer/partials/import_status.html',
'domain': domain,
'report': {
'name': 'Import: Completed'
},
'slug': base.ImportCases.slug
}
)
def excel_commit(request, domain):
"""
Step three of three.
This page is submitted with the list of column to
case property mappings for this upload.
The config variable is an ImporterConfig object that
has everything gathered from previous steps, with the
addition of all the field data. See that class for
more information.
"""
config = importer_util.ImporterConfig.from_request(request)
excel_id = request.session.get(EXCEL_SESSION_ID)
excel_ref = DownloadBase.get(excel_id)
spreadsheet = importer_util.get_spreadsheet(excel_ref, config.named_columns)
if not spreadsheet:
return _spreadsheet_expired(request, domain)
if spreadsheet.has_errors:
messages.error(request, _('The session containing the file you '
'uploaded has expired - please upload '
'a new one.'))
return HttpResponseRedirect(base.ImportCases.get_url(domain=domain) + "?error=cache")
download = DownloadBase()
download.set_task(bulk_import_async.delay(
download.download_id,
config,
domain,
excel_id,
))
try:
del request.session[EXCEL_SESSION_ID]
except KeyError:
pass
return render(
request,
"importer/excel_commit.html", {
'download_id': download.download_id,
'template': 'importer/partials/import_status.html',
'domain': domain,
'report': {
'name': 'Import: Completed'
},
'slug': base.ImportCases.slug
}
)
def bulk_import_async(config, domain, excel_id):
excel_ref = DownloadBase.get(excel_id)
try:
spreadsheet = importer_util.get_spreadsheet(excel_ref,
config.named_columns)
except Exception as e:
return {'errors': get_importer_error_message(e)}
try:
result = do_import(spreadsheet, config, domain, task=bulk_import_async)
except Exception as e:
return {'errors': 'Error: ' + e.message}
# return compatible with soil
return {'messages': result}
def excel_commit(request, domain):
config = importer_util.ImporterConfig.from_request(request)
excel_id = request.session.get(EXCEL_SESSION_ID)
excel_ref = DownloadBase.get(excel_id)
spreadsheet = importer_util.get_spreadsheet(excel_ref,
config.named_columns)
if not spreadsheet:
return _spreadsheet_expired(request, domain)
if spreadsheet.has_errors:
messages.error(
request,
_('The session containing the file you '
'uploaded has expired - please upload '
'a new one.'))
return HttpResponseRedirect(
base.ImportCases.get_url(domain=domain) + "?error=cache")
download = DownloadBase()
download.set_task(
bulk_import_async.delay(
download.download_id,
config,
domain,
excel_id,
))
try:
del request.session[EXCEL_SESSION_ID]
except KeyError:
pass
return render(
request, "importer/excel_commit.html", {
'download_id': download.download_id,
'template': 'importer/partials/import_status.html',
'domain': domain,
'report': {
'name': 'Import: Completed'
},
'slug': base.ImportCases.slug
})
def excel_commit(request, domain):
config = ImporterConfig(request)
excel_id = request.session.get(EXCEL_SESSION_ID)
excel_ref = DownloadBase.get(excel_id)
spreadsheet = importer_util.get_spreadsheet(excel_ref, config.named_columns)
if not spreadsheet:
return _spreadsheet_expired(request, domain)
if spreadsheet.has_errors:
messages.error(request, _('The session containing the file you '
'uploaded has expired - please upload '
'a new one.'))
return HttpResponseRedirect(base.ImportCases.get_url(domain=domain) + "?error=cache")
download = DownloadBase()
download.set_task(bulk_import_async.delay(
download.download_id,
config,
domain,
excel_id,
))
try:
del request.session[EXCEL_SESSION_ID]
except KeyError:
pass
return render(request, "importer/excel_commit.html", {
'download_id': download.download_id,
'template': 'importer/partials/import_status.html',
'domain': domain,
'report': {
'name': 'Import: Completed'
},
'slug': base.ImportCases.slug})
def bulk_import_async(import_id, config, domain, excel_id):
excel_ref = DownloadBase.get(excel_id)
spreadsheet = importer_util.get_spreadsheet(excel_ref, config.named_columns)
return do_import(spreadsheet, config, domain, task=bulk_import_async)
def excel_config(request, domain):
"""
Step one of three.
This is the initial post when the user uploads the excel file
named_columns:
Whether or not the first row of the excel sheet contains
header strings for the columns. This defaults to True and
should potentially not be an option as it is always used
due to how important it is to see column headers
in the rest of the importer.
"""
if request.method != 'POST':
return HttpResponseRedirect(base.ImportCases.get_url(domain=domain))
if not request.FILES:
return render_error(request, domain, 'Please choose an Excel file to import.')
named_columns = request.POST.get('named_columns') == "on"
uploaded_file_handle = request.FILES['file']
extension = os.path.splitext(uploaded_file_handle.name)[1][1:].strip().lower()
# NOTE: We may not always be able to reference files from subsequent
# views if your worker changes, so we have to store it elsewhere
# using the soil framework.
if extension not in importer_util.ExcelFile.ALLOWED_EXTENSIONS:
return render_error(request, domain,
'The Excel file you chose could not be processed. '
'Please check that it is saved as a Microsoft '
'Excel 97/2000 .xls file.')
# stash content in the default storage for subsequent views
file_ref = expose_cached_download(
uploaded_file_handle.read(),
expiry=1*60*60,
file_extension=file_extention_from_filename(uploaded_file_handle.name),
)
request.session[EXCEL_SESSION_ID] = file_ref.download_id
spreadsheet = importer_util.get_spreadsheet(file_ref, named_columns)
if not spreadsheet:
return _spreadsheet_expired(request, domain)
columns = spreadsheet.get_header_columns()
row_count = spreadsheet.get_num_rows()
if row_count == 0:
return render_error(request, domain,
'Your spreadsheet is empty. '
'Please try again with a different spreadsheet.')
case_types_from_apps = []
# load types from all modules
for row in ApplicationBase.view(
'app_manager/types_by_module',
reduce=True,
group=True,
startkey=[domain],
endkey=[domain, {}]
).all():
if not row['key'][1] in case_types_from_apps:
case_types_from_apps.append(row['key'][1])
case_types_from_cases = get_case_types_for_domain(domain)
# for this we just want cases that have data but aren't being used anymore
case_types_from_cases = filter(lambda x: x not in case_types_from_apps, case_types_from_cases)
if len(case_types_from_apps) == 0 and len(case_types_from_cases) == 0:
return render_error(
request,
domain,
'No cases have been submitted to this domain and there are no '
'applications yet. You cannot import case details from an Excel '
'file until you have existing cases or applications.'
)
return render(
request,
"importer/excel_config.html", {
'named_columns': named_columns,
'columns': columns,
'case_types_from_cases': case_types_from_cases,
'case_types_from_apps': case_types_from_apps,
'domain': domain,
'report': {
'name': 'Import: Configuration'
},
'slug': base.ImportCases.slug
}
)
def excel_fields(request, domain):
"""
Step two of three.
Important values that are grabbed from the POST or defined by
the user on this page:
named_columns:
Passed through from last step, see that for documentation
case_type:
The type of case we are matching to. When creating new cases,
this is the type they will be created as. When updating
existing cases, this is the type that we will search for.
If the wrong case type is used when looking up existing cases,
we will not update them.
create_new_cases:
A boolean that controls whether or not the user wanted
to create new cases for any case that doesn't have a matching
case id in the upload.
search_column:
Which column of the excel file we are using to specify either
case ids or external ids. This is, strangely, required. If
creating new cases only you would expect these to be blank with
the create_new_cases flag set.
search_field:
Either case id or external id, determines which type of
identification we are using to match to cases.
key_column/value_column:
These correspond to an advanced feature allowing a user
to modify a single case with multiple rows.
"""
named_columns = request.POST['named_columns']
case_type = request.POST['case_type']
try:
search_column = request.POST['search_column']
except MultiValueDictKeyError:
# this is only true if your configuration is messed up in an irreparable way
messages.error(request, _('It looks like you may have accessed this page from a stale page. '
'Please start over.'))
return _spreadsheet_expired(request, domain)
search_field = request.POST['search_field']
create_new_cases = request.POST.get('create_new_cases') == 'on'
key_value_columns = request.POST.get('key_value_columns') == 'on'
key_column = ''
value_column = ''
download_ref = DownloadBase.get(request.session.get(EXCEL_SESSION_ID))
spreadsheet = importer_util.get_spreadsheet(download_ref, named_columns)
if not spreadsheet:
return _spreadsheet_expired(request, domain)
columns = spreadsheet.get_header_columns()
if key_value_columns:
key_column = request.POST['key_column']
value_column = request.POST['value_column']
excel_fields = []
key_column_index = columns.index(key_column)
# if key/value columns were specified, get all the unique keys listed
if key_column_index:
excel_fields = spreadsheet.get_unique_column_values(key_column_index)
# concatenate unique key fields with the rest of the columns
excel_fields = columns + excel_fields
# remove key/value column names from list
excel_fields.remove(key_column)
if value_column in excel_fields:
excel_fields.remove(value_column)
else:
excel_fields = columns
case_fields = get_case_properties(domain, case_type)
# hide search column and matching case fields from the update list
try:
excel_fields.remove(search_column)
except:
pass
try:
case_fields.remove(search_field)
except:
pass
# we can't actually update this so don't show it
try:
case_fields.remove('type')
except:
pass
return render(
request,
"importer/excel_fields.html", {
'named_columns': named_columns,
'case_type': case_type,
'search_column': search_column,
'search_field': search_field,
'create_new_cases': create_new_cases,
'key_column': key_column,
'value_column': value_column,
'columns': columns,
'excel_fields': excel_fields,
'case_fields': case_fields,
'domain': domain,
'report': {
'name': 'Import: Match columns to fields'
},
'slug': base.ImportCases.slug
}
)
def bulk_import_async(import_id, config, domain, excel_id):
task = bulk_import_async
excel_ref = DownloadBase.get(excel_id)
spreadsheet = importer_util.get_spreadsheet(excel_ref, config.named_columns)
if not spreadsheet:
return {'error': 'EXPIRED'}
if spreadsheet.has_errors:
return {'error': 'HAS_ERRORS'}
row_count = spreadsheet.get_num_rows()
columns = spreadsheet.get_header_columns()
match_count = created_count = too_many_matches = errors = 0
blank_external_ids = []
invalid_dates = []
owner_id_errors = []
prime_offset = 1 # used to prevent back-to-back priming
user = CouchUser.get_by_user_id(config.couch_user_id, domain)
username = user.username
user_id = user._id
# keep a cache of id lookup successes to help performance
id_cache = {}
for i in range(row_count):
DownloadBase.set_progress(task, i, row_count)
# skip first row if it is a header field
if i == 0 and config.named_columns:
continue
priming_progress = match_count + created_count + prime_offset
if priming_progress % PRIME_VIEW_FREQUENCY == 0:
prime_views(POOL_SIZE)
# increment so we can't possibly prime on next iteration
prime_offset += 1
row = spreadsheet.get_row(i)
search_id = importer_util.parse_search_id(config, columns, row)
if config.search_field == 'external_id' and not search_id:
# do not allow blank external id since we save this
blank_external_ids.append(i + 1)
continue
case, error = importer_util.lookup_case(
config.search_field,
search_id,
domain,
config.case_type
)
try:
fields_to_update = importer_util.populate_updated_fields(
config,
columns,
row
)
except importer_util.InvalidDateException:
invalid_dates.append(i + 1)
continue
if case:
pass
elif error == LookupErrors.NotFound:
if not config.create_new_cases:
continue
elif error == LookupErrors.MultipleResults:
too_many_matches += 1
continue
uploaded_owner_id = fields_to_update.pop('owner_id', None)
if uploaded_owner_id:
# If an owner_id mapping exists, verify it is a valid user
# or case sharing group
if importer_util.is_valid_id(uploaded_owner_id, domain, id_cache):
owner_id = uploaded_owner_id
id_cache[uploaded_owner_id] = True
else:
owner_id_errors.append(i + 1)
id_cache[uploaded_owner_id] = False
continue
else:
# if they didn't supply an owner_id mapping, default to current
# user
owner_id = user_id
external_id = fields_to_update.pop('external_id', None)
if not case:
id = uuid.uuid4().hex
try:
caseblock = CaseBlock(
create=True,
case_id=id,
version=V2,
user_id=user_id,
owner_id=owner_id,
case_type=config.case_type,
update=fields_to_update
)
if config.search_field == 'external_id':
caseblock['external_id'] = search_id
submit_case_block(caseblock, domain, username, user_id)
created_count += 1
except CaseBlockError:
errors += 1
elif case and case.type == config.case_type:
extras = {}
if external_id:
extras['external_id'] = external_id
try:
caseblock = CaseBlock(
create=False,
case_id=case._id,
owner_id=owner_id,
version=V2,
update=fields_to_update,
**extras
)
submit_case_block(caseblock, domain, username, user_id)
match_count += 1
except CaseBlockError:
errors += 1
return {
'created_count': created_count,
'match_count': match_count,
'too_many_matches': too_many_matches,
'blank_externals': blank_external_ids,
'invalid_dates': invalid_dates,
'owner_id_errors': owner_id_errors,
'errors': errors,
}
def excel_fields(request, domain):
named_columns = request.POST['named_columns']
case_type = request.POST['case_type']
search_column = request.POST['search_column']
search_field = request.POST['search_field']
create_new_cases = request.POST.get('create_new_cases') == 'on'
key_value_columns = request.POST.get('key_value_columns') == 'on'
key_column = ''
value_column = ''
download_ref = DownloadBase.get(request.session.get(EXCEL_SESSION_ID))
spreadsheet = importer_util.get_spreadsheet(download_ref, named_columns)
if not spreadsheet:
return _spreadsheet_expired(request, domain)
columns = spreadsheet.get_header_columns()
if key_value_columns:
key_column = request.POST['key_column']
value_column = request.POST['value_column']
excel_fields = []
key_column_index = columns.index(key_column)
# if key/value columns were specified, get all the unique keys listed
if key_column_index:
excel_fields = spreadsheet.get_unique_column_values(
key_column_index)
# concatenate unique key fields with the rest of the columns
excel_fields = columns + excel_fields
# remove key/value column names from list
excel_fields.remove(key_column)
if value_column in excel_fields:
excel_fields.remove(value_column)
else:
excel_fields = columns
case_fields = importer_util.get_case_properties(domain, case_type)
# hide search column and matching case fields from the update list
try:
excel_fields.remove(search_column)
except:
pass
try:
case_fields.remove(search_field)
except:
pass
# we can't actually update this so don't show it
try:
case_fields.remove('type')
except:
pass
return render(
request, "importer/excel_fields.html", {
'named_columns': named_columns,
'case_type': case_type,
'search_column': search_column,
'search_field': search_field,
'create_new_cases': create_new_cases,
'key_column': key_column,
'value_column': value_column,
'columns': columns,
'excel_fields': excel_fields,
'case_fields': case_fields,
'domain': domain,
'report': {
'name': 'Import: Match columns to fields'
},
'slug': base.ImportCases.slug
})
def excel_config(request, domain):
if request.method != 'POST':
return HttpResponseRedirect(base.ImportCases.get_url(domain=domain))
if not request.FILES:
return render_error(request, domain,
'Please choose an Excel file to import.')
named_columns = request.POST.get('named_columns') == "on"
uploaded_file_handle = request.FILES['file']
extension = os.path.splitext(
uploaded_file_handle.name)[1][1:].strip().lower()
# NOTE: We may not always be able to reference files from subsequent
# views if your worker changes, so we have to store it elsewhere
# using the soil framework.
if extension not in importer_util.ExcelFile.ALLOWED_EXTENSIONS:
return render_error(
request, domain,
'The Excel file you chose could not be processed. '
'Please check that it is saved as a Microsoft '
'Excel 97/2000 .xls file.')
# stash content in the default storage for subsequent views
file_ref = expose_download(uploaded_file_handle.read(), expiry=1 * 60 * 60)
request.session[EXCEL_SESSION_ID] = file_ref.download_id
spreadsheet = importer_util.get_spreadsheet(file_ref, named_columns)
if not spreadsheet:
return _spreadsheet_expired(request, domain)
columns = spreadsheet.get_header_columns()
row_count = spreadsheet.get_num_rows()
if row_count == 0:
return render_error(
request, domain, 'Your spreadsheet is empty. '
'Please try again with a different spreadsheet.')
case_types_from_apps = []
# load types from all modules
for row in ApplicationBase.view('app_manager/types_by_module',
reduce=True,
group=True,
startkey=[domain],
endkey=[domain, {}]).all():
if not row['key'][1] in case_types_from_apps:
case_types_from_apps.append(row['key'][1])
case_types_from_cases = []
# load types from all case records
for row in CommCareCase.view('hqcase/types_by_domain',
reduce=True,
group=True,
startkey=[domain],
endkey=[domain, {}]).all():
if row['key'][1] and not row['key'][1] in case_types_from_cases:
case_types_from_cases.append(row['key'][1])
# for this we just want cases that have data but aren't being used anymore
case_types_from_cases = filter(lambda x: x not in case_types_from_apps,
case_types_from_cases)
if len(case_types_from_apps) == 0 and len(case_types_from_cases) == 0:
return render_error(
request, domain,
'No cases have been submitted to this domain and there are no '
'applications yet. You cannot import case details from an Excel '
'file until you have existing cases or applications.')
return render(
request, "importer/excel_config.html", {
'named_columns': named_columns,
'columns': columns,
'case_types_from_cases': case_types_from_cases,
'case_types_from_apps': case_types_from_apps,
'domain': domain,
'report': {
'name': 'Import: Configuration'
},
'slug': base.ImportCases.slug
})
def excel_fields(request, domain):
"""
Step two of three.
Important values that are grabbed from the POST or defined by
the user on this page:
named_columns:
Passed through from last step, see that for documentation
case_type:
The type of case we are matching to. When creating new cases,
this is the type they will be created as. When updating
existing cases, this is the type that we will search for.
If the wrong case type is used when looking up existing cases,
we will not update them.
create_new_cases:
A boolean that controls whether or not the user wanted
to create new cases for any case that doesn't have a matching
case id in the upload.
search_column:
Which column of the excel file we are using to specify either
case ids or external ids. This is, strangely, required. If
creating new cases only you would expect these to be blank with
the create_new_cases flag set.
search_field:
Either case id or external id, determines which type of
identification we are using to match to cases.
key_column/value_column:
These correspond to an advanced feature allowing a user
to modify a single case with multiple rows.
"""
named_columns = request.POST['named_columns']
case_type = request.POST['case_type']
try:
search_column = request.POST['search_column']
except MultiValueDictKeyError:
# this is only true if your configuration is messed up in an irreparable way
messages.error(request, _('It looks like you may have accessed this page from a stale page. '
'Please start over.'))
return _spreadsheet_expired(request, domain)
search_field = request.POST['search_field']
create_new_cases = request.POST.get('create_new_cases') == 'on'
key_value_columns = request.POST.get('key_value_columns') == 'on'
key_column = ''
value_column = ''
download_ref = DownloadBase.get(request.session.get(EXCEL_SESSION_ID))
try:
spreadsheet = importer_util.get_spreadsheet(download_ref, named_columns)
except ImporterError as e:
return render_error(request, domain, get_importer_error_message(e))
columns = spreadsheet.get_header_columns()
if key_value_columns:
key_column = request.POST['key_column']
value_column = request.POST['value_column']
excel_fields = []
key_column_index = columns.index(key_column)
# if key/value columns were specified, get all the unique keys listed
if key_column_index:
excel_fields = spreadsheet.get_unique_column_values(key_column_index)
# concatenate unique key fields with the rest of the columns
excel_fields = columns + excel_fields
# remove key/value column names from list
excel_fields.remove(key_column)
if value_column in excel_fields:
excel_fields.remove(value_column)
else:
excel_fields = columns
case_fields = get_case_properties_for_case_type(domain, case_type)
# hide search column and matching case fields from the update list
try:
excel_fields.remove(search_column)
except:
pass
try:
case_fields.remove(search_field)
except:
pass
# we can't actually update this so don't show it
try:
case_fields.remove('type')
except:
pass
return render(
request,
"importer/excel_fields.html", {
'named_columns': named_columns,
'case_type': case_type,
'search_column': search_column,
'search_field': search_field,
'create_new_cases': create_new_cases,
'key_column': key_column,
'value_column': value_column,
'columns': columns,
'excel_fields': excel_fields,
'case_fields': case_fields,
'domain': domain,
'report': {
'name': 'Import: Match columns to fields'
},
'slug': base.ImportCases.slug
}
)
def bulk_import_async(import_id, config, domain, excel_id):
excel_ref = DownloadBase.get(excel_id)
spreadsheet = importer_util.get_spreadsheet(excel_ref,
config.named_columns)
return do_import(spreadsheet, config, domain, task=bulk_import_async)
def excel_fields(request, domain):
named_columns = request.POST['named_columns']
case_type = request.POST['case_type']
search_column = request.POST['search_column']
search_field = request.POST['search_field']
create_new_cases = request.POST.get('create_new_cases') == 'on'
key_value_columns = request.POST.get('key_value_columns') == 'on'
key_column = ''
value_column = ''
download_ref = DownloadBase.get(request.session.get(EXCEL_SESSION_ID))
spreadsheet = importer_util.get_spreadsheet(download_ref, named_columns)
if not spreadsheet:
return _spreadsheet_expired(request, domain)
columns = spreadsheet.get_header_columns()
if key_value_columns:
key_column = request.POST['key_column']
value_column = request.POST['value_column']
excel_fields = []
key_column_index = columns.index(key_column)
# if key/value columns were specified, get all the unique keys listed
if key_column_index:
excel_fields = spreadsheet.get_unique_column_values(key_column_index)
# concatenate unique key fields with the rest of the columns
excel_fields = columns + excel_fields
# remove key/value column names from list
excel_fields.remove(key_column)
if value_column in excel_fields:
excel_fields.remove(value_column)
else:
excel_fields = columns
case_fields = importer_util.get_case_properties(domain, case_type)
# hide search column and matching case fields from the update list
try:
excel_fields.remove(search_column)
except:
pass
try:
case_fields.remove(search_field)
except:
pass
# we can't actually update this so don't show it
try:
case_fields.remove('type')
except:
pass
return render(request, "importer/excel_fields.html", {
'named_columns': named_columns,
'case_type': case_type,
'search_column': search_column,
'search_field': search_field,
'create_new_cases': create_new_cases,
'key_column': key_column,
'value_column': value_column,
'columns': columns,
'excel_fields': excel_fields,
'case_fields': case_fields,
'domain': domain,
'report': {
'name': 'Import: Match columns to fields'
},
'slug': base.ImportCases.slug})
def bulk_import_async(import_id, config, domain, excel_id):
task = bulk_import_async
excel_ref = DownloadBase.get(excel_id)
spreadsheet = importer_util.get_spreadsheet(excel_ref, config.named_columns)
if not spreadsheet:
return {'error': 'EXPIRED'}
if spreadsheet.has_errors:
return {'error': 'HAS_ERRORS'}
row_count = spreadsheet.get_num_rows()
columns = spreadsheet.get_header_columns()
match_count = created_count = too_many_matches = 0
blank_external_ids = []
invalid_dates = []
prime_offset = 1 # used to prevent back-to-back priming
user = CouchUser.get_by_user_id(config.couch_user_id, domain)
username = user.username
user_id = user._id
for i in range(row_count):
DownloadBase.set_progress(task, i, row_count)
# skip first row if it is a header field
if i == 0 and config.named_columns:
continue
priming_progress = match_count + created_count + prime_offset
if priming_progress % PRIME_VIEW_FREQUENCY == 0:
prime_views(POOL_SIZE)
# increment so we can't possibly prime on next iteration
prime_offset += 1
row = spreadsheet.get_row(i)
search_id = importer_util.parse_search_id(config, columns, row)
if config.search_field == 'external_id' and not search_id:
# do not allow blank external id since we save this
blank_external_ids.append(i + 1)
continue
case, error = importer_util.lookup_case(config.search_field,
search_id, domain)
try:
fields_to_update = importer_util.populate_updated_fields(
config,
columns,
row
)
except importer_util.InvalidDateException:
invalid_dates.append(i + 1)
continue
if case:
match_count += 1
elif error == LookupErrors.NotFound:
if not config.create_new_cases:
continue
created_count += 1
elif error == LookupErrors.MultipleResults:
too_many_matches += 1
continue
if 'owner_id' in fields_to_update:
owner_id = fields_to_update['owner_id']
del fields_to_update['owner_id']
else:
owner_id = user_id
if not case:
id = uuid.uuid4().hex
caseblock = CaseBlock(
create=True,
case_id=id,
version=V2,
user_id=user_id,
owner_id=owner_id,
case_type=config.case_type,
external_id=search_id if config.search_field == 'external_id' else '',
update=fields_to_update
)
submit_case_block(caseblock, domain, username, user_id)
elif case and case.type == config.case_type:
caseblock = CaseBlock(
create=False,
case_id=case._id,
owner_id=owner_id,
version=V2,
update=fields_to_update
)
submit_case_block(caseblock, domain, username, user_id)
return {
'created_count': created_count,
'match_count': match_count,
'too_many_matches': too_many_matches,
'blank_externals': blank_external_ids,
'invalid_dates': invalid_dates,
}
def excel_config(request, domain):
"""
Step one of three.
This is the initial post when the user uploads the excel file
named_columns:
Whether or not the first row of the excel sheet contains
header strings for the columns. This defaults to True and
should potentially not be an option as it is always used
due to how important it is to see column headers
in the rest of the importer.
"""
if request.method != 'POST':
return HttpResponseRedirect(base.ImportCases.get_url(domain=domain))
if not request.FILES:
return render_error(request, domain, 'Please choose an Excel file to import.')
named_columns = request.POST.get('named_columns') == "on"
uploaded_file_handle = request.FILES['file']
extension = os.path.splitext(uploaded_file_handle.name)[1][1:].strip().lower()
# NOTE: We may not always be able to reference files from subsequent
# views if your worker changes, so we have to store it elsewhere
# using the soil framework.
if extension not in importer_util.ExcelFile.ALLOWED_EXTENSIONS:
return render_error(request, domain,
'The Excel file you chose could not be processed. '
'Please check that it is saved as a Microsoft '
'Excel 97/2000 .xls file.')
# stash content in the default storage for subsequent views
file_ref = expose_cached_download(
uploaded_file_handle.read(),
expiry=1*60*60,
file_extension=file_extention_from_filename(uploaded_file_handle.name),
)
request.session[EXCEL_SESSION_ID] = file_ref.download_id
try:
spreadsheet = importer_util.get_spreadsheet(file_ref, named_columns)
except ImporterError as e:
return render_error(request, domain, get_importer_error_message(e))
columns = spreadsheet.get_header_columns()
row_count = spreadsheet.get_num_rows()
if row_count == 0:
return render_error(request, domain,
'Your spreadsheet is empty. '
'Please try again with a different spreadsheet.')
case_types_from_apps = get_case_types_from_apps(domain)
unrecognized_case_types = [t for t in CaseAccessors(domain).get_case_types()
if t not in case_types_from_apps]
if len(case_types_from_apps) == 0 and len(unrecognized_case_types) == 0:
return render_error(
request,
domain,
'No cases have been submitted to this domain and there are no '
'applications yet. You cannot import case details from an Excel '
'file until you have existing cases or applications.'
)
return render(
request,
"importer/excel_config.html", {
'named_columns': named_columns,
'columns': columns,
'unrecognized_case_types': unrecognized_case_types,
'case_types_from_apps': case_types_from_apps,
'domain': domain,
'report': {
'name': 'Import: Configuration'
},
'slug': base.ImportCases.slug
}
)
