def import_locations(self, upload_id, mappings, location_set_id, channel=None):
upload = UserUpload.query.filter(UserUpload.id == upload_id).first()
if not upload:
logger.error('Upload %s does not exist, aborting', upload_id)
return
filepath = uploads.path(upload.upload_filename)
if not os.path.exists(filepath):
logger.error('Upload file %s does not exist, aborting', filepath)
upload.delete()
return
with open(filepath, 'rb') as f:
dataframe = helpers.load_source_file(f)
location_set = LocationSet.query.filter(
LocationSet.id == location_set_id).first()
update_locations(dataframe, mappings, location_set, self)
os.remove(filepath)
upload.delete()
msg_body = render_template_string(email_template)
send_email(_('Locations Import Report'), msg_body, [upload.user.email])
def locations_headers(view, location_set_id, upload_id):
user = current_user._get_current_object()
location_set = LocationSet.query.filter(
LocationSet.id == location_set_id).first_or_404()
# disallow processing other users' files
upload = UserUpload.query.filter(
UserUpload.id == upload_id, UserUpload.user == user).first_or_404()
filepath = uploads.path(upload.upload_filename)
try:
with open(filepath, 'rb') as source_file:
mapping_form_class = forms.make_import_mapping_form(
source_file, location_set)
except Exception:
# log exception (if Sentry is enabled)
sentry.captureException()
# delete loaded file
os.remove(filepath)
upload.delete()
return abort(400)
template_name = 'admin/location_headers.html'
if request.method == 'GET':
form = mapping_form_class()
return view.render(template_name, form=form)
else:
form = mapping_form_class()
if not form.validate():
error_msgs = []
for key in form.errors:
for msg in form.errors[key]:
error_msgs.append(msg)
return view.render(
'admin/location_headers_errors.html',
error_msgs=error_msgs), 400
else:
if 'X-Validate' not in request.headers:
# get header mappings
data = {
field.data: field.label.text
for field in form if field.data
}
# invoke task asynchronously
kwargs = {
'upload_id': upload.id,
'mappings': data,
'location_set_id': location_set_id,
'channel': session.get('_id')
}
tasks.import_locations.apply_async(kwargs=kwargs)
return redirect(url_for('locationset.locations_list',
location_set_id=location_set_id))
def import_participants(
self, upload_id, mappings, participant_set_id, channel=None):
upload = services.user_uploads.find(id=upload_id).first()
if not upload:
logger.error('Upload %s does not exist, aborting', upload_id)
return
filepath = uploads.path(upload.upload_filename)
if not os.path.exists(filepath):
logger.error('Upload file %s does not exist, aborting', filepath)
upload.delete()
return
with open(filepath, 'rb') as f:
dataframe = helpers.load_source_file(f)
participant_set = services.participant_sets.find(
id=participant_set_id).first()
if not participant_set:
_cleanup_upload(filepath, upload)
logger.error(
'Participant set with id {} does not exist, aborting'.format(
participant_set_id))
count, errors, warnings = update_participants(
dataframe,
mappings,
participant_set,
self
)
# delete uploaded file
os.remove(filepath)
upload.delete()
msg_body = generate_response_email(count, errors, warnings)
send_email(_('Import report'), msg_body, [upload.user.email])
def participant_headers(upload_id, participant_set_id=0, view=None):
if participant_set_id:
participant_set = ParticipantSet.query.get_or_404(participant_set_id)
else:
participant_set = g.event.participant_set or abort(404)
user = current_user._get_current_object()
# disallow processing other users' files
upload = UserUpload.query.filter(UserUpload.id == upload_id,
UserUpload.user == user).first_or_404()
filepath = uploads.path(upload.upload_filename)
try:
with open(filepath, 'rb') as source_file:
mapping_form_class = forms.make_import_mapping_form(
source_file, participant_set)
except Exception:
# delete loaded file
os.remove(filepath)
upload.delete()
sentry.captureException()
return abort(400)
form = mapping_form_class()
template_name = 'frontend/participant_headers.html'
if request.method == 'GET':
return render_template(template_name, form=form)
else:
if not form.validate():
error_msgs = []
for key in form.errors:
for msg in form.errors[key]:
error_msgs.append(msg)
if view:
return view.render('frontend/participant_headers_errors.html',
error_msgs=error_msgs), 400
else:
return render_template(
'frontend/participant_headers_errors.html',
error_msgs=error_msgs), 400
else:
if 'X-Validate' not in request.headers:
# get header mappings
multi_column_fields = ('group', 'phone', 'sample')
data = {fi: [] for fi in multi_column_fields}
for field in form:
if not field.data:
continue
if field.data in multi_column_fields:
data[field.data].append(field.label.text)
else:
data[field.data] = field.label.text
for extra_field in participant_set.extra_fields:
value = data.get(str(extra_field.id))
if value is None:
continue
data[extra_field.name] = value
# invoke task asynchronously
kwargs = {
'upload_id': upload.id,
'mappings': data,
'participant_set_id': participant_set.id,
'channel': session.get('_id'),
}
tasks.import_participants.apply_async(kwargs=kwargs)
if participant_set_id:
return redirect(
url_for('participantset.participants_list',
participant_set_id=participant_set_id))
else:
return redirect(url_for('participants.participant_list'))
def init_survey_submissions(self, event_id, form_id, upload_id):
event = models.Event.query.filter_by(id=event_id).first()
form = models.Form.query.filter_by(id=form_id).first()
upload = UserUpload.query.filter(UserUpload.id == upload_id).first()
if not upload:
logger.error('Upload %s does not exist, aborting', upload_id)
return
filepath = uploads.path(upload.upload_filename)
if not os.path.exists(filepath):
logger.error('Upload file %s does not exist, aborting', filepath)
upload.delete()
return
with open(filepath, 'rb') as f:
dataframe = helpers.load_source_file(f)
total_records = dataframe.shape[0]
processed_records = 0
error_records = 0
warning_records = 0
error_log = []
for idx in range(len(dataframe.index)):
row = dataframe.iloc[idx]
pid = row.get('PARTICIPANT_ID')
serial = row.get('FORM_SERIAL')
participant = Participant.query.filter(
Participant.participant_id == pid,
Participant.participant_set == event.participant_set).first()
if not participant:
error_records += 1
error_log.append({
'label':
'ERROR',
'message':
gettext('Participant ID %(part_id)s was not found',
part_id=pid)
})
self.update_task_info(total_records=total_records,
processed_records=processed_records,
error_records=error_records,
warning_records=warning_records,
error_log=error_log)
continue
if not serial:
error_records += 1
error_log.append({
'label':
'ERROR',
'message':
gettext(
'No form serial number specified on line %(lineno)d',
lineno=idx + 1)
})
self.update_task_info(total_records=total_records,
processed_records=processed_records,
error_records=error_records,
warning_records=warning_records,
error_log=error_log)
continue
submission = Submission.query.filter(
Submission.form == form, Submission.participant == participant,
Submission.location == participant.location,
Submission.deployment == event.deployment,
Submission.event == event, Submission.serial_no == serial,
Submission.submission_type == 'O').first()
if not submission:
submission = Submission(form_id=form.id,
participant_id=participant.id,
location_id=participant.location.id,
deployment_id=event.deployment.id,
event_id=event.id,
submission_type='O',
serial_no=serial,
data={})
submission.save()
master = Submission.query.filter(
Submission.form == form,
Submission.participant_id == None, # noqa
Submission.location == participant.location,
Submission.deployment == event.deployment,
Submission.event == event,
Submission.serial_no == serial,
Submission.submission_type == 'M').first()
if not master:
master = Submission(form_id=form.id,
participant_id=None,
location_id=participant.location.id,
deployment_id=event.deployment.id,
event_id=event.id,
submission_type='M',
serial_no=serial,
data={})
master.save()
processed_records += 1
self.update_task_info(total_records=total_records,
processed_records=processed_records,
error_records=error_records,
warning_records=warning_records,
error_log=error_log)