def location_headers(pk):
user = current_user._get_current_object()
# disallow processing other users' files
upload = services.user_uploads.get_or_404(pk=pk, user=user)
try:
dataframe = load_source_file(upload.data)
except Exception:
# delete loaded file
upload.data.delete()
upload.delete()
return abort(400)
deployment = g.deployment
headers = dataframe.columns
template_name = 'frontend/location_headers.html'
if request.method == 'GET':
form = generate_location_update_mapping_form(deployment, headers)
return render_template(template_name, form=form)
else:
form = generate_location_update_mapping_form(deployment, headers,
request.form)
if not form.validate():
return render_template(template_name, form=form)
else:
# get header mappings
data = form.data.copy()
# invoke task asynchronously
kwargs = {'upload_id': unicode(upload.id), 'mappings': data}
tasks.import_locations.apply_async(kwargs=kwargs)
return redirect(url_for('locations.locations_list'))
def import_locations(self, upload_id, mappings, location_set_id, channel=None):
upload = UserUpload.query.filter(UserUpload.id == upload_id).first()
if not upload:
logger.error('Upload %s does not exist, aborting', upload_id)
return
filepath = uploads.path(upload.upload_filename)
if not os.path.exists(filepath):
logger.error('Upload file %s does not exist, aborting', filepath)
upload.delete()
return
with open(filepath, 'rb') as f:
dataframe = helpers.load_source_file(f)
location_set = LocationSet.query.filter(
LocationSet.id == location_set_id).first()
update_locations(dataframe, mappings, location_set, self)
os.remove(filepath)
upload.delete()
msg_body = render_template_string(email_template)
send_email(_('Locations Import Report'), msg_body, [upload.user.email])
def make_import_mapping_form(import_file, location_set):
attributes = {'location_set': location_set, 'validate': _validate_mappings}
map_choices = _make_choices(location_set.get_import_fields().items())
data_frame = load_source_file(import_file)
for index, column in enumerate(data_frame.columns):
attributes[str(index)] = SelectField(column, choices=map_choices)
return type('LocationImportMapForm', (FlaskForm, ), attributes)
def make_import_mapping_form(import_file, participant_set):
attributes = {'validate': _validate_required_fields}
map_choices = _make_choices(participant_set.get_import_fields().items())
data_frame = load_source_file(import_file)
for column in data_frame.columns:
attributes[slugify(column)] = SelectField(column, choices=map_choices)
return type('ParticipantImportMapForm', (FlaskForm, ), attributes)
def import_participants(upload_id, mappings):
upload = services.user_uploads.get(pk=upload_id)
dataframe = helpers.load_source_file(upload.data)
count, errors, warnings = update_participants(dataframe, upload.event,
mappings)
# delete uploaded file
upload.data.delete()
upload.delete()
msg_body = generate_response_email(count, errors, warnings)
send_email(_('Import report'), msg_body, [upload.user.email])
def import_participants(upload_id, mappings):
upload = services.user_uploads.get(pk=upload_id)
dataframe = helpers.load_source_file(upload.data)
count, errors, warnings = update_participants(
dataframe,
upload.event,
mappings
)
# delete uploaded file
upload.data.delete()
upload.delete()
msg_body = generate_response_email(count, errors, warnings)
send_email(_('Import report'), msg_body, [upload.user.email])
def participant_headers(pk):
user = current_user._get_current_object()
# disallow processing other users' files
upload = services.user_uploads.get_or_404(pk=pk, user=user)
try:
dataframe = load_source_file(upload.data)
except Exception:
# delete loaded file
upload.data.delete()
upload.delete()
return abort(400)
deployment = g.deployment
headers = dataframe.columns
template_name = 'frontend/participant_headers.html'
if request.method == 'GET':
form = generate_participant_import_mapping_form(deployment, headers)
return render_template(template_name, form=form)
else:
form = generate_participant_import_mapping_form(
deployment, headers, request.form)
if not form.validate():
return render_template(
template_name,
form=form
)
else:
# get header mappings
data = form.data.copy()
# phone_header = data.pop('phone')
# group_header = data.pop('group')
# mappings = {v: k for k, v in data.iteritems() if v}
# mappings.update(phone=phone_header)
# mappings.update(group=group_header)
# invoke task asynchronously
kwargs = {
'upload_id': unicode(upload.id),
'mappings': data
}
import_participants.apply_async(kwargs=kwargs)
return redirect(url_for('participants.participant_list'))
def location_headers(pk):
user = current_user._get_current_object()
# disallow processing other users' files
upload = services.user_uploads.get_or_404(pk=pk, user=user)
try:
dataframe = load_source_file(upload.data)
except Exception:
# delete loaded file
upload.data.delete()
upload.delete()
return abort(400)
deployment = g.deployment
headers = dataframe.columns
template_name = 'frontend/location_headers.html'
if request.method == 'GET':
form = generate_location_update_mapping_form(deployment, headers)
return render_template(template_name, form=form)
else:
form = generate_location_update_mapping_form(
deployment, headers, request.form)
if not form.validate():
return render_template(
template_name,
form=form
)
else:
# get header mappings
data = form.data.copy()
# invoke task asynchronously
kwargs = {
'upload_id': unicode(upload.id),
'mappings': data
}
import_locations.apply_async(kwargs=kwargs)
return redirect(url_for('locations.locations_list'))
def participant_headers(pk):
user = current_user._get_current_object()
# disallow processing other users' files
upload = services.user_uploads.get_or_404(pk=pk, user=user)
try:
dataframe = load_source_file(upload.data)
except Exception:
# delete loaded file
upload.data.delete()
upload.delete()
return abort(400)
deployment = g.deployment
headers = dataframe.columns
template_name = 'frontend/participant_headers.html'
if request.method == 'GET':
form = generate_participant_import_mapping_form(deployment, headers)
return render_template(template_name, form=form)
else:
form = generate_participant_import_mapping_form(
deployment, headers, request.form)
if not form.validate():
return render_template(template_name, form=form)
else:
# get header mappings
data = form.data.copy()
# phone_header = data.pop('phone')
# group_header = data.pop('group')
# mappings = {v: k for k, v in data.iteritems() if v}
# mappings.update(phone=phone_header)
# mappings.update(group=group_header)
# invoke task asynchronously
kwargs = {'upload_id': unicode(upload.id), 'mappings': data}
import_participants.apply_async(kwargs=kwargs)
return redirect(url_for('participants.participant_list'))
def import_locations(upload_id, mappings):
upload = services.user_uploads.get(pk=upload_id)
dataframe = helpers.load_source_file(upload.data)
update_locations(dataframe, mappings, upload.event)
# fetch and update all submissions
deployment = upload.event.deployment
submissions = services.submissions.all().filter(deployment=deployment)
for submission in submissions:
if submission.location:
submission.location_name_path = helpers.compute_location_path(submission.location)
submission.save(clean=False)
# delete uploaded file
upload.data.delete()
upload.delete()
msg_body = render_template_string(email_template)
send_email(_("Locations Import Report"), msg_body, [upload.user.email])
def import_locations(upload_id, mappings):
upload = services.user_uploads.get(pk=upload_id)
dataframe = helpers.load_source_file(upload.data)
update_locations(dataframe, mappings, upload.event)
# fetch and update all submissions
deployment = upload.event.deployment
submissions = services.submissions.all().filter(deployment=deployment)
for submission in submissions:
if submission.location:
submission.location_name_path = helpers.compute_location_path(
submission.location)
submission.save(clean=False)
# delete uploaded file
upload.data.delete()
upload.delete()
msg_body = render_template_string(email_template)
send_email(_('Locations Import Report'), msg_body, [upload.user.email])
def import_participants(
self, upload_id, mappings, participant_set_id, channel=None):
upload = services.user_uploads.find(id=upload_id).first()
if not upload:
logger.error('Upload %s does not exist, aborting', upload_id)
return
filepath = uploads.path(upload.upload_filename)
if not os.path.exists(filepath):
logger.error('Upload file %s does not exist, aborting', filepath)
upload.delete()
return
with open(filepath, 'rb') as f:
dataframe = helpers.load_source_file(f)
participant_set = services.participant_sets.find(
id=participant_set_id).first()
if not participant_set:
_cleanup_upload(filepath, upload)
logger.error(
'Participant set with id {} does not exist, aborting'.format(
participant_set_id))
count, errors, warnings = update_participants(
dataframe,
mappings,
participant_set,
self
)
# delete uploaded file
os.remove(filepath)
upload.delete()
msg_body = generate_response_email(count, errors, warnings)
send_email(_('Import report'), msg_body, [upload.user.email])
def init_survey_submissions(self, event_id, form_id, upload_id):
event = models.Event.query.filter_by(id=event_id).first()
form = models.Form.query.filter_by(id=form_id).first()
upload = UserUpload.query.filter(UserUpload.id == upload_id).first()
if not upload:
logger.error('Upload %s does not exist, aborting', upload_id)
return
filepath = uploads.path(upload.upload_filename)
if not os.path.exists(filepath):
logger.error('Upload file %s does not exist, aborting', filepath)
upload.delete()
return
with open(filepath, 'rb') as f:
dataframe = helpers.load_source_file(f)
total_records = dataframe.shape[0]
processed_records = 0
error_records = 0
warning_records = 0
error_log = []
for idx in range(len(dataframe.index)):
row = dataframe.iloc[idx]
pid = row.get('PARTICIPANT_ID')
serial = row.get('FORM_SERIAL')
participant = Participant.query.filter(
Participant.participant_id == pid,
Participant.participant_set == event.participant_set).first()
if not participant:
error_records += 1
error_log.append({
'label':
'ERROR',
'message':
gettext('Participant ID %(part_id)s was not found',
part_id=pid)
})
self.update_task_info(total_records=total_records,
processed_records=processed_records,
error_records=error_records,
warning_records=warning_records,
error_log=error_log)
continue
if not serial:
error_records += 1
error_log.append({
'label':
'ERROR',
'message':
gettext(
'No form serial number specified on line %(lineno)d',
lineno=idx + 1)
})
self.update_task_info(total_records=total_records,
processed_records=processed_records,
error_records=error_records,
warning_records=warning_records,
error_log=error_log)
continue
submission = Submission.query.filter(
Submission.form == form, Submission.participant == participant,
Submission.location == participant.location,
Submission.deployment == event.deployment,
Submission.event == event, Submission.serial_no == serial,
Submission.submission_type == 'O').first()
if not submission:
submission = Submission(form_id=form.id,
participant_id=participant.id,
location_id=participant.location.id,
deployment_id=event.deployment.id,
event_id=event.id,
submission_type='O',
serial_no=serial,
data={})
submission.save()
master = Submission.query.filter(
Submission.form == form,
Submission.participant_id == None, # noqa
Submission.location == participant.location,
Submission.deployment == event.deployment,
Submission.event == event,
Submission.serial_no == serial,
Submission.submission_type == 'M').first()
if not master:
master = Submission(form_id=form.id,
participant_id=None,
location_id=participant.location.id,
deployment_id=event.deployment.id,
event_id=event.id,
submission_type='M',
serial_no=serial,
data={})
master.save()
processed_records += 1
self.update_task_info(total_records=total_records,
processed_records=processed_records,
error_records=error_records,
warning_records=warning_records,
error_log=error_log)
