def generate_fh_submission(csv_in, form):
headers = []
errors = []
first = True
count = 0
success = 0
for line in UnicodeReader(open(csv_in)):
xforms = []
if first:
headers = line
first = False
continue
sub_dict = dict(zip(headers, line))
nest_flat_dict(sub_dict)
for key in INDEXES.keys():
bc = sub_dict.get(u'found', {}).get(key, NA_START)
if bc.startswith(NA_START):
continue
count += 1
submission_dict = copy.deepcopy(sub_dict)
submission_dict.update({u'barcode': bc,
u'position': key})
print(u"%d Adding %s:%s" % (count, key, bc))
xforms.append(json2xform(submission_dict, NEW_FORMS[0]))
try:
print(u"Submitting %d xforms" % len(xforms))
submit_xml_forms_formhub_raw(submission_url=SUBMISSION_URL,
xforms=xforms, as_bulk=True,
attachments=None,
bulk_submission_url=BULK_SUBMISSION_URL,
timeout=FH_TIMEOUT)
success += 1
except (ErrorUploadingDataToFormhub,
ErrorMultipleUploadingDataToFormhub) as e:
print(e)
print(e.details())
errors.extend(xforms)
print(u"Submitted %d forms successfuly." % success)
print(u"Re-submitting %d forms." % len(errors))
for error in errors:
try:
submit_xml_forms_formhub_raw(submission_url=SUBMISSION_URL,
xforms=[error], as_bulk=False,
attachments=None,
bulk_submission_url=BULK_SUBMISSION_URL,
timeout=FH_TIMEOUT)
success += 1
except (ErrorUploadingDataToFormhub,
ErrorMultipleUploadingDataToFormhub) as e:
print(e)
print(e.details())
errors.extend(xforms)
def generate_fh_steps(csv_in, form, step):
steps = {
'imei': NA,
'start_time': NA,
'end_time': NA,
'survey_day': NA,
'deviceid': NA,
'sim_serial_number': NA,
'phone_number': NA,
'step': step,
'name': NA,
'cp_num': NA,
'pc_destination': NA,
'processing_center': NA,
'scan': [],
}
headers = []
first = True
count = 0
errors = []
success = 0
indivs = 0
for line in UnicodeReader(open(csv_in)):
xforms = []
if first:
headers = line
first = False
continue
submission = copy.deepcopy(steps)
parsed_line = dict(zip(headers, line))
# fill up with common values
for key in submission.keys():
value = parsed_line.get(key, None)
if value is not None:
submission[key] = value
# per form logic
for key in ('pc_destination', 'processing_center'):
if submission.get(key, NA) != NA:
submission[key] = clean_pc_slug(submission[key])
# cp_num used to be a float
if submission.get('cp_num') != NA:
try:
submission['cp_num'] = int(submission['cp_num'])
except:
pass
# form-specific rematch
if parsed_line.get('pc_name'):
submission['processing_center'] = \
clean_pc_slug(parsed_line.get('pc_name'))
# loop on soil_id
for key in parsed_line.keys():
if not key.startswith('scan['):
continue
soil_id = parsed_line.get(key)
if not soil_id in AVAILABLES:
continue
if soil_id in USED[step]:
continue
USED[step].append(soil_id)
submission['scan'].append({'soil_id': soil_id})
if not len(submission['scan']):
continue
else:
indivs += len(submission['scan'])
print(u"%d Adding a form with %d scans" % (count,
len(submission['scan'])))
xforms.append(json2xform(submission, STEPS_FORMS))
count += 1
try:
print(u"Submitting %d xforms" % len(xforms))
submit_xml_forms_formhub_raw(submission_url=SUBMISSION_URL,
xforms=xforms, as_bulk=False,
attachments=None,
bulk_submission_url=BULK_SUBMISSION_URL,
timeout=FH_TIMEOUT)
success += 1
except (ErrorUploadingDataToFormhub,
ErrorMultipleUploadingDataToFormhub) as e:
print(e)
print(e.details())
errors.extend(xforms)
print(u"Submitted %d forms successfuly (%d indivs)." % (success, indivs))
print(u"Re-submitting %d forms." % len(errors))
for error in errors:
try:
submit_xml_forms_formhub_raw(submission_url=SUBMISSION_URL,
xforms=[error], as_bulk=False,
attachments=None,
bulk_submission_url=BULK_SUBMISSION_URL,
timeout=FH_TIMEOUT)
success += 1
except (ErrorUploadingDataToFormhub,
ErrorMultipleUploadingDataToFormhub) as e:
print(e)
print(e.details())
errors.extend(xforms)
