def generate_fh_submission(csv_in, form):
headers = []
errors = []
first = True
count = 0
success = 0
for line in UnicodeReader(open(csv_in)):
xforms = []
if first:
headers = line
first = False
continue
sub_dict = dict(zip(headers, line))
nest_flat_dict(sub_dict)
for key in INDEXES.keys():
bc = sub_dict.get(u'found', {}).get(key, NA_START)
if bc.startswith(NA_START):
continue
count += 1
submission_dict = copy.deepcopy(sub_dict)
submission_dict.update({u'barcode': bc,
u'position': key})
print(u"%d Adding %s:%s" % (count, key, bc))
xforms.append(json2xform(submission_dict, NEW_FORMS[0]))
try:
print(u"Submitting %d xforms" % len(xforms))
submit_xml_forms_formhub_raw(submission_url=SUBMISSION_URL,
xforms=xforms, as_bulk=True,
attachments=None,
bulk_submission_url=BULK_SUBMISSION_URL,
timeout=FH_TIMEOUT)
success += 1
except (ErrorUploadingDataToFormhub,
ErrorMultipleUploadingDataToFormhub) as e:
print(e)
print(e.details())
errors.extend(xforms)
print(u"Submitted %d forms successfuly." % success)
print(u"Re-submitting %d forms." % len(errors))
for error in errors:
try:
submit_xml_forms_formhub_raw(submission_url=SUBMISSION_URL,
xforms=[error], as_bulk=False,
attachments=None,
bulk_submission_url=BULK_SUBMISSION_URL,
timeout=FH_TIMEOUT)
success += 1
except (ErrorUploadingDataToFormhub,
ErrorMultipleUploadingDataToFormhub) as e:
print(e)
print(e.details())
errors.extend(xforms)
def main_form_splitter(request, project_slug='soiltrack'):
''' Unified EthioSIS_ET Form to per-sample (ESTS_ET) ones (submits to FH)
1. Receives a JSON POST from formhub containing qr_* barcodes
2. For each level (qr), prepare a step-specific submission
3. Submits the resulting XForms to formhub. '''
# we need a project to guess formhub URL
try:
project = Project.objects.get(slug=project_slug)
except:
project = Project.objects.all()[0]
try:
jsform = json.loads(request.raw_post_data)
# re-organize JSON to match semantic
nest_flat_dict(jsform)
except:
return HttpResponse(u"Unable to parse JSON data", status=400)
positions = {
'top_qr': None,
'sub_qr': None,
'qr_0_20': None,
'qr_20_40': None,
'qr_40_60': None,
'qr_60_80': None,
'qr_80_100': None,
}
forms = []
found = bool(len(jsform.get(u'found', {})) > 1)
for key in positions.keys():
positions[key] = jsform.get(u'found', {}).get(key, '')
if not re.match(r'[a-zA-Z0-9\_\-]+', positions[key]):
positions[key] = None
# delete all position keys
for key in positions.keys():
try:
jsform[u'found'].pop(key)
except KeyError:
pass
for position in positions.keys():
if not positions[position]:
continue
# extract sample identifier
barcode = positions.get(position)
# duplicate whole form to grab meta data.
form = copy.deepcopy(jsform)
# add `barcode` field
form[u'barcode'] = barcode
form[u'position'] = position
forms.append(form)
if not found:
# make a single submission since we don't have barcodes
form = copy.deepcopy(jsform)
form[u'barcode'] = u'n/a__%s' % uuid.uuid4().hex
form[u'position'] = u'not_found'
forms.append(form)
# free some mem
del(jsform)
def json2xform(jsform):
# changing the form_id to match correct Step
dd = {'form_id': u'ESTS_ET_sample'}
xml_head = u"<?xml version='1.0' ?><%(form_id)s id='%(form_id)s'>" % dd
xml_tail = u"</%(form_id)s>" % dd
# remove the parent's instance ID and step
try:
jsform[u'meta'].pop('instanceID')
except KeyError:
pass
for field in jsform.keys():
# treat field starting with underscore are internal ones.
# and remove them
if field.startswith('_'):
jsform.pop(field)
return xml_head + dict2xml(jsform) + xml_tail
xforms = [json2xform(form) for form in forms]
try:
submit_xml_forms_formhub(project, xforms, as_bulk=True)
except (ErrorUploadingDataToFormhub,
ErrorMultipleUploadingDataToFormhub) as e:
return HttpResponse(u"%(intro)s\n%(detail)s"
% {'intro': e,
'detail': e.details()}, status=502)
except Exception as e:
return HttpResponse(str(e), status=500)
return HttpResponse('OK', status=201)
def steps_form_splitter(request, project_slug='soiltrack'):
''' Unified (ESTS_Steps) Form to individual ones (submits to FH)
1. Receives a JSON POST from formhub containing x samples barcode
2. For each sample, prepare a step-specific submission
3. Submits the resulting XForms to formhub. '''
# we need a project to guess formhub URL
try:
project = Project.objects.get(slug=project_slug)
except:
project = Project.objects.all()[0]
try:
jsform = json.loads(request.raw_post_data)
# re-organize JSON to match semantic
nest_flat_dict(jsform)
except:
return HttpResponse(u"Unable to parse JSON data", status=400)
# we have a json dict containing all fields.
# we need to:
# 1. explode the form into x ones from soil_id
# 2. find out step
# 2. rename all fields except barcode to prefix with step
forms = []
step = jsform[u'step']
for soil_field in jsform[u'scan']:
# looping on repeat field `scan` which contains `soil_id`
barcode = soil_field[u'scan/soil_id']
# duplicate whole form to grab meta data.
form = copy.deepcopy(jsform)
# rename all fields but `_` starting ones
keys = form.keys()
for key in keys:
# _ starting keys are internal.
if key.startswith('_'):
continue
form.update({'%s_%s' % (step, key): form[key]})
# delete key once duplicated
form.pop(key)
# remove repeat section and replace with `barcode`
form.pop(u'%s_scan' % step)
form.update({u'barcode': barcode})
forms.append(form)
del(jsform)
def json2xform(jsform):
# changing the form_id to match correct Step
dd = {'form_id': u'ESTS_%s' % step.title()}
xml_head = u"<?xml version='1.0' ?><%(form_id)s id='%(form_id)s'>" % dd
xml_tail = u"</%(form_id)s>" % dd
# remove the parent's instance ID and step
try:
jsform['%s_meta' % step].pop('instanceID')
# jsform['%s_formhub' % step].pop('uuid')
jsform.pop('%s_step' % step)
except KeyError:
pass
for field in jsform.keys():
# treat field starting with underscore are internal ones.
# and remove them
if field.startswith('_'):
jsform.pop(field)
return xml_head + dict2xml(jsform) + xml_tail
xforms = [json2xform(form) for form in forms]
try:
submit_xml_forms_formhub(project, xforms, as_bulk=True)
except (ErrorUploadingDataToFormhub,
ErrorMultipleUploadingDataToFormhub) as e:
return HttpResponse(u"%(intro)s\n%(detail)s"
% {'intro': e,
'detail': e.details()}, status=502)
except Exception as e:
return HttpResponse(str(e), status=500)
return HttpResponse('OK', status=201)