def generate_fh_submission(csv_in, form): headers = [] errors = [] first = True count = 0 success = 0 for line in UnicodeReader(open(csv_in)): xforms = [] if first: headers = line first = False continue sub_dict = dict(zip(headers, line)) nest_flat_dict(sub_dict) for key in INDEXES.keys(): bc = sub_dict.get(u'found', {}).get(key, NA_START) if bc.startswith(NA_START): continue count += 1 submission_dict = copy.deepcopy(sub_dict) submission_dict.update({u'barcode': bc, u'position': key}) print(u"%d Adding %s:%s" % (count, key, bc)) xforms.append(json2xform(submission_dict, NEW_FORMS[0])) try: print(u"Submitting %d xforms" % len(xforms)) submit_xml_forms_formhub_raw(submission_url=SUBMISSION_URL, xforms=xforms, as_bulk=True, attachments=None, bulk_submission_url=BULK_SUBMISSION_URL, timeout=FH_TIMEOUT) success += 1 except (ErrorUploadingDataToFormhub, ErrorMultipleUploadingDataToFormhub) as e: print(e) print(e.details()) errors.extend(xforms) print(u"Submitted %d forms successfuly." % success) print(u"Re-submitting %d forms." % len(errors)) for error in errors: try: submit_xml_forms_formhub_raw(submission_url=SUBMISSION_URL, xforms=[error], as_bulk=False, attachments=None, bulk_submission_url=BULK_SUBMISSION_URL, timeout=FH_TIMEOUT) success += 1 except (ErrorUploadingDataToFormhub, ErrorMultipleUploadingDataToFormhub) as e: print(e) print(e.details()) errors.extend(xforms)
def main_form_splitter(request, project_slug='soiltrack'): ''' Unified EthioSIS_ET Form to per-sample (ESTS_ET) ones (submits to FH) 1. Receives a JSON POST from formhub containing qr_* barcodes 2. For each level (qr), prepare a step-specific submission 3. Submits the resulting XForms to formhub. ''' # we need a project to guess formhub URL try: project = Project.objects.get(slug=project_slug) except: project = Project.objects.all()[0] try: jsform = json.loads(request.raw_post_data) # re-organize JSON to match semantic nest_flat_dict(jsform) except: return HttpResponse(u"Unable to parse JSON data", status=400) positions = { 'top_qr': None, 'sub_qr': None, 'qr_0_20': None, 'qr_20_40': None, 'qr_40_60': None, 'qr_60_80': None, 'qr_80_100': None, } forms = [] found = bool(len(jsform.get(u'found', {})) > 1) for key in positions.keys(): positions[key] = jsform.get(u'found', {}).get(key, '') if not re.match(r'[a-zA-Z0-9\_\-]+', positions[key]): positions[key] = None # delete all position keys for key in positions.keys(): try: jsform[u'found'].pop(key) except KeyError: pass for position in positions.keys(): if not positions[position]: continue # extract sample identifier barcode = positions.get(position) # duplicate whole form to grab meta data. form = copy.deepcopy(jsform) # add `barcode` field form[u'barcode'] = barcode form[u'position'] = position forms.append(form) if not found: # make a single submission since we don't have barcodes form = copy.deepcopy(jsform) form[u'barcode'] = u'n/a__%s' % uuid.uuid4().hex form[u'position'] = u'not_found' forms.append(form) # free some mem del(jsform) def json2xform(jsform): # changing the form_id to match correct Step dd = {'form_id': u'ESTS_ET_sample'} xml_head = u"<?xml version='1.0' ?><%(form_id)s id='%(form_id)s'>" % dd xml_tail = u"</%(form_id)s>" % dd # remove the parent's instance ID and step try: jsform[u'meta'].pop('instanceID') except KeyError: pass for field in jsform.keys(): # treat field starting with underscore are internal ones. # and remove them if field.startswith('_'): jsform.pop(field) return xml_head + dict2xml(jsform) + xml_tail xforms = [json2xform(form) for form in forms] try: submit_xml_forms_formhub(project, xforms, as_bulk=True) except (ErrorUploadingDataToFormhub, ErrorMultipleUploadingDataToFormhub) as e: return HttpResponse(u"%(intro)s\n%(detail)s" % {'intro': e, 'detail': e.details()}, status=502) except Exception as e: return HttpResponse(str(e), status=500) return HttpResponse('OK', status=201)
def steps_form_splitter(request, project_slug='soiltrack'): ''' Unified (ESTS_Steps) Form to individual ones (submits to FH) 1. Receives a JSON POST from formhub containing x samples barcode 2. For each sample, prepare a step-specific submission 3. Submits the resulting XForms to formhub. ''' # we need a project to guess formhub URL try: project = Project.objects.get(slug=project_slug) except: project = Project.objects.all()[0] try: jsform = json.loads(request.raw_post_data) # re-organize JSON to match semantic nest_flat_dict(jsform) except: return HttpResponse(u"Unable to parse JSON data", status=400) # we have a json dict containing all fields. # we need to: # 1. explode the form into x ones from soil_id # 2. find out step # 2. rename all fields except barcode to prefix with step forms = [] step = jsform[u'step'] for soil_field in jsform[u'scan']: # looping on repeat field `scan` which contains `soil_id` barcode = soil_field[u'scan/soil_id'] # duplicate whole form to grab meta data. form = copy.deepcopy(jsform) # rename all fields but `_` starting ones keys = form.keys() for key in keys: # _ starting keys are internal. if key.startswith('_'): continue form.update({'%s_%s' % (step, key): form[key]}) # delete key once duplicated form.pop(key) # remove repeat section and replace with `barcode` form.pop(u'%s_scan' % step) form.update({u'barcode': barcode}) forms.append(form) del(jsform) def json2xform(jsform): # changing the form_id to match correct Step dd = {'form_id': u'ESTS_%s' % step.title()} xml_head = u"<?xml version='1.0' ?><%(form_id)s id='%(form_id)s'>" % dd xml_tail = u"</%(form_id)s>" % dd # remove the parent's instance ID and step try: jsform['%s_meta' % step].pop('instanceID') # jsform['%s_formhub' % step].pop('uuid') jsform.pop('%s_step' % step) except KeyError: pass for field in jsform.keys(): # treat field starting with underscore are internal ones. # and remove them if field.startswith('_'): jsform.pop(field) return xml_head + dict2xml(jsform) + xml_tail xforms = [json2xform(form) for form in forms] try: submit_xml_forms_formhub(project, xforms, as_bulk=True) except (ErrorUploadingDataToFormhub, ErrorMultipleUploadingDataToFormhub) as e: return HttpResponse(u"%(intro)s\n%(detail)s" % {'intro': e, 'detail': e.details()}, status=502) except Exception as e: return HttpResponse(str(e), status=500) return HttpResponse('OK', status=201)