def generate_fh_submission(csv_in, form): headers = [] errors = [] first = True count = 0 success = 0 for line in UnicodeReader(open(csv_in)): xforms = [] if first: headers = line first = False continue sub_dict = dict(zip(headers, line)) nest_flat_dict(sub_dict) for key in INDEXES.keys(): bc = sub_dict.get(u'found', {}).get(key, NA_START) if bc.startswith(NA_START): continue count += 1 submission_dict = copy.deepcopy(sub_dict) submission_dict.update({u'barcode': bc, u'position': key}) print(u"%d Adding %s:%s" % (count, key, bc)) xforms.append(json2xform(submission_dict, NEW_FORMS[0])) try: print(u"Submitting %d xforms" % len(xforms)) submit_xml_forms_formhub_raw(submission_url=SUBMISSION_URL, xforms=xforms, as_bulk=True, attachments=None, bulk_submission_url=BULK_SUBMISSION_URL, timeout=FH_TIMEOUT) success += 1 except (ErrorUploadingDataToFormhub, ErrorMultipleUploadingDataToFormhub) as e: print(e) print(e.details()) errors.extend(xforms) print(u"Submitted %d forms successfuly." % success) print(u"Re-submitting %d forms." % len(errors)) for error in errors: try: submit_xml_forms_formhub_raw(submission_url=SUBMISSION_URL, xforms=[error], as_bulk=False, attachments=None, bulk_submission_url=BULK_SUBMISSION_URL, timeout=FH_TIMEOUT) success += 1 except (ErrorUploadingDataToFormhub, ErrorMultipleUploadingDataToFormhub) as e: print(e) print(e.details()) errors.extend(xforms)
def generate_fh_steps(csv_in, form, step): steps = { 'imei': NA, 'start_time': NA, 'end_time': NA, 'survey_day': NA, 'deviceid': NA, 'sim_serial_number': NA, 'phone_number': NA, 'step': step, 'name': NA, 'cp_num': NA, 'pc_destination': NA, 'processing_center': NA, 'scan': [], } headers = [] first = True count = 0 errors = [] success = 0 indivs = 0 for line in UnicodeReader(open(csv_in)): xforms = [] if first: headers = line first = False continue submission = copy.deepcopy(steps) parsed_line = dict(zip(headers, line)) # fill up with common values for key in submission.keys(): value = parsed_line.get(key, None) if value is not None: submission[key] = value # per form logic for key in ('pc_destination', 'processing_center'): if submission.get(key, NA) != NA: submission[key] = clean_pc_slug(submission[key]) # cp_num used to be a float if submission.get('cp_num') != NA: try: submission['cp_num'] = int(submission['cp_num']) except: pass # form-specific rematch if parsed_line.get('pc_name'): submission['processing_center'] = \ clean_pc_slug(parsed_line.get('pc_name')) # loop on soil_id for key in parsed_line.keys(): if not key.startswith('scan['): continue soil_id = parsed_line.get(key) if not soil_id in AVAILABLES: continue if soil_id in USED[step]: continue USED[step].append(soil_id) submission['scan'].append({'soil_id': soil_id}) if not len(submission['scan']): continue else: indivs += len(submission['scan']) print(u"%d Adding a form with %d scans" % (count, len(submission['scan']))) xforms.append(json2xform(submission, STEPS_FORMS)) count += 1 try: print(u"Submitting %d xforms" % len(xforms)) submit_xml_forms_formhub_raw(submission_url=SUBMISSION_URL, xforms=xforms, as_bulk=False, attachments=None, bulk_submission_url=BULK_SUBMISSION_URL, timeout=FH_TIMEOUT) success += 1 except (ErrorUploadingDataToFormhub, ErrorMultipleUploadingDataToFormhub) as e: print(e) print(e.details()) errors.extend(xforms) print(u"Submitted %d forms successfuly (%d indivs)." % (success, indivs)) print(u"Re-submitting %d forms." % len(errors)) for error in errors: try: submit_xml_forms_formhub_raw(submission_url=SUBMISSION_URL, xforms=[error], as_bulk=False, attachments=None, bulk_submission_url=BULK_SUBMISSION_URL, timeout=FH_TIMEOUT) success += 1 except (ErrorUploadingDataToFormhub, ErrorMultipleUploadingDataToFormhub) as e: print(e) print(e.details()) errors.extend(xforms)