def fetch_directly_observed(): """ Update edited records. """ count = 0 dos = DirectlyObservedSprayingForm.objects.last() formid = dos.data.get("_xform_id") if dos else DIRECTLY_OBSERVED_FORM_ID if formid: data = fetch_form_data(formid, dataids_only=True) if not data: return count pks = [i["_id"] for i in data] received = DirectlyObservedSprayingForm.objects.values_list( "submission_id", flat=True) new_ids = set(pks).difference(set((i for i in received))) for rec in new_ids: data = fetch_form_data(formid, dataid=rec) if data: from mspray.apps.main.utils import ( add_directly_observed_spraying_data) # NOQA add_directly_observed_spraying_data(data) count += 1 return count
def test_fetch_form_data(self): """ Test fetch_form_data. """ with HTTMock(onaio_json_mock): data = fetch_form_data(3563261) self.assertEqual(SUBMISSION_DATA[0], data[0]) data = fetch_form_data(3563261, 3563260) self.assertEqual(SUBMISSION_DATA[0], data[0])
def get_sample_geom(spray_area, household_id, form_id): """Downloads OSM information for a household from the collections form. """ osm_field = 'osmstructure' query = {'spray_area': spray_area, 'HHID': household_id} form_data = fetch_form_data(form_id, query=query) for row in form_data: osmid = row.get('%s:way:id' % osm_field) if osmid: try: household = Household.objects.get(hh_id=osmid) except Household.DoesNotExist: pass else: return household.bgeom # We already have a sample with the geom sample = Sample.objects.filter(spray_area__name=spray_area, household_id=household_id, geom__isnull=False).first() if sample: return sample.geom # download OSM file geom = download_osm_geom(row, osm_field) if geom: return geom return None
def sync_missing_data(formid, ModelClass, sync_func, log_writer): """ Fetches missing data for a form """ if not formid: log_writer("'formid' is required.") return None old_data = ( ModelClass.objects.filter() .order_by("-submission_id") .values_list("submission_id", flat=True) ) raw_data = fetch_form_data(formid, dataids_only=True) if isinstance(raw_data, list): all_data = [rec["_id"] for rec in raw_data] all_data.sort() if all_data is not None and isinstance(all_data, list): new_data = sorted(list(set(all_data) - set(old_data))) count = len(new_data) counter = 0 log_writer("Need to pull {} records from Ona.".format(count)) for dataid in new_data: counter += 1 log_writer( "Pulling {} {} of {}".format(dataid, counter, count) ) rec = fetch_form_data(formid, dataid=dataid) if isinstance(rec, dict): try: sync_func(rec) except IntegrityError: log_writer("Already saved {}".format(dataid)) continue except ValidationError as err: log_writer( "An error occurred while proccessing {}.".format( err ) ) continue else: log_writer("Unable to process {} {}".format(dataid, rec)) if counter % 100 == 0: gc.collect() else: log_writer("DATA not fetched: {}".format(raw_data))
def handle(self, *args, **options): formid = int(options['formid']) latest = DirectlyObservedSprayingForm.objects.filter().order_by( '-submission_id').values_list('submission_id', flat=True).first() data = fetch_form_data(formid, latest) pre_count = DirectlyObservedSprayingForm.objects.count() for a in data: add_directly_observed_spraying_data(a) count = DirectlyObservedSprayingForm.objects.count() print("successfully imported %d data" % (count - pre_count))
def handle(self, *args, **options): formid = int(options["formid"]) latest = (SprayOperatorDailySummary.objects.filter().order_by( "-submission_id").values_list("submission_id", flat=True).first()) data = fetch_form_data(formid, latest) pre_count = SprayOperatorDailySummary.objects.count() for a in data: add_spray_operator_daily(a) count = SprayOperatorDailySummary.objects.count() print("successfully imported %d data" % (count - pre_count))
def get_updated_osm_from_ona(sp): if FORM_ID: data = fetch_form_data(FORM_ID, dataid=sp.submission_id) if data: osmid = get_osmid(data) if osmid: sp.data = data sp.save() return osmid
def get_missing_ids(formid, target_class): """Return submission ids not yet synchronised.""" data_ids = fetch_form_data(formid, dataids_only=True) if data_ids: data_ids = set(i["_id"] for i in data_ids) existing = set(i for i in target_class.objects.values_list( "submission_id", flat=True)) return data_ids - existing return []
def update_edited_records(): """ Update edited records. """ count = 0 if FORM_ID: data = fetch_form_data(FORM_ID, dataids_only=True, edited_only=True) if not data: return count pks = [i["_id"] for i in data] edited_submissions = SprayDay.objects.filter(submission_id__in=pks) for rec in edited_submissions: data = fetch_form_data(FORM_ID, dataid=rec.submission_id) if data: from mspray.apps.main.utils import add_spray_data add_spray_data(data) count += 1 return count
def fetch_mobilisation(): """Fetch mobilisation submissions.""" formid = getattr(settings, "MOBILISATION_FORM_ID", None) if formid: data_ids = get_missing_ids(formid, Mobilisation) for data_id in data_ids: data = fetch_form_data(formid, dataid=data_id) if data: try: create_mobilisation_visit(data) except IntegrityError: logger.exception("{} Record not found.".format(formid)) continue
def fetch_sensitization_visits(): """Fetch sensitization visit submissions.""" formid = getattr(settings, "SENSITIZATION_VISIT_FORM_ID", None) if formid: data_ids = get_missing_ids(formid, SensitizationVisit) for data_id in data_ids: data = fetch_form_data(formid, dataid=data_id) if data: try: create_sensitization_visit(data) except IntegrityError: # Fail silently, likely we did not find the household # matching the osm id. pass
def fetch_updated_directly_observed(): """ Update edited records. """ count = 0 dos = DirectlyObservedSprayingForm.objects.last() formid = dos.data.get("_xform_id") if dos else DIRECTLY_OBSERVED_FORM_ID if formid: data = fetch_form_data(formid, dataids_only=True, edited_only=True) if not data: return count pks = [i["_id"] for i in data] for rec in pks: data = fetch_form_data(formid, dataid=rec) if data: from mspray.apps.main.utils import ( add_directly_observed_spraying_data) # NOQA add_directly_observed_spraying_data(data) count += 1 return count
def remove_deleted_records(): """Remove deleted records.""" count = 0 if FORM_ID: data = fetch_form_data(FORM_ID, dataids_only=True) if not data: return count pks = [i["_id"] for i in data] deleted_submissions = SprayDay.objects.exclude(submission_id__in=pks) count = deleted_submissions.count() deleted_submissions.delete() return count
def handle(self, *args, **options): formid = int(options["formid"]) if "formid" in options else FORMID if formid != 0: latest = (SprayDay.objects.filter().order_by( "-submission_id").values_list("submission_id", flat=True).first()) data = fetch_form_data(formid, latest) if data is not None and isinstance(data, list): for rec in data: if SprayDay.objects.filter( submission_id=rec.get("_id")).count(): continue try: add_spray_data(rec) except IntegrityError: continue
def remove_deleted_dos_records(): """ Remove directly observed records.. """ count = 0 dos = DirectlyObservedSprayingForm.objects.last() formid = dos.data.get("_xform_id") if dos else DIRECTLY_OBSERVED_FORM_ID if formid: data = fetch_form_data(formid, dataids_only=True) if not data: return count pks = [i["_id"] for i in data] deleted_submissions = DirectlyObservedSprayingForm.objects.exclude( submission_id__in=pks) count = deleted_submissions.count() deleted_submissions.delete() return count
def remove_deleted_daily_summary_records(): """ Deletes deleted Daily Summary records. """ count = 0 summary = SprayOperatorDailySummary.objects.last() formid = summary and summary.data.get("_xform_id") if formid: data = fetch_form_data(formid, dataids_only=True) if not data: return count pks = [i["_id"] for i in data] records = SprayOperatorDailySummary.objects.exclude( submission_id__in=pks) count = records.count() records.delete() return count
def handle(self, *args, **options): formid = int(options['formid']) if 'formid' in options else FORMID if formid != 0: data = fetch_form_data(formid, edited_only=True) if data is not None and isinstance(data, list): for rec in data: try: sprayday = SprayDay.objects.get( submission_id=rec.get('_id') ) except SprayDay.DoesNotExist: pass else: sprayday.delete() try: add_spray_data(rec) except IntegrityError: continue
def load_samples_from_ona(form_id=None, collections_form_id=None): """Loads the Ento Counts form into the Sample model.""" if not form_id: form_id = getattr(settings, 'SAMPLE_FORM_ID', None) if not form_id: return None if not collections_form_id: collections_form_id = getattr(settings, 'COLLECTIONS_FORM_ID', None) data = fetch_form_data(form_id) or [] for row in data: submission_id = row['_id'] district = Location.objects.filter(level='district').get( code=int(row['district'])) try: spray_area = Location.objects.filter( level='ta', parent__parent=district).get(name=row['spray_area']) except Location.DoesNotExist: print("Skipping:", row['spray_area']) continue row['fem_anoph'] = int(row['fem_anoph']) row['bl_anoph'] = int(row['bl_anoph']) row['grav_anoph'] = int(row['grav_anoph']) row['male_anoph'] = int(row['male_anoph']) row['tot_anoph'] = int(row['tot_anoph']) row['fem_culex'] = int(row['fem_culex']) row['male_culex'] = int(row['male_culex']) row['tot_culex'] = int(row['tot_culex']) row['fem_aedes'] = int(row['fem_aedes']) row['male_aedes'] = int(row['male_aedes']) row['tot_aedes'] = int(row['tot_aedes']) row['tot_mosq'] = int(row['tot_mosq']) if 'visit' in row: try: row['visit'] = int(row['visit']) except ValueError: if row['visit'] == 'n/a': row['visit'] = 1 else: row['visit'] = 1 try: sample = Sample.objects.create( collection_method=row['collection_method'], district=district, household_id=row['HHID'], sample_date=row['date'], spray_area=spray_area, submission_id=submission_id, visit=row['visit'], data=row) except IntegrityError: sample = Sample.objects.get(submission_id=submission_id) if not sample.geom: set_sample_geom(sample, collections_form_id) continue else: set_sample_geom(sample, collections_form_id)