def fetch_directly_observed():
"""
Update edited records.
"""
count = 0
dos = DirectlyObservedSprayingForm.objects.last()
formid = dos.data.get("_xform_id") if dos else DIRECTLY_OBSERVED_FORM_ID
if formid:
data = fetch_form_data(formid, dataids_only=True)
if not data:
return count
pks = [i["_id"] for i in data]
received = DirectlyObservedSprayingForm.objects.values_list(
"submission_id", flat=True)
new_ids = set(pks).difference(set((i for i in received)))
for rec in new_ids:
data = fetch_form_data(formid, dataid=rec)
if data:
from mspray.apps.main.utils import (
add_directly_observed_spraying_data) # NOQA
add_directly_observed_spraying_data(data)
count += 1
return count
def test_fetch_form_data(self):
"""
Test fetch_form_data.
"""
with HTTMock(onaio_json_mock):
data = fetch_form_data(3563261)
self.assertEqual(SUBMISSION_DATA[0], data[0])
data = fetch_form_data(3563261, 3563260)
self.assertEqual(SUBMISSION_DATA[0], data[0])
def get_sample_geom(spray_area, household_id, form_id):
"""Downloads OSM information for a household from the collections form.
"""
osm_field = 'osmstructure'
query = {'spray_area': spray_area, 'HHID': household_id}
form_data = fetch_form_data(form_id, query=query)
for row in form_data:
osmid = row.get('%s:way:id' % osm_field)
if osmid:
try:
household = Household.objects.get(hh_id=osmid)
except Household.DoesNotExist:
pass
else:
return household.bgeom
# We already have a sample with the geom
sample = Sample.objects.filter(spray_area__name=spray_area,
household_id=household_id,
geom__isnull=False).first()
if sample:
return sample.geom
# download OSM file
geom = download_osm_geom(row, osm_field)
if geom:
return geom
return None
def sync_missing_data(formid, ModelClass, sync_func, log_writer):
"""
Fetches missing data for a form
"""
if not formid:
log_writer("'formid' is required.")
return None
old_data = (
ModelClass.objects.filter()
.order_by("-submission_id")
.values_list("submission_id", flat=True)
)
raw_data = fetch_form_data(formid, dataids_only=True)
if isinstance(raw_data, list):
all_data = [rec["_id"] for rec in raw_data]
all_data.sort()
if all_data is not None and isinstance(all_data, list):
new_data = sorted(list(set(all_data) - set(old_data)))
count = len(new_data)
counter = 0
log_writer("Need to pull {} records from Ona.".format(count))
for dataid in new_data:
counter += 1
log_writer(
"Pulling {} {} of {}".format(dataid, counter, count)
)
rec = fetch_form_data(formid, dataid=dataid)
if isinstance(rec, dict):
try:
sync_func(rec)
except IntegrityError:
log_writer("Already saved {}".format(dataid))
continue
except ValidationError as err:
log_writer(
"An error occurred while proccessing {}.".format(
err
)
)
continue
else:
log_writer("Unable to process {} {}".format(dataid, rec))
if counter % 100 == 0:
gc.collect()
else:
log_writer("DATA not fetched: {}".format(raw_data))
def handle(self, *args, **options):
formid = int(options['formid'])
latest = DirectlyObservedSprayingForm.objects.filter().order_by(
'-submission_id').values_list('submission_id', flat=True).first()
data = fetch_form_data(formid, latest)
pre_count = DirectlyObservedSprayingForm.objects.count()
for a in data:
add_directly_observed_spraying_data(a)
count = DirectlyObservedSprayingForm.objects.count()
print("successfully imported %d data" % (count - pre_count))
def handle(self, *args, **options):
formid = int(options["formid"])
latest = (SprayOperatorDailySummary.objects.filter().order_by(
"-submission_id").values_list("submission_id", flat=True).first())
data = fetch_form_data(formid, latest)
pre_count = SprayOperatorDailySummary.objects.count()
for a in data:
add_spray_operator_daily(a)
count = SprayOperatorDailySummary.objects.count()
print("successfully imported %d data" % (count - pre_count))
def get_updated_osm_from_ona(sp):
if FORM_ID:
data = fetch_form_data(FORM_ID, dataid=sp.submission_id)
if data:
osmid = get_osmid(data)
if osmid:
sp.data = data
sp.save()
return osmid
def get_missing_ids(formid, target_class):
"""Return submission ids not yet synchronised."""
data_ids = fetch_form_data(formid, dataids_only=True)
if data_ids:
data_ids = set(i["_id"] for i in data_ids)
existing = set(i for i in target_class.objects.values_list(
"submission_id", flat=True))
return data_ids - existing
return []
def update_edited_records():
"""
Update edited records.
"""
count = 0
if FORM_ID:
data = fetch_form_data(FORM_ID, dataids_only=True, edited_only=True)
if not data:
return count
pks = [i["_id"] for i in data]
edited_submissions = SprayDay.objects.filter(submission_id__in=pks)
for rec in edited_submissions:
data = fetch_form_data(FORM_ID, dataid=rec.submission_id)
if data:
from mspray.apps.main.utils import add_spray_data
add_spray_data(data)
count += 1
return count
def fetch_mobilisation():
"""Fetch mobilisation submissions."""
formid = getattr(settings, "MOBILISATION_FORM_ID", None)
if formid:
data_ids = get_missing_ids(formid, Mobilisation)
for data_id in data_ids:
data = fetch_form_data(formid, dataid=data_id)
if data:
try:
create_mobilisation_visit(data)
except IntegrityError:
logger.exception("{} Record not found.".format(formid))
continue
def fetch_sensitization_visits():
"""Fetch sensitization visit submissions."""
formid = getattr(settings, "SENSITIZATION_VISIT_FORM_ID", None)
if formid:
data_ids = get_missing_ids(formid, SensitizationVisit)
for data_id in data_ids:
data = fetch_form_data(formid, dataid=data_id)
if data:
try:
create_sensitization_visit(data)
except IntegrityError:
# Fail silently, likely we did not find the household
# matching the osm id.
pass
def fetch_updated_directly_observed():
"""
Update edited records.
"""
count = 0
dos = DirectlyObservedSprayingForm.objects.last()
formid = dos.data.get("_xform_id") if dos else DIRECTLY_OBSERVED_FORM_ID
if formid:
data = fetch_form_data(formid, dataids_only=True, edited_only=True)
if not data:
return count
pks = [i["_id"] for i in data]
for rec in pks:
data = fetch_form_data(formid, dataid=rec)
if data:
from mspray.apps.main.utils import (
add_directly_observed_spraying_data) # NOQA
add_directly_observed_spraying_data(data)
count += 1
return count
def remove_deleted_records():
"""Remove deleted records."""
count = 0
if FORM_ID:
data = fetch_form_data(FORM_ID, dataids_only=True)
if not data:
return count
pks = [i["_id"] for i in data]
deleted_submissions = SprayDay.objects.exclude(submission_id__in=pks)
count = deleted_submissions.count()
deleted_submissions.delete()
return count
def handle(self, *args, **options):
formid = int(options["formid"]) if "formid" in options else FORMID
if formid != 0:
latest = (SprayDay.objects.filter().order_by(
"-submission_id").values_list("submission_id",
flat=True).first())
data = fetch_form_data(formid, latest)
if data is not None and isinstance(data, list):
for rec in data:
if SprayDay.objects.filter(
submission_id=rec.get("_id")).count():
continue
try:
add_spray_data(rec)
except IntegrityError:
continue
def remove_deleted_dos_records():
"""
Remove directly observed records..
"""
count = 0
dos = DirectlyObservedSprayingForm.objects.last()
formid = dos.data.get("_xform_id") if dos else DIRECTLY_OBSERVED_FORM_ID
if formid:
data = fetch_form_data(formid, dataids_only=True)
if not data:
return count
pks = [i["_id"] for i in data]
deleted_submissions = DirectlyObservedSprayingForm.objects.exclude(
submission_id__in=pks)
count = deleted_submissions.count()
deleted_submissions.delete()
return count
def remove_deleted_daily_summary_records():
"""
Deletes deleted Daily Summary records.
"""
count = 0
summary = SprayOperatorDailySummary.objects.last()
formid = summary and summary.data.get("_xform_id")
if formid:
data = fetch_form_data(formid, dataids_only=True)
if not data:
return count
pks = [i["_id"] for i in data]
records = SprayOperatorDailySummary.objects.exclude(
submission_id__in=pks)
count = records.count()
records.delete()
return count
def handle(self, *args, **options):
formid = int(options['formid']) if 'formid' in options else FORMID
if formid != 0:
data = fetch_form_data(formid, edited_only=True)
if data is not None and isinstance(data, list):
for rec in data:
try:
sprayday = SprayDay.objects.get(
submission_id=rec.get('_id')
)
except SprayDay.DoesNotExist:
pass
else:
sprayday.delete()
try:
add_spray_data(rec)
except IntegrityError:
continue
def load_samples_from_ona(form_id=None, collections_form_id=None):
"""Loads the Ento Counts form into the Sample model."""
if not form_id:
form_id = getattr(settings, 'SAMPLE_FORM_ID', None)
if not form_id:
return None
if not collections_form_id:
collections_form_id = getattr(settings, 'COLLECTIONS_FORM_ID', None)
data = fetch_form_data(form_id) or []
for row in data:
submission_id = row['_id']
district = Location.objects.filter(level='district').get(
code=int(row['district']))
try:
spray_area = Location.objects.filter(
level='ta',
parent__parent=district).get(name=row['spray_area'])
except Location.DoesNotExist:
print("Skipping:", row['spray_area'])
continue
row['fem_anoph'] = int(row['fem_anoph'])
row['bl_anoph'] = int(row['bl_anoph'])
row['grav_anoph'] = int(row['grav_anoph'])
row['male_anoph'] = int(row['male_anoph'])
row['tot_anoph'] = int(row['tot_anoph'])
row['fem_culex'] = int(row['fem_culex'])
row['male_culex'] = int(row['male_culex'])
row['tot_culex'] = int(row['tot_culex'])
row['fem_aedes'] = int(row['fem_aedes'])
row['male_aedes'] = int(row['male_aedes'])
row['tot_aedes'] = int(row['tot_aedes'])
row['tot_mosq'] = int(row['tot_mosq'])
if 'visit' in row:
try:
row['visit'] = int(row['visit'])
except ValueError:
if row['visit'] == 'n/a':
row['visit'] = 1
else:
row['visit'] = 1
try:
sample = Sample.objects.create(
collection_method=row['collection_method'],
district=district,
household_id=row['HHID'],
sample_date=row['date'],
spray_area=spray_area,
submission_id=submission_id,
visit=row['visit'],
data=row)
except IntegrityError:
sample = Sample.objects.get(submission_id=submission_id)
if not sample.geom:
set_sample_geom(sample, collections_form_id)
continue
else:
set_sample_geom(sample, collections_form_id)
