def CreateSchool(self, data, id, short_code):
school = Entity(self.manager, self.entity_type, data, id=id, short_code=short_code)
school_id = school.save()
school.add_data(data=[("SchoolID", id, self.default_ddtype),
("SchollName", short_code, self.default_ddtype),
("SchoolLocation", data, self.default_ddtype)])
return school_id
def test_invalidate_entity(self):
e = Entity(self.dbm, entity_type='store', location=['nyc'])
e.save()
self.assertFalse(e._doc.void)
apple_type = DataDictType(self.dbm,
name='Apples',
slug='apples',
primitive_type='number')
orange_type = DataDictType(self.dbm,
name='Oranges',
slug='oranges',
primitive_type='number')
apple_type.save()
orange_type.save()
data = [[('apples', 20, apple_type), ('oranges', 30, orange_type)],
[('apples', 10, apple_type), ('oranges', 20, orange_type)]]
data_ids = []
for d in data:
id = e.add_data(d)
self.assertFalse(self.dbm._load_document(id).void)
data_ids.append(id)
e.invalidate()
self.assertTrue(e._doc.void)
for id in data_ids:
self.assertTrue(self.dbm._load_document(id).void)
def test_should_return_data_types(self):
med_type = DataDictType(self.dbm,
name='Medicines',
slug='meds',
primitive_type='number',
description='Number of medications',
tags=['med'])
med_type.save()
doctor_type = DataDictType(self.dbm,
name='Doctor',
slug='doc',
primitive_type='string',
description='Name of doctor',
tags=['doctor', 'med'])
doctor_type.save()
facility_type = DataDictType(self.dbm,
name='Facility',
slug='facility',
primitive_type='string',
description='Name of facility')
facility_type.save()
e = Entity(self.dbm, entity_type='foo')
e.save()
data_record = [('meds', 20, med_type),
('doc', "aroj", doctor_type),
('facility', 'clinic', facility_type)]
e.add_data(data_record)
# med (tag in list)
types = [typ.slug for typ in e.data_types(['med'])]
self.assertTrue(med_type.slug in types)
self.assertTrue(doctor_type.slug in types)
self.assertTrue(facility_type.slug not in types)
# doctor (tag as string)
types = [typ.slug for typ in e.data_types('doctor')]
self.assertTrue(doctor_type.slug in types)
self.assertTrue(med_type.slug not in types)
self.assertTrue(facility_type.slug not in types)
# med and doctor (more than one tag)
types = [typ.slug for typ in e.data_types(['med', 'doctor'])]
self.assertTrue(doctor_type.slug in types)
self.assertTrue(med_type.slug not in types)
self.assertTrue(facility_type.slug not in types)
# no tags
types = [typ.slug for typ in e.data_types()]
self.assertTrue(med_type.slug in types)
self.assertTrue(doctor_type.slug in types)
self.assertTrue(facility_type.slug in types)
def test_should_return_data_types(self):
med_type = DataDictType(self.dbm,
name='Medicines',
slug='meds',
primitive_type='number',
description='Number of medications',
tags=['med'])
med_type.save()
doctor_type = DataDictType(self.dbm,
name='Doctor',
slug='doc',
primitive_type='string',
description='Name of doctor',
tags=['doctor', 'med'])
doctor_type.save()
facility_type = DataDictType(self.dbm,
name='Facility',
slug='facility',
primitive_type='string',
description='Name of facility')
facility_type.save()
e = Entity(self.dbm, entity_type='foo')
e.save()
data_record = [('meds', 20, med_type), ('doc', "aroj", doctor_type),
('facility', 'clinic', facility_type)]
e.add_data(data_record)
# med (tag in list)
types = [typ.slug for typ in e.data_types(['med'])]
self.assertTrue(med_type.slug in types)
self.assertTrue(doctor_type.slug in types)
self.assertTrue(facility_type.slug not in types)
# doctor (tag as string)
types = [typ.slug for typ in e.data_types('doctor')]
self.assertTrue(doctor_type.slug in types)
self.assertTrue(med_type.slug not in types)
self.assertTrue(facility_type.slug not in types)
# med and doctor (more than one tag)
types = [typ.slug for typ in e.data_types(['med', 'doctor'])]
self.assertTrue(doctor_type.slug in types)
self.assertTrue(med_type.slug not in types)
self.assertTrue(facility_type.slug not in types)
# no tags
types = [typ.slug for typ in e.data_types()]
self.assertTrue(med_type.slug in types)
self.assertTrue(doctor_type.slug in types)
self.assertTrue(facility_type.slug in types)
def test_invalidate_data(self):
e = Entity(self.dbm, entity_type='store', location=['nyc'])
e.save()
apple_type = DataDictType(self.dbm, name='Apples', slug='apples', primitive_type='number')
orange_type = DataDictType(self.dbm, name='Oranges', slug='oranges', primitive_type='number')
apple_type.save()
orange_type.save()
data = e.add_data([('apples', 20, apple_type), ('oranges', 30, orange_type)])
valid_doc = self.dbm._load_document(data)
self.assertFalse(valid_doc.void)
e.invalidate_data(data)
invalid_doc = self.dbm._load_document(data)
self.assertTrue(invalid_doc.void)
def test_subject_index_dict(self):
dbm = Mock(spec=DatabaseManager)
entity_type = ['entity_type']
mock_entity = Entity(dbm=dbm, entity_type=entity_type)
mock_entity.add_data(data=[('name1', 'ans1'), ('name2', 'ans2')])
with patch.object(Entity, 'get') as get:
get.return_value = mock_entity
form_model = EntityFormModel(dbm=dbm, entity_type=entity_type)
form_model.add_field(TextField('name1', 'code1', 'label1'))
form_model.add_field(TextField('name2', 'code2', 'label2'))
form_model._doc = Mock(form_code="abc", id='form_id')
entity_doc = Mock()
result = subject_dict(entity_type, entity_doc, dbm, form_model)
expected = {
'form_id_code1': 'ans1',
'form_id_code2': 'ans2',
'entity_type': ['entity_type'],
'void': False
}
self.assertEquals(result, expected)
def test_invalidate_data(self):
e = Entity(self.dbm, entity_type='store', location=['nyc'])
e.save()
apple_type = DataDictType(self.dbm,
name='Apples',
slug='apples',
primitive_type='number')
orange_type = DataDictType(self.dbm,
name='Oranges',
slug='oranges',
primitive_type='number')
apple_type.save()
orange_type.save()
data = e.add_data([('apples', 20, apple_type),
('oranges', 30, orange_type)])
valid_doc = self.dbm._load_document(data)
self.assertFalse(valid_doc.void)
e.invalidate_data(data)
invalid_doc = self.dbm._load_document(data)
self.assertTrue(invalid_doc.void)
def test_invalidate_entity(self):
e = Entity(self.dbm, entity_type='store', location=['nyc'])
e.save()
self.assertFalse(e._doc.void)
apple_type = DataDictType(self.dbm, name='Apples', slug='apples', primitive_type='number')
orange_type = DataDictType(self.dbm, name='Oranges', slug='oranges', primitive_type='number')
apple_type.save()
orange_type.save()
data = [
[('apples', 20, apple_type), ('oranges', 30, orange_type)],
[('apples', 10, apple_type), ('oranges', 20, orange_type)]
]
data_ids = []
for d in data:
id = e.add_data(d)
self.assertFalse(self.dbm._load_document(id).void)
data_ids.append(id)
e.invalidate()
self.assertTrue(e._doc.void)
for id in data_ids:
self.assertTrue(self.dbm._load_document(id).void)
def create_reporter(self):
r = Entity(self.manager, entity_type=["reporter"], short_code='ashwin')
r.add_data((('mobile_number', '2998288332'), ('name', 'ashwin')))
r.save()
return r
def _create_entities_and_data_records(self):
ENTITY_TYPE = ["Health_Facility", "Clinic"]
self.entity_type = ENTITY_TYPE
FEB = datetime.datetime(2011, 02, 01, tzinfo=UTC)
MARCH = datetime.datetime(2011, 03, 01, tzinfo=UTC)
# Entities for State 1: Maharashtra
e = Entity(self.manager,
entity_type=ENTITY_TYPE,
location=['India', 'MH', 'Pune'])
e.save()
e.add_data(data=[("beds", 300, self.dd_types['beds']),
("meds", 20, self.dd_types['meds']),
("director", "Dr. A", self.dd_types['director']),
("patients", 10, self.dd_types['patients'])],
event_time=FEB)
e.add_data(data=[("beds", 500, self.dd_types['beds']),
("meds", 20, self.dd_types['meds']),
("patients", 20, self.dd_types['patients'])],
event_time=MARCH)
e = Entity(self.manager,
entity_type=ENTITY_TYPE,
location=['India', 'MH', 'Pune'])
e.save()
e.add_data(data=[("beds", 100, self.dd_types['beds']),
("meds", 10, self.dd_types['meds']),
("director", "Dr. AA", self.dd_types['director']),
("patients", 50, self.dd_types['patients'])],
event_time=FEB)
e.add_data(data=[("beds", 200, self.dd_types['beds']),
("meds", 20, self.dd_types['meds']),
("patients", 20, self.dd_types['patients'])],
event_time=MARCH)
e = Entity(self.manager,
entity_type=ENTITY_TYPE,
location=['India', 'MH', 'Mumbai'])
e.save()
e.add_data(data=[("beds", 100, self.dd_types['beds']),
("meds", 10, self.dd_types['meds']),
("director", "Dr. AAA", self.dd_types['director']),
("patients", 50, self.dd_types['patients'])],
event_time=FEB)
e.add_data(data=[("beds", 200, self.dd_types['beds']),
("meds", 20, self.dd_types['meds']),
("patients", 50, self.dd_types['patients'])],
event_time=MARCH)
# Entities for State 2: karnataka
e = Entity(self.manager,
entity_type=ENTITY_TYPE,
location=['India', 'Karnataka', 'Bangalore'])
e.save()
e.add_data(data=[("beds", 100, self.dd_types['beds']),
("meds", 250, self.dd_types['meds']),
("director", "Dr. B1", self.dd_types['director']),
("patients", 50, self.dd_types['patients'])],
event_time=FEB)
e.add_data(data=[("beds", 200, self.dd_types['beds']),
("meds", 400, self.dd_types['meds']),
("director", "Dr. B2", self.dd_types['director']),
("patients", 20, self.dd_types['patients'])],
event_time=MARCH)
e = Entity(self.manager,
entity_type=ENTITY_TYPE,
location=['India', 'Karnataka', 'Hubli'])
e.save()
e.add_data(data=[("beds", 100, self.dd_types['beds']),
("meds", 250, self.dd_types['meds']),
("director", "Dr. B1", self.dd_types['director']),
("patients", 50, self.dd_types['patients'])],
event_time=FEB)
e.add_data(data=[("beds", 200, self.dd_types['beds']),
("meds", 400, self.dd_types['meds']),
("director", "Dr. B2", self.dd_types['director']),
("patients", 20, self.dd_types['patients'])],
event_time=MARCH)
# Entities for State 3: Kerala
e = Entity(self.manager,
entity_type=ENTITY_TYPE,
location=['India', 'Kerala', 'Kochi'])
e.save()
e.add_data(data=[("beds", 200, self.dd_types['beds']),
("meds", 50, self.dd_types['meds']),
("director", "Dr. C", self.dd_types['director']),
("patients", 12, self.dd_types['patients'])],
event_time=MARCH)
def _import_data(self, server, database, max_facilities_to_import=DEFAULT_FACILITIES_TO_IMPORT,
max_mdg_to_import=DEFAULT_MDG_TO_IMPORT):
from mangrove.datastore.database import DatabaseManager
from mangrove.datastore.entity import Entity, get_entities_by_value
from mangrove.datastore.datadict import DataDictType, get_datadict_type, create_datadict_type
from mangrove.utils import GoogleSpreadsheetsClient
from mangrove.utils.google_spreadsheets import get_string, get_number, get_boolean, get_list
from mangrove.utils.spreadsheets import CsvReader
from mangrove.utils.helpers import slugify
from mangrove.georegistry.api import get_feature_by_id
import os
import datetime
import json
from pytz import UTC
print "Loading 'NIMS Data'..."
print "\tServer: %s" % server
print "\tDatabase: %s" % database
dbm = DatabaseManager(server=server, database=database)
user_spreadsheets = GoogleSpreadsheetsClient(settings.GMAIL_USERNAME, settings.GMAIL_PASSWORD)
nims_data = user_spreadsheets['NIMS Data Deux']
load_population = True
load_other = True
load_mdg = True
load_health = True
load_water = True
load_education = True
countries = {}
states = {}
locations = {}
num_cgs = 0
datadict_types = {}
geo_id_dict = {}
cgs_type = create_datadict_type(
dbm,
slug='cgs',
name='CGS',
primitive_type='boolean'
)
datadict_types['cgs'] = cgs_type.id
geo_id_type = create_datadict_type(
dbm,
slug='geo_id',
name='Geographic ID',
primitive_type='string'
)
datadict_types['geo_id'] = geo_id_type.id
name_type = create_datadict_type(
dbm,
slug='name',
name='Name',
primitive_type='string'
)
datadict_types['name'] = name_type.id
mdg_type = create_datadict_type(
dbm,
slug='mdg',
name='MDG',
primitive_type='string'
)
datadict_types['mdg'] = mdg_type.id
country_geo_id = {}
for row in nims_data['Nigeria Country ALL']:
country_geo_id[row['name']] = row['grid']
state_geo_ids = {}
for row in nims_data['Nigeria States ALL']:
state_geo_ids[row['name']] = row['grid']
num_rows = 0
print "Importing location entities from 'Nigeria LGAs ALL' worksheet"
for row in nims_data['Nigeria LGAs ALL']:
country = get_string('country', row)
state = get_string('state', row)
lga = get_string('lga', row)
cgs = get_boolean('cgs', row)
geo_id = get_string('geoid', row)
lga_gr_id = get_string('grid', row)
location = (country, state, lga)
if country not in countries:
gr_id = country_geo_id[country]
feature = get_feature_by_id(gr_id)
# geometry = feature['geometry']
centroid = json.loads(feature['properties']['geometry_centroid'])
e = Entity(dbm,
entity_type=["Location", "Country"],
location=[country],
centroid=centroid,
gr_id=gr_id)
locations[(country,)] = e.save()
countries[country] = e.id
data = [(name_type.slug, country, name_type)]
e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC))
num_rows += 1
print "[%s]...(%s) -- %s" % (num_rows, country, e.id)
e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC))
num_rows += 1
print "[%s]...(%s, %s) -- %s" % (num_rows, country, state, e.id)
gr_id = lga_gr_id
feature = get_feature_by_id(gr_id)
# geometry = feature['geometry']
centroid = json.loads(feature['properties']['geometry_centroid'])
e = Entity(dbm,
entity_type=["Location", "LGA"],
location=[country, state, lga],
centroid=centroid,
gr_id=gr_id)
locations[location] = e.save()
geo_id_dict[geo_id] = e
data = [(name_type.slug, lga, name_type)]
e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC))
data = [(geo_id_type.slug, geo_id, geo_id_type)]
e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC))
num_rows += 1
print "[%s]...(%s, %s, %s) -- %s" % (num_rows, country, state, lga, e.id)
if cgs:
num_cgs += 1
e.add_data(data=[(cgs_type.slug, cgs, cgs_type)])
print "Countries (%d)" % len(countries)
print "States (%d)" % len(states)
print "LGAs (%d) (%d as CGS)" % ((len(locations) - len(countries) - len(states)), num_cgs)
print "Total locations (%d)" % len(locations)
lga_loaded = []
def CreateReporter(self):
reporter = Entity(self.manager, location=["China", "Beijing"], short_code="rpt_bj", id="Reporter001",
entity_type=self.reporter_type)
reporter.add_data(data=[(MOBILE_NUMBER_FIELD, "1234567890", self.phone_number_type),(NAME_FIELD, "001s", self.first_name_type)])
reporter_id = reporter.save()
return reporter_id, reporter
def _import_data(self,
server,
database,
max_facilities_to_import=DEFAULT_FACILITIES_TO_IMPORT,
max_mdg_to_import=DEFAULT_MDG_TO_IMPORT):
from mangrove.datastore.database import DatabaseManager
from mangrove.datastore.entity import Entity, get_entities_by_value
from mangrove.datastore.datadict import DataDictType, get_datadict_type, create_datadict_type
from mangrove.utils import GoogleSpreadsheetsClient
from mangrove.utils.google_spreadsheets import get_string, get_number, get_boolean, get_list
from mangrove.utils.spreadsheets import CsvReader
from mangrove.utils.helpers import slugify
from mangrove.georegistry.api import get_feature_by_id
import os
import datetime
import json
from pytz import UTC
print "Loading 'NIMS Data'..."
print "\tServer: %s" % server
print "\tDatabase: %s" % database
dbm = DatabaseManager(server=server, database=database)
user_spreadsheets = GoogleSpreadsheetsClient(settings.GMAIL_USERNAME,
settings.GMAIL_PASSWORD)
nims_data = user_spreadsheets['NIMS Data Deux']
load_population = True
load_other = True
load_mdg = True
load_health = True
load_water = True
load_education = True
countries = {}
states = {}
locations = {}
num_cgs = 0
datadict_types = {}
geo_id_dict = {}
cgs_type = create_datadict_type(dbm,
slug='cgs',
name='CGS',
primitive_type='boolean')
datadict_types['cgs'] = cgs_type.id
geo_id_type = create_datadict_type(dbm,
slug='geo_id',
name='Geographic ID',
primitive_type='string')
datadict_types['geo_id'] = geo_id_type.id
name_type = create_datadict_type(dbm,
slug='name',
name='Name',
primitive_type='string')
datadict_types['name'] = name_type.id
mdg_type = create_datadict_type(dbm,
slug='mdg',
name='MDG',
primitive_type='string')
datadict_types['mdg'] = mdg_type.id
country_geo_id = {}
for row in nims_data['Nigeria Country ALL']:
country_geo_id[row['name']] = row['grid']
state_geo_ids = {}
for row in nims_data['Nigeria States ALL']:
state_geo_ids[row['name']] = row['grid']
num_rows = 0
print "Importing location entities from 'Nigeria LGAs ALL' worksheet"
for row in nims_data['Nigeria LGAs ALL']:
country = get_string('country', row)
state = get_string('state', row)
lga = get_string('lga', row)
cgs = get_boolean('cgs', row)
geo_id = get_string('geoid', row)
lga_gr_id = get_string('grid', row)
location = (country, state, lga)
if country not in countries:
gr_id = country_geo_id[country]
feature = get_feature_by_id(gr_id)
# geometry = feature['geometry']
centroid = json.loads(
feature['properties']['geometry_centroid'])
e = Entity(dbm,
entity_type=["Location", "Country"],
location=[country],
centroid=centroid,
gr_id=gr_id)
locations[(country, )] = e.save()
countries[country] = e.id
data = [(name_type.slug, country, name_type)]
e.add_data(data,
event_time=datetime.datetime(2011,
03,
01,
tzinfo=UTC))
num_rows += 1
print "[%s]...(%s) -- %s" % (num_rows, country, e.id)
num_rows += 1
print "[%s]...(%s, %s) -- %s" % (num_rows, country, state,
e.id)
gr_id = lga_gr_id
feature = get_feature_by_id(gr_id)
# geometry = feature['geometry']
centroid = json.loads(feature['properties']['geometry_centroid'])
e = Entity(dbm,
entity_type=["Location", "LGA"],
location=[country, state, lga],
centroid=centroid,
gr_id=gr_id)
locations[location] = e.save()
geo_id_dict[geo_id] = e
data = [(name_type.slug, lga, name_type)]
e.add_data(data,
event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC))
data = [(geo_id_type.slug, geo_id, geo_id_type)]
e.add_data(data,
event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC))
num_rows += 1
print "[%s]...(%s, %s, %s) -- %s" % (num_rows, country, state, lga,
e.id)
if cgs:
num_cgs += 1
e.add_data(data=[(cgs_type.slug, cgs, cgs_type)])
print "Countries (%d)" % len(countries)
print "States (%d)" % len(states)
print "LGAs (%d) (%d as CGS)" % (
(len(locations) - len(countries) - len(states)), num_cgs)
def _create_entities_and_data_records(self):
ENTITY_TYPE = ["Health_Facility", "Clinic"]
self.entity_type = ENTITY_TYPE
FEB = datetime.datetime(2011, 02, 01, tzinfo=UTC)
MARCH = datetime.datetime(2011, 03, 01, tzinfo=UTC)
# Entities for State 1: Maharashtra
e = Entity(self.manager, entity_type=ENTITY_TYPE, location=['India', 'MH', 'Pune'])
e.save()
e.add_data(data=[("beds", 300, self.dd_types['beds']), ("meds", 20, self.dd_types['meds']),
("director", "Dr. A", self.dd_types['director']), ("patients", 10, self.dd_types['patients'])],
event_time=FEB)
e.add_data(data=[("beds", 500, self.dd_types['beds']), ("meds", 20, self.dd_types['meds']),
("patients", 20, self.dd_types['patients'])],
event_time=MARCH)
e = Entity(self.manager, entity_type=ENTITY_TYPE, location=['India', 'MH', 'Pune'])
e.save()
e.add_data(data=[("beds", 100, self.dd_types['beds']), ("meds", 10, self.dd_types['meds']),
("director", "Dr. AA", self.dd_types['director']), ("patients", 50, self.dd_types['patients'])],
event_time=FEB)
e.add_data(data=[("beds", 200, self.dd_types['beds']), ("meds", 20, self.dd_types['meds']),
("patients", 20, self.dd_types['patients'])],
event_time=MARCH)
e = Entity(self.manager, entity_type=ENTITY_TYPE, location=['India', 'MH', 'Mumbai'])
e.save()
e.add_data(data=[("beds", 100, self.dd_types['beds']), ("meds", 10, self.dd_types['meds']),
("director", "Dr. AAA", self.dd_types['director']), ("patients", 50, self.dd_types['patients'])],
event_time=FEB)
e.add_data(data=[("beds", 200, self.dd_types['beds']), ("meds", 20, self.dd_types['meds']),
("patients", 50, self.dd_types['patients'])],
event_time=MARCH)
# Entities for State 2: karnataka
e = Entity(self.manager, entity_type=ENTITY_TYPE, location=['India', 'Karnataka', 'Bangalore'])
e.save()
e.add_data(data=[("beds", 100, self.dd_types['beds']), ("meds", 250, self.dd_types['meds']),
("director", "Dr. B1", self.dd_types['director']), ("patients", 50, self.dd_types['patients'])],
event_time=FEB)
e.add_data(data=[("beds", 200, self.dd_types['beds']), ("meds", 400, self.dd_types['meds']),
("director", "Dr. B2", self.dd_types['director']), ("patients", 20, self.dd_types['patients'])],
event_time=MARCH)
e = Entity(self.manager, entity_type=ENTITY_TYPE, location=['India', 'Karnataka', 'Hubli'])
e.save()
e.add_data(data=[("beds", 100, self.dd_types['beds']), ("meds", 250, self.dd_types['meds']),
("director", "Dr. B1", self.dd_types['director']), ("patients", 50, self.dd_types['patients'])],
event_time=FEB)
e.add_data(data=[("beds", 200, self.dd_types['beds']), ("meds", 400, self.dd_types['meds']),
("director", "Dr. B2", self.dd_types['director']), ("patients", 20, self.dd_types['patients'])],
event_time=MARCH)
# Entities for State 3: Kerala
e = Entity(self.manager, entity_type=ENTITY_TYPE, location=['India', 'Kerala', 'Kochi'])
e.save()
e.add_data(data=[("beds", 200, self.dd_types['beds']), ("meds", 50, self.dd_types['meds']),
("director", "Dr. C", self.dd_types['director']), ("patients", 12, self.dd_types['patients'])],
event_time=MARCH)
