def CreateSchool(self, data, id, short_code): school = Entity(self.manager, self.entity_type, data, id=id, short_code=short_code) school_id = school.save() school.add_data(data=[("SchoolID", id, self.default_ddtype), ("SchollName", short_code, self.default_ddtype), ("SchoolLocation", data, self.default_ddtype)]) return school_id
def test_invalidate_entity(self): e = Entity(self.dbm, entity_type='store', location=['nyc']) e.save() self.assertFalse(e._doc.void) apple_type = DataDictType(self.dbm, name='Apples', slug='apples', primitive_type='number') orange_type = DataDictType(self.dbm, name='Oranges', slug='oranges', primitive_type='number') apple_type.save() orange_type.save() data = [[('apples', 20, apple_type), ('oranges', 30, orange_type)], [('apples', 10, apple_type), ('oranges', 20, orange_type)]] data_ids = [] for d in data: id = e.add_data(d) self.assertFalse(self.dbm._load_document(id).void) data_ids.append(id) e.invalidate() self.assertTrue(e._doc.void) for id in data_ids: self.assertTrue(self.dbm._load_document(id).void)
def test_should_return_data_types(self): med_type = DataDictType(self.dbm, name='Medicines', slug='meds', primitive_type='number', description='Number of medications', tags=['med']) med_type.save() doctor_type = DataDictType(self.dbm, name='Doctor', slug='doc', primitive_type='string', description='Name of doctor', tags=['doctor', 'med']) doctor_type.save() facility_type = DataDictType(self.dbm, name='Facility', slug='facility', primitive_type='string', description='Name of facility') facility_type.save() e = Entity(self.dbm, entity_type='foo') e.save() data_record = [('meds', 20, med_type), ('doc', "aroj", doctor_type), ('facility', 'clinic', facility_type)] e.add_data(data_record) # med (tag in list) types = [typ.slug for typ in e.data_types(['med'])] self.assertTrue(med_type.slug in types) self.assertTrue(doctor_type.slug in types) self.assertTrue(facility_type.slug not in types) # doctor (tag as string) types = [typ.slug for typ in e.data_types('doctor')] self.assertTrue(doctor_type.slug in types) self.assertTrue(med_type.slug not in types) self.assertTrue(facility_type.slug not in types) # med and doctor (more than one tag) types = [typ.slug for typ in e.data_types(['med', 'doctor'])] self.assertTrue(doctor_type.slug in types) self.assertTrue(med_type.slug not in types) self.assertTrue(facility_type.slug not in types) # no tags types = [typ.slug for typ in e.data_types()] self.assertTrue(med_type.slug in types) self.assertTrue(doctor_type.slug in types) self.assertTrue(facility_type.slug in types)
def test_should_return_data_types(self): med_type = DataDictType(self.dbm, name='Medicines', slug='meds', primitive_type='number', description='Number of medications', tags=['med']) med_type.save() doctor_type = DataDictType(self.dbm, name='Doctor', slug='doc', primitive_type='string', description='Name of doctor', tags=['doctor', 'med']) doctor_type.save() facility_type = DataDictType(self.dbm, name='Facility', slug='facility', primitive_type='string', description='Name of facility') facility_type.save() e = Entity(self.dbm, entity_type='foo') e.save() data_record = [('meds', 20, med_type), ('doc', "aroj", doctor_type), ('facility', 'clinic', facility_type)] e.add_data(data_record) # med (tag in list) types = [typ.slug for typ in e.data_types(['med'])] self.assertTrue(med_type.slug in types) self.assertTrue(doctor_type.slug in types) self.assertTrue(facility_type.slug not in types) # doctor (tag as string) types = [typ.slug for typ in e.data_types('doctor')] self.assertTrue(doctor_type.slug in types) self.assertTrue(med_type.slug not in types) self.assertTrue(facility_type.slug not in types) # med and doctor (more than one tag) types = [typ.slug for typ in e.data_types(['med', 'doctor'])] self.assertTrue(doctor_type.slug in types) self.assertTrue(med_type.slug not in types) self.assertTrue(facility_type.slug not in types) # no tags types = [typ.slug for typ in e.data_types()] self.assertTrue(med_type.slug in types) self.assertTrue(doctor_type.slug in types) self.assertTrue(facility_type.slug in types)
def test_invalidate_data(self): e = Entity(self.dbm, entity_type='store', location=['nyc']) e.save() apple_type = DataDictType(self.dbm, name='Apples', slug='apples', primitive_type='number') orange_type = DataDictType(self.dbm, name='Oranges', slug='oranges', primitive_type='number') apple_type.save() orange_type.save() data = e.add_data([('apples', 20, apple_type), ('oranges', 30, orange_type)]) valid_doc = self.dbm._load_document(data) self.assertFalse(valid_doc.void) e.invalidate_data(data) invalid_doc = self.dbm._load_document(data) self.assertTrue(invalid_doc.void)
def test_subject_index_dict(self): dbm = Mock(spec=DatabaseManager) entity_type = ['entity_type'] mock_entity = Entity(dbm=dbm, entity_type=entity_type) mock_entity.add_data(data=[('name1', 'ans1'), ('name2', 'ans2')]) with patch.object(Entity, 'get') as get: get.return_value = mock_entity form_model = EntityFormModel(dbm=dbm, entity_type=entity_type) form_model.add_field(TextField('name1', 'code1', 'label1')) form_model.add_field(TextField('name2', 'code2', 'label2')) form_model._doc = Mock(form_code="abc", id='form_id') entity_doc = Mock() result = subject_dict(entity_type, entity_doc, dbm, form_model) expected = { 'form_id_code1': 'ans1', 'form_id_code2': 'ans2', 'entity_type': ['entity_type'], 'void': False } self.assertEquals(result, expected)
def test_invalidate_data(self): e = Entity(self.dbm, entity_type='store', location=['nyc']) e.save() apple_type = DataDictType(self.dbm, name='Apples', slug='apples', primitive_type='number') orange_type = DataDictType(self.dbm, name='Oranges', slug='oranges', primitive_type='number') apple_type.save() orange_type.save() data = e.add_data([('apples', 20, apple_type), ('oranges', 30, orange_type)]) valid_doc = self.dbm._load_document(data) self.assertFalse(valid_doc.void) e.invalidate_data(data) invalid_doc = self.dbm._load_document(data) self.assertTrue(invalid_doc.void)
def test_invalidate_entity(self): e = Entity(self.dbm, entity_type='store', location=['nyc']) e.save() self.assertFalse(e._doc.void) apple_type = DataDictType(self.dbm, name='Apples', slug='apples', primitive_type='number') orange_type = DataDictType(self.dbm, name='Oranges', slug='oranges', primitive_type='number') apple_type.save() orange_type.save() data = [ [('apples', 20, apple_type), ('oranges', 30, orange_type)], [('apples', 10, apple_type), ('oranges', 20, orange_type)] ] data_ids = [] for d in data: id = e.add_data(d) self.assertFalse(self.dbm._load_document(id).void) data_ids.append(id) e.invalidate() self.assertTrue(e._doc.void) for id in data_ids: self.assertTrue(self.dbm._load_document(id).void)
def create_reporter(self): r = Entity(self.manager, entity_type=["reporter"], short_code='ashwin') r.add_data((('mobile_number', '2998288332'), ('name', 'ashwin'))) r.save() return r
def _create_entities_and_data_records(self): ENTITY_TYPE = ["Health_Facility", "Clinic"] self.entity_type = ENTITY_TYPE FEB = datetime.datetime(2011, 02, 01, tzinfo=UTC) MARCH = datetime.datetime(2011, 03, 01, tzinfo=UTC) # Entities for State 1: Maharashtra e = Entity(self.manager, entity_type=ENTITY_TYPE, location=['India', 'MH', 'Pune']) e.save() e.add_data(data=[("beds", 300, self.dd_types['beds']), ("meds", 20, self.dd_types['meds']), ("director", "Dr. A", self.dd_types['director']), ("patients", 10, self.dd_types['patients'])], event_time=FEB) e.add_data(data=[("beds", 500, self.dd_types['beds']), ("meds", 20, self.dd_types['meds']), ("patients", 20, self.dd_types['patients'])], event_time=MARCH) e = Entity(self.manager, entity_type=ENTITY_TYPE, location=['India', 'MH', 'Pune']) e.save() e.add_data(data=[("beds", 100, self.dd_types['beds']), ("meds", 10, self.dd_types['meds']), ("director", "Dr. AA", self.dd_types['director']), ("patients", 50, self.dd_types['patients'])], event_time=FEB) e.add_data(data=[("beds", 200, self.dd_types['beds']), ("meds", 20, self.dd_types['meds']), ("patients", 20, self.dd_types['patients'])], event_time=MARCH) e = Entity(self.manager, entity_type=ENTITY_TYPE, location=['India', 'MH', 'Mumbai']) e.save() e.add_data(data=[("beds", 100, self.dd_types['beds']), ("meds", 10, self.dd_types['meds']), ("director", "Dr. AAA", self.dd_types['director']), ("patients", 50, self.dd_types['patients'])], event_time=FEB) e.add_data(data=[("beds", 200, self.dd_types['beds']), ("meds", 20, self.dd_types['meds']), ("patients", 50, self.dd_types['patients'])], event_time=MARCH) # Entities for State 2: karnataka e = Entity(self.manager, entity_type=ENTITY_TYPE, location=['India', 'Karnataka', 'Bangalore']) e.save() e.add_data(data=[("beds", 100, self.dd_types['beds']), ("meds", 250, self.dd_types['meds']), ("director", "Dr. B1", self.dd_types['director']), ("patients", 50, self.dd_types['patients'])], event_time=FEB) e.add_data(data=[("beds", 200, self.dd_types['beds']), ("meds", 400, self.dd_types['meds']), ("director", "Dr. B2", self.dd_types['director']), ("patients", 20, self.dd_types['patients'])], event_time=MARCH) e = Entity(self.manager, entity_type=ENTITY_TYPE, location=['India', 'Karnataka', 'Hubli']) e.save() e.add_data(data=[("beds", 100, self.dd_types['beds']), ("meds", 250, self.dd_types['meds']), ("director", "Dr. B1", self.dd_types['director']), ("patients", 50, self.dd_types['patients'])], event_time=FEB) e.add_data(data=[("beds", 200, self.dd_types['beds']), ("meds", 400, self.dd_types['meds']), ("director", "Dr. B2", self.dd_types['director']), ("patients", 20, self.dd_types['patients'])], event_time=MARCH) # Entities for State 3: Kerala e = Entity(self.manager, entity_type=ENTITY_TYPE, location=['India', 'Kerala', 'Kochi']) e.save() e.add_data(data=[("beds", 200, self.dd_types['beds']), ("meds", 50, self.dd_types['meds']), ("director", "Dr. C", self.dd_types['director']), ("patients", 12, self.dd_types['patients'])], event_time=MARCH)
def _import_data(self, server, database, max_facilities_to_import=DEFAULT_FACILITIES_TO_IMPORT, max_mdg_to_import=DEFAULT_MDG_TO_IMPORT): from mangrove.datastore.database import DatabaseManager from mangrove.datastore.entity import Entity, get_entities_by_value from mangrove.datastore.datadict import DataDictType, get_datadict_type, create_datadict_type from mangrove.utils import GoogleSpreadsheetsClient from mangrove.utils.google_spreadsheets import get_string, get_number, get_boolean, get_list from mangrove.utils.spreadsheets import CsvReader from mangrove.utils.helpers import slugify from mangrove.georegistry.api import get_feature_by_id import os import datetime import json from pytz import UTC print "Loading 'NIMS Data'..." print "\tServer: %s" % server print "\tDatabase: %s" % database dbm = DatabaseManager(server=server, database=database) user_spreadsheets = GoogleSpreadsheetsClient(settings.GMAIL_USERNAME, settings.GMAIL_PASSWORD) nims_data = user_spreadsheets['NIMS Data Deux'] load_population = True load_other = True load_mdg = True load_health = True load_water = True load_education = True countries = {} states = {} locations = {} num_cgs = 0 datadict_types = {} geo_id_dict = {} cgs_type = create_datadict_type( dbm, slug='cgs', name='CGS', primitive_type='boolean' ) datadict_types['cgs'] = cgs_type.id geo_id_type = create_datadict_type( dbm, slug='geo_id', name='Geographic ID', primitive_type='string' ) datadict_types['geo_id'] = geo_id_type.id name_type = create_datadict_type( dbm, slug='name', name='Name', primitive_type='string' ) datadict_types['name'] = name_type.id mdg_type = create_datadict_type( dbm, slug='mdg', name='MDG', primitive_type='string' ) datadict_types['mdg'] = mdg_type.id country_geo_id = {} for row in nims_data['Nigeria Country ALL']: country_geo_id[row['name']] = row['grid'] state_geo_ids = {} for row in nims_data['Nigeria States ALL']: state_geo_ids[row['name']] = row['grid'] num_rows = 0 print "Importing location entities from 'Nigeria LGAs ALL' worksheet" for row in nims_data['Nigeria LGAs ALL']: country = get_string('country', row) state = get_string('state', row) lga = get_string('lga', row) cgs = get_boolean('cgs', row) geo_id = get_string('geoid', row) lga_gr_id = get_string('grid', row) location = (country, state, lga) if country not in countries: gr_id = country_geo_id[country] feature = get_feature_by_id(gr_id) # geometry = feature['geometry'] centroid = json.loads(feature['properties']['geometry_centroid']) e = Entity(dbm, entity_type=["Location", "Country"], location=[country], centroid=centroid, gr_id=gr_id) locations[(country,)] = e.save() countries[country] = e.id data = [(name_type.slug, country, name_type)] e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC)) num_rows += 1 print "[%s]...(%s) -- %s" % (num_rows, country, e.id)
e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC)) num_rows += 1 print "[%s]...(%s, %s) -- %s" % (num_rows, country, state, e.id) gr_id = lga_gr_id feature = get_feature_by_id(gr_id) # geometry = feature['geometry'] centroid = json.loads(feature['properties']['geometry_centroid']) e = Entity(dbm, entity_type=["Location", "LGA"], location=[country, state, lga], centroid=centroid, gr_id=gr_id) locations[location] = e.save() geo_id_dict[geo_id] = e data = [(name_type.slug, lga, name_type)] e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC)) data = [(geo_id_type.slug, geo_id, geo_id_type)] e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC)) num_rows += 1 print "[%s]...(%s, %s, %s) -- %s" % (num_rows, country, state, lga, e.id) if cgs: num_cgs += 1 e.add_data(data=[(cgs_type.slug, cgs, cgs_type)]) print "Countries (%d)" % len(countries) print "States (%d)" % len(states) print "LGAs (%d) (%d as CGS)" % ((len(locations) - len(countries) - len(states)), num_cgs) print "Total locations (%d)" % len(locations) lga_loaded = []
def CreateReporter(self): reporter = Entity(self.manager, location=["China", "Beijing"], short_code="rpt_bj", id="Reporter001", entity_type=self.reporter_type) reporter.add_data(data=[(MOBILE_NUMBER_FIELD, "1234567890", self.phone_number_type),(NAME_FIELD, "001s", self.first_name_type)]) reporter_id = reporter.save() return reporter_id, reporter
def _import_data(self, server, database, max_facilities_to_import=DEFAULT_FACILITIES_TO_IMPORT, max_mdg_to_import=DEFAULT_MDG_TO_IMPORT): from mangrove.datastore.database import DatabaseManager from mangrove.datastore.entity import Entity, get_entities_by_value from mangrove.datastore.datadict import DataDictType, get_datadict_type, create_datadict_type from mangrove.utils import GoogleSpreadsheetsClient from mangrove.utils.google_spreadsheets import get_string, get_number, get_boolean, get_list from mangrove.utils.spreadsheets import CsvReader from mangrove.utils.helpers import slugify from mangrove.georegistry.api import get_feature_by_id import os import datetime import json from pytz import UTC print "Loading 'NIMS Data'..." print "\tServer: %s" % server print "\tDatabase: %s" % database dbm = DatabaseManager(server=server, database=database) user_spreadsheets = GoogleSpreadsheetsClient(settings.GMAIL_USERNAME, settings.GMAIL_PASSWORD) nims_data = user_spreadsheets['NIMS Data Deux'] load_population = True load_other = True load_mdg = True load_health = True load_water = True load_education = True countries = {} states = {} locations = {} num_cgs = 0 datadict_types = {} geo_id_dict = {} cgs_type = create_datadict_type(dbm, slug='cgs', name='CGS', primitive_type='boolean') datadict_types['cgs'] = cgs_type.id geo_id_type = create_datadict_type(dbm, slug='geo_id', name='Geographic ID', primitive_type='string') datadict_types['geo_id'] = geo_id_type.id name_type = create_datadict_type(dbm, slug='name', name='Name', primitive_type='string') datadict_types['name'] = name_type.id mdg_type = create_datadict_type(dbm, slug='mdg', name='MDG', primitive_type='string') datadict_types['mdg'] = mdg_type.id country_geo_id = {} for row in nims_data['Nigeria Country ALL']: country_geo_id[row['name']] = row['grid'] state_geo_ids = {} for row in nims_data['Nigeria States ALL']: state_geo_ids[row['name']] = row['grid'] num_rows = 0 print "Importing location entities from 'Nigeria LGAs ALL' worksheet" for row in nims_data['Nigeria LGAs ALL']: country = get_string('country', row) state = get_string('state', row) lga = get_string('lga', row) cgs = get_boolean('cgs', row) geo_id = get_string('geoid', row) lga_gr_id = get_string('grid', row) location = (country, state, lga) if country not in countries: gr_id = country_geo_id[country] feature = get_feature_by_id(gr_id) # geometry = feature['geometry'] centroid = json.loads( feature['properties']['geometry_centroid']) e = Entity(dbm, entity_type=["Location", "Country"], location=[country], centroid=centroid, gr_id=gr_id) locations[(country, )] = e.save() countries[country] = e.id data = [(name_type.slug, country, name_type)] e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC)) num_rows += 1 print "[%s]...(%s) -- %s" % (num_rows, country, e.id)
num_rows += 1 print "[%s]...(%s, %s) -- %s" % (num_rows, country, state, e.id) gr_id = lga_gr_id feature = get_feature_by_id(gr_id) # geometry = feature['geometry'] centroid = json.loads(feature['properties']['geometry_centroid']) e = Entity(dbm, entity_type=["Location", "LGA"], location=[country, state, lga], centroid=centroid, gr_id=gr_id) locations[location] = e.save() geo_id_dict[geo_id] = e data = [(name_type.slug, lga, name_type)] e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC)) data = [(geo_id_type.slug, geo_id, geo_id_type)] e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC)) num_rows += 1 print "[%s]...(%s, %s, %s) -- %s" % (num_rows, country, state, lga, e.id) if cgs: num_cgs += 1 e.add_data(data=[(cgs_type.slug, cgs, cgs_type)]) print "Countries (%d)" % len(countries) print "States (%d)" % len(states) print "LGAs (%d) (%d as CGS)" % ( (len(locations) - len(countries) - len(states)), num_cgs)
def _create_entities_and_data_records(self): ENTITY_TYPE = ["Health_Facility", "Clinic"] self.entity_type = ENTITY_TYPE FEB = datetime.datetime(2011, 02, 01, tzinfo=UTC) MARCH = datetime.datetime(2011, 03, 01, tzinfo=UTC) # Entities for State 1: Maharashtra e = Entity(self.manager, entity_type=ENTITY_TYPE, location=['India', 'MH', 'Pune']) e.save() e.add_data(data=[("beds", 300, self.dd_types['beds']), ("meds", 20, self.dd_types['meds']), ("director", "Dr. A", self.dd_types['director']), ("patients", 10, self.dd_types['patients'])], event_time=FEB) e.add_data(data=[("beds", 500, self.dd_types['beds']), ("meds", 20, self.dd_types['meds']), ("patients", 20, self.dd_types['patients'])], event_time=MARCH) e = Entity(self.manager, entity_type=ENTITY_TYPE, location=['India', 'MH', 'Pune']) e.save() e.add_data(data=[("beds", 100, self.dd_types['beds']), ("meds", 10, self.dd_types['meds']), ("director", "Dr. AA", self.dd_types['director']), ("patients", 50, self.dd_types['patients'])], event_time=FEB) e.add_data(data=[("beds", 200, self.dd_types['beds']), ("meds", 20, self.dd_types['meds']), ("patients", 20, self.dd_types['patients'])], event_time=MARCH) e = Entity(self.manager, entity_type=ENTITY_TYPE, location=['India', 'MH', 'Mumbai']) e.save() e.add_data(data=[("beds", 100, self.dd_types['beds']), ("meds", 10, self.dd_types['meds']), ("director", "Dr. AAA", self.dd_types['director']), ("patients", 50, self.dd_types['patients'])], event_time=FEB) e.add_data(data=[("beds", 200, self.dd_types['beds']), ("meds", 20, self.dd_types['meds']), ("patients", 50, self.dd_types['patients'])], event_time=MARCH) # Entities for State 2: karnataka e = Entity(self.manager, entity_type=ENTITY_TYPE, location=['India', 'Karnataka', 'Bangalore']) e.save() e.add_data(data=[("beds", 100, self.dd_types['beds']), ("meds", 250, self.dd_types['meds']), ("director", "Dr. B1", self.dd_types['director']), ("patients", 50, self.dd_types['patients'])], event_time=FEB) e.add_data(data=[("beds", 200, self.dd_types['beds']), ("meds", 400, self.dd_types['meds']), ("director", "Dr. B2", self.dd_types['director']), ("patients", 20, self.dd_types['patients'])], event_time=MARCH) e = Entity(self.manager, entity_type=ENTITY_TYPE, location=['India', 'Karnataka', 'Hubli']) e.save() e.add_data(data=[("beds", 100, self.dd_types['beds']), ("meds", 250, self.dd_types['meds']), ("director", "Dr. B1", self.dd_types['director']), ("patients", 50, self.dd_types['patients'])], event_time=FEB) e.add_data(data=[("beds", 200, self.dd_types['beds']), ("meds", 400, self.dd_types['meds']), ("director", "Dr. B2", self.dd_types['director']), ("patients", 20, self.dd_types['patients'])], event_time=MARCH) # Entities for State 3: Kerala e = Entity(self.manager, entity_type=ENTITY_TYPE, location=['India', 'Kerala', 'Kochi']) e.save() e.add_data(data=[("beds", 200, self.dd_types['beds']), ("meds", 50, self.dd_types['meds']), ("director", "Dr. C", self.dd_types['director']), ("patients", 12, self.dd_types['patients'])], event_time=MARCH)