def _get_or_create_data_dict(dbm, name, slug, primitive_type, description=None): try: ddtype = get_datadict_type_by_slug(dbm, slug) except DataObjectNotFound: ddtype = create_datadict_type(dbm=dbm, name=name, slug=slug, primitive_type=primitive_type, description=description) return ddtype
def create_data_dict(dbm, name, slug, primitive_type, description=None): try: existing = get_datadict_type_by_slug(dbm, slug) existing.delete() except DataObjectNotFound: pass return create_datadict_type(dbm, name, slug, primitive_type, description)
def get_or_create_data_dict(dbm, name, slug, primitive_type, description=None): try: # Check if is existing ddtype = get_datadict_type_by_slug(dbm, slug) except DataObjectNotFound: # Create new one ddtype = create_datadict_type(dbm=dbm, name=name, slug=slug, primitive_type=primitive_type, description=description) return ddtype
def test_should_create_ddtype(self): NAME = 'Default Datadict Type' SLUG = 'default' TYPE = 'string' DESC = 'description' ddtype = create_datadict_type(self.dbm, name=NAME, slug=SLUG, primitive_type=TYPE, constraints={}, description=DESC) self.dbm._save_document.assert_called_once_with(ddtype._doc) self.assertEqual(NAME, ddtype.name) self.assertEqual(DESC, ddtype.description) self.assertEqual(SLUG, ddtype.slug) self.assertEqual(TYPE, ddtype.primitive_type)
def test_should_create_load_edit_datadict(self): FIRST_NAME_SLUG = 'first_name' name_type = create_datadict_type(self.dbm, name='First name', slug=FIRST_NAME_SLUG, primitive_type='string') saved_type = get_datadict_type(self.dbm, name_type.id) self.assertEqual(name_type.id, saved_type.id) self.assertEqual(name_type.slug, saved_type.slug) ddtype = get_datadict_type_by_slug(self.dbm, slug=FIRST_NAME_SLUG) self.assertEqual(name_type.id, ddtype.id) self.assertEqual(name_type.slug, ddtype.slug) ddtype.description = "new desc" ddtype.save() saved = get_datadict_type_by_slug(self.dbm, slug=FIRST_NAME_SLUG) self.assertEqual("new desc", saved.description)
def _import_data(self, server, database, max_facilities_to_import=DEFAULT_FACILITIES_TO_IMPORT, max_mdg_to_import=DEFAULT_MDG_TO_IMPORT): from mangrove.datastore.database import DatabaseManager from mangrove.datastore.entity import Entity, get_entities_by_value from mangrove.datastore.datadict import DataDictType, get_datadict_type, create_datadict_type from mangrove.utils import GoogleSpreadsheetsClient from mangrove.utils.google_spreadsheets import get_string, get_number, get_boolean, get_list from mangrove.utils.spreadsheets import CsvReader from mangrove.utils.helpers import slugify from mangrove.georegistry.api import get_feature_by_id import os import datetime import json from pytz import UTC print "Loading 'NIMS Data'..." print "\tServer: %s" % server print "\tDatabase: %s" % database dbm = DatabaseManager(server=server, database=database) user_spreadsheets = GoogleSpreadsheetsClient(settings.GMAIL_USERNAME, settings.GMAIL_PASSWORD) nims_data = user_spreadsheets['NIMS Data Deux'] load_population = True load_other = True load_mdg = True load_health = True load_water = True load_education = True countries = {} states = {} locations = {} num_cgs = 0 datadict_types = {} geo_id_dict = {} cgs_type = create_datadict_type( dbm, slug='cgs', name='CGS', primitive_type='boolean' ) datadict_types['cgs'] = cgs_type.id geo_id_type = create_datadict_type( dbm, slug='geo_id', name='Geographic ID', primitive_type='string' ) datadict_types['geo_id'] = geo_id_type.id name_type = create_datadict_type( dbm, slug='name', name='Name', primitive_type='string' ) datadict_types['name'] = name_type.id mdg_type = create_datadict_type( dbm, slug='mdg', name='MDG', primitive_type='string' ) datadict_types['mdg'] = mdg_type.id country_geo_id = {} for row in nims_data['Nigeria Country ALL']: country_geo_id[row['name']] = row['grid'] state_geo_ids = {} for row in nims_data['Nigeria States ALL']: state_geo_ids[row['name']] = row['grid'] num_rows = 0 print "Importing location entities from 'Nigeria LGAs ALL' worksheet" for row in nims_data['Nigeria LGAs ALL']: country = get_string('country', row) state = get_string('state', row) lga = get_string('lga', row) cgs = get_boolean('cgs', row) geo_id = get_string('geoid', row) lga_gr_id = get_string('grid', row) location = (country, state, lga) if country not in countries: gr_id = country_geo_id[country] feature = get_feature_by_id(gr_id) # geometry = feature['geometry'] centroid = json.loads(feature['properties']['geometry_centroid']) e = Entity(dbm, entity_type=["Location", "Country"], location=[country], centroid=centroid, gr_id=gr_id) locations[(country,)] = e.save() countries[country] = e.id data = [(name_type.slug, country, name_type)] e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC)) num_rows += 1 print "[%s]...(%s) -- %s" % (num_rows, country, e.id)
lga_loaded = [] lga_failed = [] if load_population: print "Adding data from 'Population Data' worksheet" for row in nims_data['Population Variables']: slug = get_string('slug', row) name = get_string('name', row) primitive_type = get_string('primitivetype', row) tags = get_list('tags', row) if not slug in datadict_types: dd_type = create_datadict_type( dbm, slug=slug, name=name, primitive_type=primitive_type, tags=tags ) datadict_types[slug] = dd_type.id for row in nims_data['Population Data']: state = get_string('state', row) lga = get_string('lga', row) location = ("Nigeria", state, lga) data = [] if not state or not lga: continue for dd_key in datadict_types.keys(): ss_key = dd_key.replace('_', '') point = (dd_key, get_number(ss_key, row), get_datadict_type(dbm, datadict_types[dd_key]))
def _import_data(self, server, database, max_facilities_to_import=DEFAULT_FACILITIES_TO_IMPORT, max_mdg_to_import=DEFAULT_MDG_TO_IMPORT): from mangrove.datastore.database import DatabaseManager from mangrove.datastore.entity import Entity, get_entities_by_value from mangrove.datastore.datadict import DataDictType, get_datadict_type, create_datadict_type from mangrove.utils import GoogleSpreadsheetsClient from mangrove.utils.google_spreadsheets import get_string, get_number, get_boolean, get_list from mangrove.utils.spreadsheets import CsvReader from mangrove.utils.helpers import slugify from mangrove.georegistry.api import get_feature_by_id import os import datetime import json from pytz import UTC print "Loading 'NIMS Data'..." print "\tServer: %s" % server print "\tDatabase: %s" % database dbm = DatabaseManager(server=server, database=database) user_spreadsheets = GoogleSpreadsheetsClient(settings.GMAIL_USERNAME, settings.GMAIL_PASSWORD) nims_data = user_spreadsheets['NIMS Data Deux'] load_population = True load_other = True load_mdg = True load_health = True load_water = True load_education = True countries = {} states = {} locations = {} num_cgs = 0 datadict_types = {} geo_id_dict = {} cgs_type = create_datadict_type(dbm, slug='cgs', name='CGS', primitive_type='boolean') datadict_types['cgs'] = cgs_type.id geo_id_type = create_datadict_type(dbm, slug='geo_id', name='Geographic ID', primitive_type='string') datadict_types['geo_id'] = geo_id_type.id name_type = create_datadict_type(dbm, slug='name', name='Name', primitive_type='string') datadict_types['name'] = name_type.id mdg_type = create_datadict_type(dbm, slug='mdg', name='MDG', primitive_type='string') datadict_types['mdg'] = mdg_type.id country_geo_id = {} for row in nims_data['Nigeria Country ALL']: country_geo_id[row['name']] = row['grid'] state_geo_ids = {} for row in nims_data['Nigeria States ALL']: state_geo_ids[row['name']] = row['grid'] num_rows = 0 print "Importing location entities from 'Nigeria LGAs ALL' worksheet" for row in nims_data['Nigeria LGAs ALL']: country = get_string('country', row) state = get_string('state', row) lga = get_string('lga', row) cgs = get_boolean('cgs', row) geo_id = get_string('geoid', row) lga_gr_id = get_string('grid', row) location = (country, state, lga) if country not in countries: gr_id = country_geo_id[country] feature = get_feature_by_id(gr_id) # geometry = feature['geometry'] centroid = json.loads( feature['properties']['geometry_centroid']) e = Entity(dbm, entity_type=["Location", "Country"], location=[country], centroid=centroid, gr_id=gr_id) locations[(country, )] = e.save() countries[country] = e.id data = [(name_type.slug, country, name_type)] e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC)) num_rows += 1 print "[%s]...(%s) -- %s" % (num_rows, country, e.id)
print "Total locations (%d)" % len(locations) lga_loaded = [] lga_failed = [] if load_population: print "Adding data from 'Population Data' worksheet" for row in nims_data['Population Variables']: slug = get_string('slug', row) name = get_string('name', row) primitive_type = get_string('primitivetype', row) tags = get_list('tags', row) if not slug in datadict_types: dd_type = create_datadict_type( dbm, slug=slug, name=name, primitive_type=primitive_type, tags=tags) datadict_types[slug] = dd_type.id for row in nims_data['Population Data']: state = get_string('state', row) lga = get_string('lga', row) location = ("Nigeria", state, lga) data = [] if not state or not lga: continue for dd_key in datadict_types.keys(): ss_key = dd_key.replace('_', '') point = (dd_key, get_number(ss_key, row), get_datadict_type(dbm, datadict_types[dd_key]))