def _import_data(self, server, database, max_facilities_to_import=DEFAULT_FACILITIES_TO_IMPORT, max_mdg_to_import=DEFAULT_MDG_TO_IMPORT): from mangrove.datastore.database import DatabaseManager from mangrove.datastore.entity import Entity, get_entities_by_value from mangrove.datastore.datadict import DataDictType, get_datadict_type, create_datadict_type from mangrove.utils import GoogleSpreadsheetsClient from mangrove.utils.google_spreadsheets import get_string, get_number, get_boolean, get_list from mangrove.utils.spreadsheets import CsvReader from mangrove.utils.helpers import slugify from mangrove.georegistry.api import get_feature_by_id import os import datetime import json from pytz import UTC print "Loading 'NIMS Data'..." print "\tServer: %s" % server print "\tDatabase: %s" % database dbm = DatabaseManager(server=server, database=database) user_spreadsheets = GoogleSpreadsheetsClient(settings.GMAIL_USERNAME, settings.GMAIL_PASSWORD) nims_data = user_spreadsheets['NIMS Data Deux'] load_population = True load_other = True load_mdg = True load_health = True load_water = True load_education = True countries = {} states = {} locations = {} num_cgs = 0 datadict_types = {} geo_id_dict = {} cgs_type = create_datadict_type( dbm, slug='cgs', name='CGS', primitive_type='boolean' ) datadict_types['cgs'] = cgs_type.id geo_id_type = create_datadict_type( dbm, slug='geo_id', name='Geographic ID', primitive_type='string' ) datadict_types['geo_id'] = geo_id_type.id name_type = create_datadict_type( dbm, slug='name', name='Name', primitive_type='string' ) datadict_types['name'] = name_type.id mdg_type = create_datadict_type( dbm, slug='mdg', name='MDG', primitive_type='string' ) datadict_types['mdg'] = mdg_type.id country_geo_id = {} for row in nims_data['Nigeria Country ALL']: country_geo_id[row['name']] = row['grid'] state_geo_ids = {} for row in nims_data['Nigeria States ALL']: state_geo_ids[row['name']] = row['grid'] num_rows = 0 print "Importing location entities from 'Nigeria LGAs ALL' worksheet" for row in nims_data['Nigeria LGAs ALL']: country = get_string('country', row) state = get_string('state', row) lga = get_string('lga', row) cgs = get_boolean('cgs', row) geo_id = get_string('geoid', row) lga_gr_id = get_string('grid', row) location = (country, state, lga) if country not in countries: gr_id = country_geo_id[country] feature = get_feature_by_id(gr_id) # geometry = feature['geometry'] centroid = json.loads(feature['properties']['geometry_centroid']) e = Entity(dbm, entity_type=["Location", "Country"], location=[country], centroid=centroid, gr_id=gr_id) locations[(country,)] = e.save() countries[country] = e.id data = [(name_type.slug, country, name_type)] e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC)) num_rows += 1 print "[%s]...(%s) -- %s" % (num_rows, country, e.id)
feature = get_feature_by_id(gr_id) # geometry = feature['geometry'] centroid = json.loads(feature['properties']['geometry_centroid']) e = Entity(dbm, entity_type=["Location", "State"], location=[country, state], centroid=centroid, gr_id=gr_id) locations[(country, state)] = e.save() states[state] = e.id data = [(name_type.slug, state, name_type)] e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC)) num_rows += 1 print "[%s]...(%s, %s) -- %s" % (num_rows, country, state, e.id) gr_id = lga_gr_id feature = get_feature_by_id(gr_id) # geometry = feature['geometry'] centroid = json.loads(feature['properties']['geometry_centroid']) e = Entity(dbm, entity_type=["Location", "LGA"], location=[country, state, lga], centroid=centroid, gr_id=gr_id) locations[location] = e.save() geo_id_dict[geo_id] = e data = [(name_type.slug, lga, name_type)] e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC)) data = [(geo_id_type.slug, geo_id, geo_id_type)] e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC)) num_rows += 1
def _import_data(self, server, database, max_facilities_to_import=DEFAULT_FACILITIES_TO_IMPORT, max_mdg_to_import=DEFAULT_MDG_TO_IMPORT): from mangrove.datastore.database import DatabaseManager from mangrove.datastore.entity import Entity, get_entities_by_value from mangrove.datastore.datadict import DataDictType, get_datadict_type, create_datadict_type from mangrove.utils import GoogleSpreadsheetsClient from mangrove.utils.google_spreadsheets import get_string, get_number, get_boolean, get_list from mangrove.utils.spreadsheets import CsvReader from mangrove.utils.helpers import slugify from mangrove.georegistry.api import get_feature_by_id import os import datetime import json from pytz import UTC print "Loading 'NIMS Data'..." print "\tServer: %s" % server print "\tDatabase: %s" % database dbm = DatabaseManager(server=server, database=database) user_spreadsheets = GoogleSpreadsheetsClient(settings.GMAIL_USERNAME, settings.GMAIL_PASSWORD) nims_data = user_spreadsheets['NIMS Data Deux'] load_population = True load_other = True load_mdg = True load_health = True load_water = True load_education = True countries = {} states = {} locations = {} num_cgs = 0 datadict_types = {} geo_id_dict = {} cgs_type = create_datadict_type(dbm, slug='cgs', name='CGS', primitive_type='boolean') datadict_types['cgs'] = cgs_type.id geo_id_type = create_datadict_type(dbm, slug='geo_id', name='Geographic ID', primitive_type='string') datadict_types['geo_id'] = geo_id_type.id name_type = create_datadict_type(dbm, slug='name', name='Name', primitive_type='string') datadict_types['name'] = name_type.id mdg_type = create_datadict_type(dbm, slug='mdg', name='MDG', primitive_type='string') datadict_types['mdg'] = mdg_type.id country_geo_id = {} for row in nims_data['Nigeria Country ALL']: country_geo_id[row['name']] = row['grid'] state_geo_ids = {} for row in nims_data['Nigeria States ALL']: state_geo_ids[row['name']] = row['grid'] num_rows = 0 print "Importing location entities from 'Nigeria LGAs ALL' worksheet" for row in nims_data['Nigeria LGAs ALL']: country = get_string('country', row) state = get_string('state', row) lga = get_string('lga', row) cgs = get_boolean('cgs', row) geo_id = get_string('geoid', row) lga_gr_id = get_string('grid', row) location = (country, state, lga) if country not in countries: gr_id = country_geo_id[country] feature = get_feature_by_id(gr_id) # geometry = feature['geometry'] centroid = json.loads( feature['properties']['geometry_centroid']) e = Entity(dbm, entity_type=["Location", "Country"], location=[country], centroid=centroid, gr_id=gr_id) locations[(country, )] = e.save() countries[country] = e.id data = [(name_type.slug, country, name_type)] e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC)) num_rows += 1 print "[%s]...(%s) -- %s" % (num_rows, country, e.id)
class Command(BaseCommand): help = "Loads the NMIS dataset from the 'NIMS Data' Google Doc." args = "<server> <database> | <database>" from optparse import make_option option_list = BaseCommand.option_list + ( make_option("--max_facilities_to_import", type="int", dest="max_facilities_to_import"), make_option( "--max_mdg_to_import", type="int", dest="max_mdg_to_import"), ) def handle(self, *args, **options): from mangrove.utils.types import is_number server = settings.MANGROVE_DATABASES['default']['SERVER'] database = settings.MANGROVE_DATABASES['default']['DATABASE'] if len(args) == 2: server = args[0] database = args[1] elif len(args) == 1: database = args[0] elif len(args) == 0: pass else: raise CommandError( 'Wrong number of arguments. Run \'python manage.py help loadnmisdata\' for usage.' ) max_facilities_to_import = options.get('max_facilities_to_import', 10000) max_mdg_to_import = options.get('max_mdg_to_import', 10000) if not is_number(max_facilities_to_import): max_facilities_to_import = DEFAULT_FACILITIES_TO_IMPORT if not is_number(max_mdg_to_import): max_mdg_to_import = DEFAULT_MDG_TO_IMPORT self._import_data(server, database, max_facilities_to_import, max_mdg_to_import) def _import_data(self, server, database, max_facilities_to_import=DEFAULT_FACILITIES_TO_IMPORT, max_mdg_to_import=DEFAULT_MDG_TO_IMPORT): from mangrove.datastore.database import DatabaseManager from mangrove.datastore.entity import Entity, get_entities_by_value from mangrove.datastore.datadict import DataDictType, get_datadict_type, create_datadict_type from mangrove.utils import GoogleSpreadsheetsClient from mangrove.utils.google_spreadsheets import get_string, get_number, get_boolean, get_list from mangrove.utils.spreadsheets import CsvReader from mangrove.utils.helpers import slugify from mangrove.georegistry.api import get_feature_by_id import os import datetime import json from pytz import UTC print "Loading 'NIMS Data'..." print "\tServer: %s" % server print "\tDatabase: %s" % database dbm = DatabaseManager(server=server, database=database) user_spreadsheets = GoogleSpreadsheetsClient(settings.GMAIL_USERNAME, settings.GMAIL_PASSWORD) nims_data = user_spreadsheets['NIMS Data Deux'] load_population = True load_other = True load_mdg = True load_health = True load_water = True load_education = True countries = {} states = {} locations = {} num_cgs = 0 datadict_types = {} geo_id_dict = {} cgs_type = create_datadict_type(dbm, slug='cgs', name='CGS', primitive_type='boolean') datadict_types['cgs'] = cgs_type.id geo_id_type = create_datadict_type(dbm, slug='geo_id', name='Geographic ID', primitive_type='string') datadict_types['geo_id'] = geo_id_type.id name_type = create_datadict_type(dbm, slug='name', name='Name', primitive_type='string') datadict_types['name'] = name_type.id mdg_type = create_datadict_type(dbm, slug='mdg', name='MDG', primitive_type='string') datadict_types['mdg'] = mdg_type.id country_geo_id = {} for row in nims_data['Nigeria Country ALL']: country_geo_id[row['name']] = row['grid'] state_geo_ids = {} for row in nims_data['Nigeria States ALL']: state_geo_ids[row['name']] = row['grid'] num_rows = 0 print "Importing location entities from 'Nigeria LGAs ALL' worksheet" for row in nims_data['Nigeria LGAs ALL']: country = get_string('country', row) state = get_string('state', row) lga = get_string('lga', row) cgs = get_boolean('cgs', row) geo_id = get_string('geoid', row) lga_gr_id = get_string('grid', row) location = (country, state, lga) if country not in countries: gr_id = country_geo_id[country] feature = get_feature_by_id(gr_id) # geometry = feature['geometry'] centroid = json.loads( feature['properties']['geometry_centroid']) e = Entity(dbm, entity_type=["Location", "Country"], location=[country], centroid=centroid, gr_id=gr_id) locations[(country, )] = e.save() countries[country] = e.id data = [(name_type.slug, country, name_type)] e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC)) num_rows += 1 print "[%s]...(%s) -- %s" % (num_rows, country, e.id) if state not in states: gr_id = state_geo_ids[state] feature = get_feature_by_id(gr_id) # geometry = feature['geometry'] centroid = json.loads( feature['properties']['geometry_centroid']) e = Entity(dbm, entity_type=["Location", "State"], location=[country, state], centroid=centroid, gr_id=gr_id) locations[(country, state)] = e.save() states[state] = e.id data = [(name_type.slug, state, name_type)] e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC))
location=[country, state], centroid=centroid, gr_id=gr_id) locations[(country, state)] = e.save() states[state] = e.id data = [(name_type.slug, state, name_type)] e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC)) num_rows += 1 print "[%s]...(%s, %s) -- %s" % (num_rows, country, state, e.id) gr_id = lga_gr_id feature = get_feature_by_id(gr_id) # geometry = feature['geometry'] centroid = json.loads(feature['properties']['geometry_centroid']) e = Entity(dbm, entity_type=["Location", "LGA"], location=[country, state, lga], centroid=centroid, gr_id=gr_id) locations[location] = e.save() geo_id_dict[geo_id] = e data = [(name_type.slug, lga, name_type)] e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC)) data = [(geo_id_type.slug, geo_id, geo_id_type)] e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC))