示例#1
0
def _get_or_create_data_dict(dbm, name, slug, primitive_type, description=None):
    try:
        ddtype = get_datadict_type_by_slug(dbm, slug)
    except DataObjectNotFound:
        ddtype = create_datadict_type(dbm=dbm, name=name, slug=slug,
            primitive_type=primitive_type, description=description)
    return ddtype
示例#2
0
def create_data_dict(dbm, name, slug, primitive_type, description=None):
    try:
        existing = get_datadict_type_by_slug(dbm, slug)
        existing.delete()
    except DataObjectNotFound:
        pass
    return create_datadict_type(dbm, name, slug, primitive_type, description)
def create_data_dict(dbm, name, slug, primitive_type, description=None):
    try:
        existing = get_datadict_type_by_slug(dbm, slug)
        existing.delete()
    except DataObjectNotFound:
        pass
    return create_datadict_type(dbm, name, slug, primitive_type, description)
示例#4
0
def get_or_create_data_dict(dbm, name, slug, primitive_type, description=None):
    try:
        #  Check if is existing
        ddtype = get_datadict_type_by_slug(dbm, slug)
    except DataObjectNotFound:
        #  Create new one
        ddtype = create_datadict_type(dbm=dbm, name=name, slug=slug,
            primitive_type=primitive_type, description=description)
    return ddtype
    def test_should_create_ddtype(self):
        NAME = 'Default Datadict Type'
        SLUG = 'default'
        TYPE = 'string'
        DESC = 'description'

        ddtype = create_datadict_type(self.dbm, name=NAME,
                                      slug=SLUG, primitive_type=TYPE, constraints={}, description=DESC)

        self.dbm._save_document.assert_called_once_with(ddtype._doc)

        self.assertEqual(NAME, ddtype.name)
        self.assertEqual(DESC, ddtype.description)
        self.assertEqual(SLUG, ddtype.slug)
        self.assertEqual(TYPE, ddtype.primitive_type)
示例#6
0
    def test_should_create_load_edit_datadict(self):
        FIRST_NAME_SLUG = 'first_name'

        name_type = create_datadict_type(self.dbm, name='First name', slug=FIRST_NAME_SLUG, primitive_type='string')

        saved_type = get_datadict_type(self.dbm, name_type.id)
        self.assertEqual(name_type.id, saved_type.id)
        self.assertEqual(name_type.slug, saved_type.slug)

        ddtype = get_datadict_type_by_slug(self.dbm, slug=FIRST_NAME_SLUG)

        self.assertEqual(name_type.id, ddtype.id)
        self.assertEqual(name_type.slug, ddtype.slug)

        ddtype.description = "new desc"
        ddtype.save()

        saved = get_datadict_type_by_slug(self.dbm, slug=FIRST_NAME_SLUG)
        self.assertEqual("new desc", saved.description)
示例#7
0
    def test_should_create_load_edit_datadict(self):
        FIRST_NAME_SLUG = 'first_name'

        name_type = create_datadict_type(self.dbm,
                                         name='First name',
                                         slug=FIRST_NAME_SLUG,
                                         primitive_type='string')

        saved_type = get_datadict_type(self.dbm, name_type.id)
        self.assertEqual(name_type.id, saved_type.id)
        self.assertEqual(name_type.slug, saved_type.slug)

        ddtype = get_datadict_type_by_slug(self.dbm, slug=FIRST_NAME_SLUG)

        self.assertEqual(name_type.id, ddtype.id)
        self.assertEqual(name_type.slug, ddtype.slug)

        ddtype.description = "new desc"
        ddtype.save()

        saved = get_datadict_type_by_slug(self.dbm, slug=FIRST_NAME_SLUG)
        self.assertEqual("new desc", saved.description)
示例#8
0
    def _import_data(self, server, database, max_facilities_to_import=DEFAULT_FACILITIES_TO_IMPORT,
                     max_mdg_to_import=DEFAULT_MDG_TO_IMPORT):
        from mangrove.datastore.database import DatabaseManager
        from mangrove.datastore.entity import Entity, get_entities_by_value
        from mangrove.datastore.datadict import DataDictType, get_datadict_type, create_datadict_type
        from mangrove.utils import GoogleSpreadsheetsClient
        from mangrove.utils.google_spreadsheets import get_string, get_number, get_boolean, get_list
        from mangrove.utils.spreadsheets import CsvReader
        from mangrove.utils.helpers import slugify
        from mangrove.georegistry.api import get_feature_by_id
        import os
        import datetime
        import json
        from pytz import UTC

        print "Loading 'NIMS Data'..."

        print "\tServer: %s" % server
        print "\tDatabase: %s" % database

        dbm = DatabaseManager(server=server, database=database)

        user_spreadsheets = GoogleSpreadsheetsClient(settings.GMAIL_USERNAME, settings.GMAIL_PASSWORD)
        nims_data = user_spreadsheets['NIMS Data Deux']

        load_population = True
        load_other = True
        load_mdg = True
        load_health = True
        load_water = True
        load_education = True

        countries = {}
        states = {}
        locations = {}
        num_cgs = 0
        datadict_types = {}
        geo_id_dict = {}

        cgs_type = create_datadict_type(
            dbm,
            slug='cgs',
            name='CGS',
            primitive_type='boolean'
        )
        datadict_types['cgs'] = cgs_type.id

        geo_id_type = create_datadict_type(
            dbm,
            slug='geo_id',
            name='Geographic ID',
            primitive_type='string'
        )
        datadict_types['geo_id'] = geo_id_type.id

        name_type = create_datadict_type(
            dbm,
            slug='name',
            name='Name',
            primitive_type='string'
        )
        datadict_types['name'] = name_type.id

        mdg_type = create_datadict_type(
            dbm,
            slug='mdg',
            name='MDG',
            primitive_type='string'
        )
        datadict_types['mdg'] = mdg_type.id

        country_geo_id = {}
        for row in nims_data['Nigeria Country ALL']:
            country_geo_id[row['name']] = row['grid']
        state_geo_ids = {}
        for row in nims_data['Nigeria States ALL']:
            state_geo_ids[row['name']] = row['grid']

        num_rows = 0
        print "Importing location entities from 'Nigeria LGAs ALL' worksheet"
        for row in nims_data['Nigeria LGAs ALL']:
            country = get_string('country', row)
            state = get_string('state', row)
            lga = get_string('lga', row)
            cgs = get_boolean('cgs', row)
            geo_id = get_string('geoid', row)
            lga_gr_id = get_string('grid', row)
            location = (country, state, lga)
            if country not in countries:
                gr_id = country_geo_id[country]
                feature = get_feature_by_id(gr_id)
#                geometry = feature['geometry']
                centroid = json.loads(feature['properties']['geometry_centroid'])
                e = Entity(dbm,
                           entity_type=["Location", "Country"],
                           location=[country],
                           centroid=centroid,
                           gr_id=gr_id)
                locations[(country,)] = e.save()
                countries[country] = e.id
                data = [(name_type.slug, country, name_type)]
                e.add_data(data, event_time=datetime.datetime(2011, 03, 01, tzinfo=UTC))
                num_rows += 1
                print "[%s]...(%s) -- %s" % (num_rows, country, e.id)
示例#9
0
        lga_loaded = []
        lga_failed = []

        if load_population:
            print "Adding data from 'Population Data' worksheet"
            for row in nims_data['Population Variables']:
                slug = get_string('slug', row)
                name = get_string('name', row)
                primitive_type = get_string('primitivetype', row)
                tags = get_list('tags', row)
                if not slug in datadict_types:
                    dd_type = create_datadict_type(
                        dbm,
                        slug=slug,
                        name=name,
                        primitive_type=primitive_type,
                        tags=tags
                    )
                    datadict_types[slug] = dd_type.id

            for row in nims_data['Population Data']:
                state = get_string('state', row)
                lga = get_string('lga', row)
                location = ("Nigeria", state, lga)
                data = []
                if not state or not lga:
                    continue
                for dd_key in datadict_types.keys():
                    ss_key = dd_key.replace('_', '')
                    point = (dd_key, get_number(ss_key, row), get_datadict_type(dbm, datadict_types[dd_key]))
示例#10
0
    def _import_data(self,
                     server,
                     database,
                     max_facilities_to_import=DEFAULT_FACILITIES_TO_IMPORT,
                     max_mdg_to_import=DEFAULT_MDG_TO_IMPORT):
        from mangrove.datastore.database import DatabaseManager
        from mangrove.datastore.entity import Entity, get_entities_by_value
        from mangrove.datastore.datadict import DataDictType, get_datadict_type, create_datadict_type
        from mangrove.utils import GoogleSpreadsheetsClient
        from mangrove.utils.google_spreadsheets import get_string, get_number, get_boolean, get_list
        from mangrove.utils.spreadsheets import CsvReader
        from mangrove.utils.helpers import slugify
        from mangrove.georegistry.api import get_feature_by_id
        import os
        import datetime
        import json
        from pytz import UTC

        print "Loading 'NIMS Data'..."

        print "\tServer: %s" % server
        print "\tDatabase: %s" % database

        dbm = DatabaseManager(server=server, database=database)

        user_spreadsheets = GoogleSpreadsheetsClient(settings.GMAIL_USERNAME,
                                                     settings.GMAIL_PASSWORD)
        nims_data = user_spreadsheets['NIMS Data Deux']

        load_population = True
        load_other = True
        load_mdg = True
        load_health = True
        load_water = True
        load_education = True

        countries = {}
        states = {}
        locations = {}
        num_cgs = 0
        datadict_types = {}
        geo_id_dict = {}

        cgs_type = create_datadict_type(dbm,
                                        slug='cgs',
                                        name='CGS',
                                        primitive_type='boolean')
        datadict_types['cgs'] = cgs_type.id

        geo_id_type = create_datadict_type(dbm,
                                           slug='geo_id',
                                           name='Geographic ID',
                                           primitive_type='string')
        datadict_types['geo_id'] = geo_id_type.id

        name_type = create_datadict_type(dbm,
                                         slug='name',
                                         name='Name',
                                         primitive_type='string')
        datadict_types['name'] = name_type.id

        mdg_type = create_datadict_type(dbm,
                                        slug='mdg',
                                        name='MDG',
                                        primitive_type='string')
        datadict_types['mdg'] = mdg_type.id

        country_geo_id = {}
        for row in nims_data['Nigeria Country ALL']:
            country_geo_id[row['name']] = row['grid']
        state_geo_ids = {}
        for row in nims_data['Nigeria States ALL']:
            state_geo_ids[row['name']] = row['grid']

        num_rows = 0
        print "Importing location entities from 'Nigeria LGAs ALL' worksheet"
        for row in nims_data['Nigeria LGAs ALL']:
            country = get_string('country', row)
            state = get_string('state', row)
            lga = get_string('lga', row)
            cgs = get_boolean('cgs', row)
            geo_id = get_string('geoid', row)
            lga_gr_id = get_string('grid', row)
            location = (country, state, lga)
            if country not in countries:
                gr_id = country_geo_id[country]
                feature = get_feature_by_id(gr_id)
                #                geometry = feature['geometry']
                centroid = json.loads(
                    feature['properties']['geometry_centroid'])
                e = Entity(dbm,
                           entity_type=["Location", "Country"],
                           location=[country],
                           centroid=centroid,
                           gr_id=gr_id)
                locations[(country, )] = e.save()
                countries[country] = e.id
                data = [(name_type.slug, country, name_type)]
                e.add_data(data,
                           event_time=datetime.datetime(2011,
                                                        03,
                                                        01,
                                                        tzinfo=UTC))
                num_rows += 1
                print "[%s]...(%s) -- %s" % (num_rows, country, e.id)
示例#11
0
        print "Total locations (%d)" % len(locations)

        lga_loaded = []
        lga_failed = []

        if load_population:
            print "Adding data from 'Population Data' worksheet"
            for row in nims_data['Population Variables']:
                slug = get_string('slug', row)
                name = get_string('name', row)
                primitive_type = get_string('primitivetype', row)
                tags = get_list('tags', row)
                if not slug in datadict_types:
                    dd_type = create_datadict_type(
                        dbm,
                        slug=slug,
                        name=name,
                        primitive_type=primitive_type,
                        tags=tags)
                    datadict_types[slug] = dd_type.id

            for row in nims_data['Population Data']:
                state = get_string('state', row)
                lga = get_string('lga', row)
                location = ("Nigeria", state, lga)
                data = []
                if not state or not lga:
                    continue
                for dd_key in datadict_types.keys():
                    ss_key = dd_key.replace('_', '')
                    point = (dd_key, get_number(ss_key, row),
                             get_datadict_type(dbm, datadict_types[dd_key]))