Exemple #1
0
    def set_up(self):

        self.d = date.today()
        st = self.d - timedelta(days=1)
        ed = self.d + timedelta(days=1)

        self.u = User.objects.create_user('polio','*****@*****.**', 'polio')
        o = Office.objects.create(name='NGA')
        lt = LocationType.objects.create(name='country',admin_level=0)
        ct = CampaignType.objects.create(name='NID')
        self.ind_0 = Indicator.objects.create(name='number of VDPV cases',short_name='V')
        self.ind_1 = Indicator.objects.create(name='number of WPV cases',short_name='W')
        ind_tag = IndicatorTag.objects.create(tag_name='Polio')
        self.tpl = Location.objects.create(name='NGA',location_code='NGA',\
            office_id = o.id,location_type_id = lt.id)
        self.doc = Document.objects.create(
            doc_title = 'test',
            created_by_id = self.u.id,
            guid = 'test')

        ### SET UP CAMPAIGN DEFINITION ###

        self.c = Campaign.objects.create(
            office_id = o.id,\
            top_lvl_location_id = self.tpl.id,
            top_lvl_indicator_tag_id = ind_tag.id,
            campaign_type_id = ct.id,
            name = 'test',\
            start_date = st,\
            end_date = ed,\
        )

        ltc = LocationTreeCache()
        ltc.main()
    def setUp(self):
        super(CampaignDataPointResourceTest, self).setUp()

        # Create a user.
        self.username = '******'
        self.password = '******'
        self.user = User.objects.create_user(self.username,\
                                        '*****@*****.**', self.password)

        self.get_credentials()
        self.ts = TestSetupHelpers()

        ## create a metadata and data for us to use to test ##
        self.create_metadata()

        self.country_lt = LocationType.objects\
            .create(name='Country',admin_level = 0)
        self.region_lt = LocationType.objects\
            .create(name='Region',admin_level = 1)
        self.province_lt = LocationType.objects\
            .create(name='Province',admin_level = 2)

        self.top_lvl_location = Location.objects.get(name='Nigeria')

        ltc = LocationTreeCache()
        ltc.main()

        LocationPermission.objects.create(user_id = self.user.id,\
            top_lvl_location_id = self.top_lvl_location.id)
Exemple #3
0
    def setUp(self):
        super(DataPointResourceTest, self).setUp()

        # Create a user.
        self.username = '******'
        self.password = '******'
        self.user = User.objects.create_user(self.username,\
                                        '*****@*****.**', self.password)

        self.lt = LocationType.objects.create(name='Country',admin_level = 0)
        self.o = Office.objects.create(name = 'Earth')

        self.top_lvl_location = Location.objects.create(
                name = 'Nigeria',
                location_code = 'Nigeria',
                location_type_id = self.lt.id,
                office_id = self.o.id,
            )

        ltc = LocationTreeCache()
        ltc.main()

        LocationPermission.objects.create(user_id = self.user.id,\
            top_lvl_location_id = self.top_lvl_location.id)

        self.get_credentials()
    def setUp(self):

        ## instantiate the test client and all other methods ##
        super(LocationResourceTest, self).setUp()

        # Create a user.
        self.username = '******'
        self.password = '******'
        self.user = User.objects\
            .create_user(self.username, '*****@*****.**', self.password)
        self.lt = LocationType.objects.create(name='test', admin_level=0)

        self.not_allowed_to_see_location = Location.objects.create(
            name='Somalia',
            location_code='Somalia',
            location_type_id=self.lt.id)
        self.top_lvl_location = Location.objects.create(
            name='Nigeria',
            location_code='Nigeria',
            location_type_id=self.lt.id)

        self.sub_location = Location.objects.create(
            name='Kano',
            location_code='Kano',
            location_type_id=self.lt.id,
            parent_location_id=self.top_lvl_location.id)

        ### set the user permission ###
        LocationPermission.objects.create(
            user_id=self.user.id, top_lvl_location_id=self.top_lvl_location.id)

        self.get_credentials()

        ltr = LocationTreeCache()
        ltr.main()
    def setUp(self):
        super(DocTransformResourceTest, self).setUp()
        self.ts = TestSetupHelpers()
        self.ts.load_some_metadata()
        ltr = LocationTreeCache()
        ltr.main()

        self.mapped_location_id = self.ts.locations[0].id
        self.mapped_location_id_2 = self.ts.locations[1].id

        self.loc_map = SourceObjectMap.objects.create(
            source_object_code = 'AF001039003000000000',
            content_type = 'location',
            mapped_by_id = self.ts.user.id,
            master_object_id = self.mapped_location_id
        )

        source_campaign_string = '2016 March NID OPV'
        self.mapped_campaign_id = self.ts.campaigns[0].id
        self.campaign_map = SourceObjectMap.objects.create(
            source_object_code = source_campaign_string,
            content_type = 'campaign',
            mapped_by_id = self.ts.user.id,
            master_object_id = self.mapped_campaign_id
        )
        self.mapped_indicator_with_data = self.ts.locations[2].id
        self.indicator_map = SourceObjectMap.objects.create(
            source_object_code = 'Percent missed due to other reasons',
            content_type = 'indicator',
            mapped_by_id = self.ts.user.id,
            master_object_id = self.mapped_indicator_with_data
        )
Exemple #6
0
    def set_up(self):

        self.d = date.today()
        st = self.d - timedelta(days=1)
        ed = self.d + timedelta(days=1)

        self.u = User.objects.create_user(
            'polio', '*****@*****.**', 'polio')
        lt = LocationType.objects.create(name='country', admin_level=0)
        ct = CampaignType.objects.create(name='NID')
        self.ind_0 = Indicator.objects.create(
            name='number of VDPV cases', short_name='V')
        self.ind_1 = Indicator.objects.create(
            name='number of WPV cases', short_name='W')
        ind_tag = IndicatorTag.objects.create(tag_name='Polio')
        self.tpl = Location.objects.create(name='NGA', location_code='NGA',
                                           location_type_id=lt.id)
        self.doc = Document.objects.create(
            doc_title='test',
            created_by_id=self.u.id,
            guid='test')

        ### SET UP CAMPAIGN DEFINITION ###

        self.c = Campaign.objects.create(
            campaign_type_id=ct.id,
            name='test',
            start_date=st,
            end_date=ed,
        )

        ltc = LocationTreeCache()
        ltc.main()
def process_meta_data():

    xl = pd.ExcelFile('migration_data/initial_data.xlsx')
    all_sheets = xl.sheet_names

    rhizome_app = get_app('rhizome')
    auth_app = get_app('auth')

    models_to_process = {}

    all_models = get_models(rhizome_app) + get_models(auth_app)

    for model in all_models:
        ## iterate through the models in the rhizome app and create a lookup
        ## for {'sheet_name': Model} .. for instance -> {'indicator': Indicator}

        if model._meta.db_table in all_sheets:
            models_to_process[model._meta.db_table] = model

    for sheet in all_sheets:
        ## if the sheet has a cooresponding model, create a data frame out of
        ## the sheet anf bulk insert the data using the model_df_to_data fn

        try:
            model = models_to_process[sheet]
            print 'processing sheet ' + sheet
            model_df = xl.parse(sheet)
            model_ids = model_df_to_data(model_df, model)
        except KeyError:
            pass

    ## once the locations are all created we need to ##
    ## cache them in the locaiton_tree table ##
    ltc = LocationTreeCache()
    ltc.main()
    def setUp(self):

        ## instantiate the test client and all other methods ##
        super(CampaignResourceTest, self).setUp()

        self.ts = TestSetupHelpers()
        self.lt = self.ts.create_arbitrary_location_type()
        self.o = self.ts.create_arbitrary_office()
        self.not_allowed_to_see_location = self.ts.create_arbitrary_location(
            self.lt.id,
            self.o.id)

        self.top_lvl_location = self.ts.create_arbitrary_location(
            self.lt.id,
            self.o.id,
            location_code ='Nigeria',
            location_name='Nigeria')

        self.sub_location = self.ts.create_arbitrary_location(
            self.lt.id,
            self.o.id,
            location_name = 'Kano',
            location_code = 'Kano',
            parent_location_id = self.top_lvl_location.id)

        self.it = IndicatorTag.objects.create(tag_name='Polio')

        self.ct = CampaignType.objects.create(name='NID')
        self.can_see_campaign = self.ts.create_arbitrary_campaign(
            office_id = self.ts.create_arbitrary_office(name='test1').id,
            campaign_type_id = self.ct.id,
            location_id = self.top_lvl_location.id,
            indicator_tag_id = self.it.id,
            name="can_see"
        )

        self.can_see_campaign_2 = self.ts.create_arbitrary_campaign(
            office_id = self.ts.create_arbitrary_office(name='test2').id,
            campaign_type_id = self.ct.id,
            location_id = self.top_lvl_location.id,
            indicator_tag_id = self.it.id,
            name="can_see2"
        )


        self.can_not_see_campaign = self.ts.create_arbitrary_campaign(
            office_id = self.o.id,
            campaign_type_id = self.ct.id,
            location_id = self.not_allowed_to_see_location.id,
            indicator_tag_id = self.it.id,
        )

        ### set the user permission ###
        LocationPermission.objects.create(user_id = self.ts.user.id,\
            top_lvl_location_id = self.top_lvl_location.id)

        self.ts.get_credentials(self)

        ltr = LocationTreeCache()
        ltr.main()
    def setUp(self):
        super(DataPointResourceTest, self).setUp()

        # Create a user.
        self.username = '******'
        self.password = '******'
        self.user = User.objects.create_user(self.username,\
                                        '*****@*****.**', self.password)

        self.lt = LocationType.objects.create(name='Country',admin_level = 0)
        self.distr, created = \
            LocationType.objects.get_or_create(name='District',admin_level = 3)
        self.prov, created = \
            LocationType.objects.get_or_create(name='Province',admin_level = 2)
        self.region, created = \
            LocationType.objects.get_or_create(name='Region', admin_level = 1)
        self.o = Office.objects.create(name = 'Earth')

        self.top_lvl_location = Location.objects.create(
                name = 'Nigeria',
                location_code = 'Nigeria',
                id=1234,
                location_type_id = self.lt.id,
                office_id = self.o.id,
            )

        ltc = LocationTreeCache()
        ltc.main()

        LocationPermission.objects.create(user_id = self.user.id,\
            top_lvl_location_id = self.top_lvl_location.id)

        self.get_credentials()
def process_meta_data():

    xl = pd.ExcelFile('migration_data/initial_data.xlsx')
    all_sheets = xl.sheet_names

    rhizome_app = get_app('rhizome')
    auth_app = get_app('auth')

    models_to_process = {}

    all_models = get_models(rhizome_app) + get_models(auth_app)

    for model in all_models:
        ## iterate through the models in the rhizome app and create a lookup
        ## for {'sheet_name': Model} .. for instance -> {'indicator': Indicator}

        if model._meta.db_table in all_sheets:
            models_to_process[model._meta.db_table] = model

    for sheet in all_sheets:
        ## if the sheet has a cooresponding model, create a data frame out of
        ## the sheet anf bulk insert the data using the model_df_to_data fn

        try:
            model = models_to_process[sheet]
            print 'processing sheet ' + sheet
            model_df = xl.parse(sheet)
            model_ids = model_df_to_data(model_df,model)
        except KeyError:
            pass

    ## once the locations are all created we need to ##
    ## cache them in the locaiton_tree table ##
    ltc = LocationTreeCache()
    ltc.main()
    def create_metadata(self):
        '''
        Creating the Indicator, location, Campaign, meta data needed for the
        system to aggregate / caclulate.
        '''

        top_lvl_tag = IndicatorTag.objects.create(id=1, tag_name='Polio')

        campaign_df = read_csv('rhizome/tests/_data/campaigns.csv')
        campaign_df['start_date'] = to_datetime(campaign_df['start_date'])
        campaign_df['end_date'] = to_datetime(campaign_df['end_date'])

        location_df = read_csv('rhizome/tests/_data/locations.csv')
        indicator_df = read_csv('rhizome/tests/_data/indicators.csv')

        campaign_type = CampaignType.objects.create(id=1, name="test")

        locations = self.model_df_to_data(location_df, Location)
        campaigns = self.model_df_to_data(campaign_df, Campaign)
        indicators = self.model_df_to_data(indicator_df, Indicator)

        self.user_id = User.objects.create_user(
            'test', '*****@*****.**', 'test').id
        self.mapped_location_id = locations[0].id

        loc_map = SourceObjectMap.objects.create(
            source_object_code='AF001039003000000000',
            content_type='location',
            mapped_by_id=self.user_id,
            master_object_id=self.mapped_location_id
        )

        source_campaign_string = '2016 March NID OPV'
        self.mapped_campaign_id = campaigns[0].id
        campaign_map = SourceObjectMap.objects.create(
            source_object_code=source_campaign_string,
            content_type='campaign',
            mapped_by_id=self.user_id,
            master_object_id=self.mapped_campaign_id
        )
        self.mapped_indicator_id_0 = indicators[0].id
        indicator_map = SourceObjectMap.objects.create(
            source_object_code='Percent missed children_PCA',
            content_type='indicator',
            mapped_by_id=self.user_id,
            master_object_id=self.mapped_indicator_id_0
        )

        self.mapped_indicator_with_data = locations[2].id
        indicator_map = SourceObjectMap.objects.create(
            source_object_code='Percent missed due to other reasons',
            content_type='indicator',
            mapped_by_id=self.user_id,
            master_object_id=self.mapped_indicator_with_data
        )
        ## make sure that the location tree is updated ##

        ltc = LocationTreeCache()
        ltc.main()
Exemple #12
0
    def setUp(self):

        ## instantiate the test client and all other methods ##
        super(CampaignResourceTest, self).setUp()

        self.ts = TestSetupHelpers()
        self.lt = self.ts.create_arbitrary_location_type()
        self.o = self.ts.create_arbitrary_office()
        self.not_allowed_to_see_location = self.ts.create_arbitrary_location(
            self.lt.id, self.o.id)

        self.top_lvl_location = self.ts.create_arbitrary_location(
            self.lt.id,
            self.o.id,
            location_code='Nigeria',
            location_name='Nigeria')

        self.sub_location = self.ts.create_arbitrary_location(
            self.lt.id,
            self.o.id,
            location_name='Kano',
            location_code='Kano',
            parent_location_id=self.top_lvl_location.id)

        self.it = IndicatorTag.objects.create(tag_name='Polio')

        self.ct = CampaignType.objects.create(name='NID')
        self.can_see_campaign = self.ts.create_arbitrary_campaign(
            office_id=self.ts.create_arbitrary_office(name='test1').id,
            campaign_type_id=self.ct.id,
            location_id=self.top_lvl_location.id,
            indicator_tag_id=self.it.id,
            name="can_see")

        self.can_see_campaign_2 = self.ts.create_arbitrary_campaign(
            office_id=self.ts.create_arbitrary_office(name='test2').id,
            campaign_type_id=self.ct.id,
            location_id=self.top_lvl_location.id,
            indicator_tag_id=self.it.id,
            name="can_see2")

        self.can_not_see_campaign = self.ts.create_arbitrary_campaign(
            office_id=self.o.id,
            campaign_type_id=self.ct.id,
            location_id=self.not_allowed_to_see_location.id,
            indicator_tag_id=self.it.id,
        )

        ### set the user permission ###
        LocationPermission.objects.create(user_id = self.ts.user.id,\
            top_lvl_location_id = self.top_lvl_location.id)

        self.ts.get_credentials(self)

        ltr = LocationTreeCache()
        ltr.main()
Exemple #13
0
    def create_metadata(self):
        '''
        Creating the Indicator, location, Campaign, meta data needed for the
        system to aggregate / caclulate.
        '''

        top_lvl_tag = IndicatorTag.objects.create(id=1, tag_name='Polio')

        campaign_df = read_csv('rhizome/tests/_data/campaigns.csv')
        campaign_df['start_date'] = to_datetime(campaign_df['start_date'])
        campaign_df['end_date'] = to_datetime(campaign_df['end_date'])

        location_df = read_csv('rhizome/tests/_data/locations.csv')
        indicator_df = read_csv('rhizome/tests/_data/indicators.csv')

        campaign_type = CampaignType.objects.create(id=1, name="test")

        locations = self.model_df_to_data(location_df, Location)
        campaigns = self.model_df_to_data(campaign_df, Campaign)
        indicators = self.model_df_to_data(indicator_df, Indicator)
        self.user_id = User.objects.create_user(
            'test', '*****@*****.**', 'test').id
        self.mapped_location_id = locations[0].id
        loc_map = SourceObjectMap.objects.create(
            source_object_code='AF001039003000000000',
            content_type='location',
            mapped_by_id=self.user_id,
            master_object_id=self.mapped_location_id
        )

        source_campaign_string = '2016 March NID OPV'
        self.mapped_campaign_id = campaigns[0].id
        campaign_map = SourceObjectMap.objects.create(
            source_object_code=source_campaign_string,
            content_type='campaign',
            mapped_by_id=self.user_id,
            master_object_id=self.mapped_campaign_id
        )
        self.mapped_indicator_id_0 = indicators[0].id
        indicator_map = SourceObjectMap.objects.create(
            source_object_code='Percent missed children_PCA',
            content_type='indicator',
            mapped_by_id=self.user_id,
            master_object_id=self.mapped_indicator_id_0
        )

        self.mapped_indicator_with_data = locations[2].id
        indicator_map = SourceObjectMap.objects.create(
            source_object_code='Percent missed due to other reasons',
            content_type='indicator',
            mapped_by_id=self.user_id,
            master_object_id=self.mapped_indicator_with_data
        )
        ltc = LocationTreeCache()
        ltc.main()
    def setUp(self):
        super(DataPointResourceTest, self).setUp()

        # Create a user.
        self.username = '******'
        self.password = '******'
        self.user = User.objects.create_user(self.username,\
                                        '*****@*****.**', self.password)

        self.lt = LocationType.objects.create(name='Country',admin_level = 0)
        self.province_lt = LocationType.objects.create(name='Province'\
            ,admin_level = 1)
        self.district_lt = LocationType.objects.create(name='District'\
            ,admin_level = 2)


        self.o = Office.objects.create(name = 'Earth')
        self.ind = Indicator.objects.create(
            name = 'Polio Cases',
            short_name = 'Polio Cases',
            data_format = 'date_int'
        )

        self.top_lvl_location = Location.objects.create(
                name = 'Afghanistan',
                location_code = 'Afghanistan',
                id=1234,
                location_type_id = self.lt.id,
                office_id = self.o.id,
            )
        self.some_province = Location.objects.create(
                name = 'Province',
                location_code = 'Province',
                id=432,
                parent_location_id = self.top_lvl_location.id,
                location_type_id = self.province_lt.id,
                office_id = self.o.id,
            )
        self.some_district = Location.objects.create(
                name = 'Achin',
                location_code = 'Achin',
                id=4321,
                parent_location_id = self.some_province.id,
                location_type_id = self.district_lt.id,
                office_id = self.o.id,
            )

        ltc = LocationTreeCache()
        ltc.main()

        LocationPermission.objects.create(user_id = self.user.id,\
            top_lvl_location_id = self.top_lvl_location.id)

        self.get_credentials()
        self.create_polio_cases()
Exemple #15
0
    def set_up(self):

        data_df = read_csv('rhizome/tests/_data/calc_data.csv')
        self.create_metadata()
        self.user = User.objects.get(username="******")

        self.test_df = data_df[data_df['is_raw'] == 1]
        self.target_df = data_df[data_df['is_raw'] == 0]

        ltr = LocationTreeCache()
        ltr.main()
Exemple #16
0
    def create_metadata(self):
        """
        Creating the Indicator, location, Campaign, meta data needed for the
        system to aggregate / caclulate.
        """

        top_lvl_tag = IndicatorTag.objects.create(id=1, tag_name="Polio")

        campaign_df = read_csv("rhizome/tests/_data/campaigns.csv")
        campaign_df["start_date"] = to_datetime(campaign_df["start_date"])
        campaign_df["end_date"] = to_datetime(campaign_df["end_date"])

        location_df = read_csv("rhizome/tests/_data/locations.csv")
        indicator_df = read_csv("rhizome/tests/_data/indicators.csv")

        campaign_type = CampaignType.objects.create(id=1, name="test")

        locations = self.model_df_to_data(location_df, Location)
        campaigns = self.model_df_to_data(campaign_df, Campaign)
        indicators = self.model_df_to_data(indicator_df, Indicator)
        self.user_id = User.objects.create_user("test", "*****@*****.**", "test").id
        self.mapped_location_id = locations[0].id
        loc_map = SourceObjectMap.objects.create(
            source_object_code="AF001039003000000000",
            content_type="location",
            mapped_by_id=self.user_id,
            master_object_id=self.mapped_location_id,
        )

        source_campaign_string = "2016 March NID OPV"
        self.mapped_campaign_id = campaigns[0].id
        campaign_map = SourceObjectMap.objects.create(
            source_object_code=source_campaign_string,
            content_type="campaign",
            mapped_by_id=self.user_id,
            master_object_id=self.mapped_campaign_id,
        )
        self.mapped_indicator_id_0 = indicators[0].id
        indicator_map = SourceObjectMap.objects.create(
            source_object_code="Percent missed children_PCA",
            content_type="indicator",
            mapped_by_id=self.user_id,
            master_object_id=self.mapped_indicator_id_0,
        )

        self.mapped_indicator_with_data = locations[2].id
        indicator_map = SourceObjectMap.objects.create(
            source_object_code="Percent missed due to other reasons",
            content_type="indicator",
            mapped_by_id=self.user_id,
            master_object_id=self.mapped_indicator_with_data,
        )
        ltc = LocationTreeCache()
        ltc.main()
def run_agg(apps, schema_editor):
    ltr = LocationTreeCache()
    ltr.main()

    # ensure that aggregation works by running the agg refresh in the migration itself.

    campaigns = Campaign.objects.all()
    for campaign in campaigns:
        if DataPoint.objects.filter(campaign_id=campaign.id).exists():
            agg = AggRefresh(campaign.id)
            agg.main()
Exemple #18
0
    def set_up(self):

        data_df = read_csv('rhizome/tests/_data/calc_data.csv')
        self.create_metadata()
        self.user = User.objects.get(username="******")

        self.test_df = data_df[data_df['is_raw'] == 1]
        self.target_df = data_df[data_df['is_raw'] == 0]

        ltr = LocationTreeCache()
        ltr.main()
def run_agg(apps, schema_editor):
	ltr = LocationTreeCache()
	ltr.main()

	# ensure that aggregation works by running the agg refresh in the migration itself.

	campaigns = Campaign.objects.all()
	for campaign in campaigns:
		if DataPoint.objects.filter(campaign_id = campaign.id).exists():
			agg = AggRefresh(campaign.id)
			agg.main()
    def setUp(self):
        super(DataPointResourceTest, self).setUp()

        # Create a user.
        self.username = '******'
        self.password = '******'
        self.user = User.objects.create_user(self.username,\
                                        '*****@*****.**', self.password)

        self.lt = LocationType.objects.create(name='Country', admin_level=0)
        self.province_lt = LocationType.objects.create(name='Province'\
            ,admin_level = 1)
        self.district_lt = LocationType.objects.create(name='District'\
            ,admin_level = 2)

        self.o = Office.objects.create(name='Earth')
        self.ind = Indicator.objects.create(name='Polio Cases',
                                            short_name='Polio Cases',
                                            data_format='date_int')

        self.top_lvl_location = Location.objects.create(
            name='Afghanistan',
            location_code='Afghanistan',
            id=1234,
            location_type_id=self.lt.id,
            office_id=self.o.id,
        )
        self.some_province = Location.objects.create(
            name='Province',
            location_code='Province',
            id=432,
            parent_location_id=self.top_lvl_location.id,
            location_type_id=self.province_lt.id,
            office_id=self.o.id,
        )
        self.some_district = Location.objects.create(
            name='Achin',
            location_code='Achin',
            id=4321,
            parent_location_id=self.some_province.id,
            location_type_id=self.district_lt.id,
            office_id=self.o.id,
        )

        ltc = LocationTreeCache()
        ltc.main()

        LocationPermission.objects.create(user_id = self.user.id,\
            top_lvl_location_id = self.top_lvl_location.id)

        self.get_credentials()
        self.create_polio_cases()
Exemple #21
0
    def setUp(self):

        self.ts = TestSetupHelpers()
        data_df = read_csv('rhizome/tests/_data/calc_data.csv')
        self.create_metadata()
        self.user = User.objects.get(username="******")

        self.test_df = data_df[data_df['is_raw'] == 1]
        self.target_df = data_df[data_df['is_raw'] == 0]
        self.campaign_id = Campaign.objects.all()[0].id
        self.top_lvl_location = Location.objects.filter(name='Nigeria')[0]
        ltr = LocationTreeCache()
        ltr.main()
Exemple #22
0
    def setUp(self):

        self.ts = TestSetupHelpers()
        data_df = read_csv('rhizome/tests/_data/calc_data.csv')
        self.create_metadata()
        self.user = User.objects.get(username="******")

        self.test_df = data_df[data_df['is_raw'] == 1]
        self.target_df = data_df[data_df['is_raw'] == 0]
        self.campaign_id = Campaign.objects.all()[0].id
        self.top_lvl_location = Location.objects.filter(name='Nigeria')[0]
        ltr = LocationTreeCache()
        ltr.main()
Exemple #23
0
    def setUp(self):

        ## instantiate the test client and all other methods ##
        super(LocationResourceTest, self).setUp()

        # Create a user.
        self.username = '******'
        self.password = '******'
        self.user = User.objects\
            .create_user(self.username,'*****@*****.**', self.password)
        self.lt = LocationType.objects.create(name='test',admin_level = 0)
        self.o = Office.objects.create(name = 'Earth')
        self.not_allowed_to_see_location = Location.objects.create(
                name = 'Somalia',
                location_code = 'Somalia',
                location_type_id = self.lt.id,
                office_id = self.o.id,
            )
        self.top_lvl_location = Location.objects.create(
                name = 'Nigeria',
                location_code = 'Nigeria',
                location_type_id = self.lt.id,
                office_id = self.o.id,
            )

        self.sub_location = Location.objects.create(
                name = 'Kano',
                location_code = 'Kano',
                location_type_id = self.lt.id,
                office_id = self.o.id,
                parent_location_id = self.top_lvl_location.id
            )

        ### set the user permission ###
        LocationPermission.objects.create(user_id = self.user.id,\
            top_lvl_location_id = self.top_lvl_location.id)

        self.get_credentials()

        ltr = LocationTreeCache()
        ltr.main()
    def test_location_id_and_location_depth(self):
        '''
        When i pass location_id and depth_level, I should get data for
        datapoints underneath the location_id requested at the specified
        depth level
        '''

        # create Afghanistan, region, and provinces
        afghanistan = Location.objects.create(
            name='Afghanistan',
            location_code='Afghanistan',
            location_type_id=self.country_lt.id)

        south = Location.objects.create(name='South',
                                        location_code='South',
                                        location_type_id=self.region_lt.id,
                                        parent_location_id=afghanistan.id)

        kandahar = Location.objects.create(
            name='Kandahar',
            location_code='Kandahar',
            location_type_id=self.province_lt.id,
            parent_location_id=south.id)

        hilmand = Location.objects.create(name='Hilmand',
                                          location_code='Hilmand',
                                          location_type_id=self.province_lt.id,
                                          parent_location_id=south.id)

        ltc = LocationTreeCache()
        ltc.main()

        indicator = Indicator.objects.create(short_name='stuff', \
            name='stuff', \
            data_format='int',\
            description='some stuff that we want to count', )

        start_date = '2014-01-01'
        end_date = '2014-01-01'

        campaign = Campaign.objects.get(start_date=start_date)

        kandahar_value = 27
        hilmand_value = 31
        # create some datapoints at province level
        dp_kandahar = DataPointComputed.objects.create(
            location_id=kandahar.id,
            value=kandahar_value,
            campaign_id=campaign.id,
            indicator_id=indicator.id)

        dp_hilmand = DataPointComputed.objects.create(
            location_id=hilmand.id,
            value=hilmand_value,
            campaign_id=campaign.id,
            indicator_id=indicator.id)

        get_parameter = 'indicator__in={0}&campaign__in={1}&location_id={2}&location_depth=2'\
            .format(indicator.id, campaign.id, afghanistan.id)

        resp = self.api_client.get('/api/v1/campaign_datapoint/?' + get_parameter, \
            format='json', authentication=self.get_credentials())

        response_data = self.deserialize(resp)
        self.assertHttpOK(resp)
        self.assertEqual(len(response_data['objects']), 2)

        # makes sure we're getting the right dp values, thus confirming that the provinces are returned
        sum_of_values = 0
        for indicator in response_data['objects']:
            sum_of_values += float(indicator['value'])

        self.assertEqual(sum_of_values, kandahar_value + hilmand_value)
    def test_cache_location_tree(self):
        '''
        The point of this test is that it the cache_location_tree function
        creates database rows that represents each parent / child relationship
        between all locations, from the stored information.

        That is, if we say that NYC is a sub-location of NYState, and NYState
        is a sub-location of U.S.A. , then that means that we should have a row
        in the LocationTree table rebresenting the fact that both Nystate AND
        U.S.A. are parent_location's on New York City.
        '''

        location_batch = []
        location_type_country = LocationType.objects\
            .create(name='Country', admin_level=0, id=1)
        location_type_state = LocationType.objects\
            .create(name='State', admin_level=1, id=2)
        location_type_city = LocationType.objects\
            .create(name='City', admin_level=2, id=3)
        location_type_city = LocationType.objects\
            .create(name='Neighborhood', admin_level=3, id=4)

        location_data = {
            'id': [1, 2, 3, 4, 5, 6, 7, 8],
            'location_type_id': [1, 2, 3, 2, 2, 2, 3, 4],
            'name': ['U.S.A.', 'New York State', 'New York City', 'Texas',
                     'California', 'Maine', 'Portland, ME', 'Brooklyn'],
            'parent_location_id': [None, 1, 2, 1, 1, 1, 6, 3]
        }

        df = DataFrame.from_dict(location_data)
        df.set_index('id', inplace=True, drop=False, append=False)
        no_nan_df = df.where((notnull(df)), None)

        for ix, loc in no_nan_df.iterrows():
            location_batch.append(Location(**{
                'id': loc.id,
                'name': loc.name,
                'location_code': unicode(loc.name).replace(' ', ''),
                'location_type_id': loc.location_type_id,
                'parent_location_id': loc.parent_location_id
            }))

        Location.objects.bulk_create(location_batch)

        ## now use the following function to transform the location_tree ##
        ltc = LocationTreeCache()
        ltc.main()

        location_tree_in_db = LocationTree.objects.all()\
            .values_list('location_id', 'parent_location_id','lvl')

        ## every location should have a row in this table with
        ## lvl = 0 and itself as the parent_location_id
        for loc in location_data['id']:
            self.assertTrue((loc, loc, 0) in location_tree_in_db)

        ## new york city (3) has a parent of NY State (2) AND U.S.A (1)##
        self.assertTrue((3, 1, 2) in location_tree_in_db)
        self.assertTrue((3, 2, 1) in location_tree_in_db)

        ## Portland ME (7) has a parent of Maine (6) AND U.S.A (1) ##
        self.assertTrue((7, 1, 2) in location_tree_in_db)
        self.assertTrue((7, 6, 1) in location_tree_in_db)

        ## new york state and maine are direct children of USA ##
        self.assertTrue((2, 1, 1) in location_tree_in_db)
        self.assertTrue((6, 1, 1) in location_tree_in_db)

        ## Brooklkyn has, NYC, NYState and USA as parents ##
        self.assertTrue((8, 3, 1) in location_tree_in_db)
        self.assertTrue((8, 2, 2) in location_tree_in_db)
        self.assertTrue((8, 1, 3) in location_tree_in_db)
Exemple #26
0
    def test_cache_location_tree(self):
        '''
        The point of this test is that it the cache_location_tree function
        creates database rows that represents each parent / child relationship
        between all locations, from the stored information.

        That is, if we say that NYC is a sub-location of NYState, and NYState
        is a sub-location of U.S.A. , then that means that we should have a row
        in the LocationTree table rebresenting the fact that both Nystate AND
        U.S.A. are parent_location's on New York City.
        '''

        location_batch = []

        office = Office.objects.create(name='not important')
        location_type_country = LocationType.objects.create(name='Country',\
            admin_level=0,id=1)
        location_type_state = LocationType.objects.create(name='State',\
            admin_level=1,id=2)
        location_type_city = LocationType.objects.create(name='City',\
            admin_level=2,id=3)
        location_type_city = LocationType.objects.create(name='Neighborhood',\
            admin_level=3,id=4)

        location_data = {
                'id':[1,2,3,4,5,6,7,8],
                'location_type_id':[1,2,3,2,2,2,3,4],
                'name': ['U.S.A.','New York State','New York City','Texas',\
                    'California','Maine','Portland, ME','Brooklyn'],
                'parent_location_id': [None,1,2,1,1,1,6,3]
            }

        df = DataFrame.from_dict(location_data)
        df.set_index('id', inplace=True, drop=False, append=False)
        no_nan_df = df.where((notnull(df)), None)

        for ix, loc in no_nan_df.iterrows():
            location_batch.append(Location(**{
                'id':loc.id,
                'name':loc.name,
                'location_code': unicode(loc.name).replace(' ',''),
                'office_id':office.id,
                'location_type_id':loc.location_type_id,
                'parent_location_id':loc.parent_location_id
            }))

        Location.objects.bulk_create(location_batch)

        ## now use the following function to transform the location_tree ##
        ltc = LocationTreeCache()
        ltc.main()

        location_tree_in_db = LocationTree.objects.all()\
            .values_list('location_id','parent_location_id')

        ## the ultimate parent, should have a with itself is parent ##
        self.assertTrue((1,1) in location_tree_in_db)

        ## Brooklyn However should not be the parent of itself ##
        self.assertFalse((8,8) in location_tree_in_db)

        ## new york city has a parent of NY State AND U.S.A ##
        self.assertTrue((3,1) in location_tree_in_db)
        self.assertTrue((3,2) in location_tree_in_db)

        ## Portland ME has a parent of Maine AND U.S.A ##
        self.assertTrue((7,1) in location_tree_in_db)
        self.assertTrue((7,6) in location_tree_in_db)

        # ## new york state and maine are children of USA #
        self.assertTrue((2,1) in location_tree_in_db)
        self.assertTrue((6,1) in location_tree_in_db)

        ## Brooklkyn has, NYC, NYState and USA as parents ##
        self.assertTrue((8,3) in location_tree_in_db)
        self.assertTrue((8,2) in location_tree_in_db)
        self.assertTrue((8,1) in location_tree_in_db)
Exemple #27
0
    def setUp(self):

        ## instantiate the test client and all other methods ##
        super(CampaignResourceTest, self).setUp()

        # Create a user.
        self.username = '******'
        self.password = '******'
        self.user = User.objects\
            .create_user(self.username,'*****@*****.**', self.password)
        self.lt = LocationType.objects.create(name='test',admin_level = 0)
        self.o = Office.objects.create(name = 'Earth')
        self.not_allowed_to_see_location = Location.objects.create(
                name = 'Somalia',
                location_code = 'Somalia',
                location_type_id = self.lt.id,
                office_id = self.o.id,
            )

        self.top_lvl_location = Location.objects.create(
                name = 'Nigeria',
                location_code = 'Nigeria',
                location_type_id = self.lt.id,
                office_id = self.o.id,
            )

        self.sub_location = Location.objects.create(
                name = 'Kano',
                location_code = 'Kano',
                location_type_id = self.lt.id,
                office_id = self.o.id,
                parent_location_id = self.top_lvl_location.id
            )
        self.it = IndicatorTag.objects.create(tag_name='Polio')

        self.ct = CampaignType.objects.create(name='NID')
        self.can_see_campaign = Campaign.objects.create(
            start_date = '2016-01-01',
            end_date = '2016-01-01',
            office_id = self.o.id,
            campaign_type_id = self.ct.id,
            top_lvl_location_id = self.top_lvl_location.id,
            top_lvl_indicator_tag_id = self.it.id
        )

        self.can_not_see_campaign = Campaign.objects.create(
            start_date = '2016-02-01',
            end_date = '2016-02-01',
            office_id = self.o.id,
            campaign_type_id = self.ct.id,
            top_lvl_location_id = self.not_allowed_to_see_location.id,
            top_lvl_indicator_tag_id = self.it.id
        )

        ### set the user permission ###
        LocationPermission.objects.create(user_id = self.user.id,\
            top_lvl_location_id = self.top_lvl_location.id)

        self.get_credentials()

        ltr = LocationTreeCache()
        ltr.main()
Exemple #28
0
    def build_location_meta(self):

        self.existing_location_map = {
            self.file_map['country_display']: self.top_lvl_location.id
        }

        ## PROVINCE ##
        ## since we ingested the shapes first, we lookup the province name,
        ## then change it to what the ODK form has.. this allows us to attach
        ## the shapes to the location IDS from ODK so that they are familiar
        ## to the owners of the data

        province_column = self.file_map['column_map']['province_column']
        province_df = pd.DataFrame(self.source_sheet_df[province_column])

        for ix, row in province_df.iterrows():

            row_dict = row.to_dict()

            province_name = row_dict[province_column]


            location_obj, created = Location.objects.get_or_create(
                name = province_name,
                defaults = {
                'location_code': province_name,
                'parent_location_id': self.top_lvl_location.id,
                'location_type_id': LocationType.objects\
                    .get(name = 'Province').id
            })
            self.existing_location_map[province_name] = location_obj.id

        # DISTRICT ##
        district_column = self.file_map['column_map']['district_column']
        district_df = pd.DataFrame(\
            self.source_sheet_df[[district_column,province_column]])

        district_df.drop_duplicates(inplace=True)
        self.process_location_df(district_df, 'District')

        # ## ADMIN 3  ##
        # settlement_column = self.file_map['column_map']['admn_2_column']
        # settlement_df = pd.DataFrame(\
        #     self.source_sheet_df[[settlement_column,district_column]])
        #
        # settlement_df.drop_duplicates(inplace=True,subset=[settlement_column])
        # self.process_location_df(settlement_df, 'Settlement')
        #
        # ## this wil lmake it so we can ingest data
        # source_object_map_batch = [SourceObjectMap(**{
        #     'master_object_id': loc.id,
        #     'content_type': 'location',
        #     'source_object_code': loc.location_code
        # }) for loc in Location.objects.all()]
        # # SourceObjectMap.objects.bulk_create(source_object_map_batch)

        ##  now let me change the names of the locations
        ##  so that they are familiar to the progam

        # for k,v in self.location_lookup.iteritems():
        #
        #     try:
        #         l = Location.objects.get(name=v)
        #         l.name = k
        #         l.location_code = k
        #         l.save()
        #     except Location.DoesNotExist:  ## LOOK INTO THIS....
        #         pass

        ## populate the location tree @
        ltc = LocationTreeCache()
        ltc.main()

        if len(LocationTree.objects.all()) == 0:
            raise Exception('Empty Location Tree')
    def test_location_type_and_depth(self):
        # create Afghanistan, region, and provinces
        afghanistan = Location.objects.create(
            name='Afghanistan',
            location_code ='Afghanistan',
            location_type_id =self.lt.id,
            office_id = self.o.id
        )

        south = Location.objects.create(
            name='South',
            location_code = 'South',
            location_type_id=self.region.id,
            office_id = self.o.id,
            parent_location_id = afghanistan.id
            )

        kandahar = Location.objects.create(
            name='Kandahar',
            location_code = 'Kandahar',
            location_type_id = self.prov.id,
            office_id = self.o.id,
            parent_location_id = south.id
            )

        hilmand = Location.objects.create(
            name='Hilmand',
            location_code = 'Hilmand',
            location_type_id = self.prov.id,
            office_id = self.o.id,
            parent_location_id = south.id
            )

        ltc = LocationTreeCache()
        ltc.main()

        indicator = Indicator.objects.create(short_name='stuff', \
            name='stuff', \
            data_format='int',\
            description='some stuff that we want to count', )

        document = Document.objects.create(doc_title='I am Every Woman -- Whitney Houston')

        start_date = '2016-01-01'
        end_date = '2016-01-01'
        campaign_type = CampaignType.objects\
            .create(name='National Immunization Days (NID)')

        ind_tag = IndicatorTag.objects.create(tag_name='Polio')
        campaign = Campaign.objects.create(office=self.o,\
            campaign_type=campaign_type,start_date=start_date,end_date=end_date,\
            top_lvl_indicator_tag_id = ind_tag.id,\
            top_lvl_location_id = afghanistan.id)

        kandahar_value =27
        hilmand_value =31
        # create some datapoints at province level
        dp_kandahar = DataPointComputed.objects.create(
            location_id = kandahar.id,
            value = kandahar_value,
            campaign_id = campaign.id,
            indicator_id = indicator.id,
            document_id = document.id,
        )

        dp_hilmand = DataPointComputed.objects.create(
            location_id = hilmand.id,
            value = hilmand_value,
            campaign_id = campaign.id,
            indicator_id = indicator.id,
            document_id = document.id
        )

        # TRY FOR LOCATION TYPE
        # ++++++++++++++++++++++

        get_parameter = 'indicator__in={0}&campaign__in={1}&location_id__in={2}&location_type={3}'\
            .format(indicator.id, campaign.id, afghanistan.id, self.prov.id)

        resp = self.api_client.get('/api/v1/datapoint/?' + get_parameter, \
            format='json', authentication=self.get_credentials())

        response_data = self.deserialize(resp)
        self.assertHttpOK(resp)
        self.assertEqual(len(response_data['objects']), 2)

        # makes sure we're getting the right dp values, thus confirming that the provinces are returned
        sum_of_values = 0
        for return_indicator in response_data['objects']:
            sum_of_values += float(return_indicator['value'])

        self.assertEqual(sum_of_values, kandahar_value+hilmand_value)



        # TRY FOR LOCATION DEPTH
        # ++++++++++++++++++++++

        get_parameter = 'indicator__in={0}&campaign__in={1}&location_id__in={2}&location_depth=2'\
            .format(indicator.id, campaign.id, afghanistan.id)

        resp = self.api_client.get('/api/v1/datapoint/?' + get_parameter, \
            format='json', authentication=self.get_credentials())

        response_data = self.deserialize(resp)
        self.assertHttpOK(resp)
        self.assertEqual(len(response_data['objects']), 2)

        # makes sure we're getting the right dp values, thus confirming that the provinces are returned
        sum_of_values = 0
        for indicator in response_data['objects']:
            sum_of_values += float(indicator['value'])

        self.assertEqual(sum_of_values, kandahar_value+hilmand_value)
    def test_indicator_filter(self):
        campaign_id = 2

        document = Document.objects.create(doc_title='some doc')

        #make a couple different types of indicators, and indicators with different values
        indicator_names_to_values = {"LPD Status":[1,2],
            'LQAS':[0, 1, 2]
        }

        indicator_ids_to_values ={}

        for indicator_name, values in indicator_names_to_values.iteritems():
            indicator = Indicator.objects.create(short_name=indicator_name, \
                                     name=indicator_name, \
                                     description=indicator_name,
                                     data_format ='class' )
            indicator_ids_to_values[indicator.id] = values



        some_provinces = ['Kandahar', 'Kunar', 'Hilmand', 'Nimroz', 'Sari-Pul', 'Kabul', 'Paktika', 'Ghazni']

        ind_id_keys = indicator_ids_to_values.keys()
        indicator_to_query = ind_id_keys[1]

       # choose which indicator/value pair to filter by, and keep track of dps that match this as they're created
        indicator_to_filter = ind_id_keys[0]
        indicator_val_to_filter = indicator_ids_to_values[indicator_to_filter][0]
        dps_to_track =[]

        for province in some_provinces:
            # create the province
            prov = Location.objects.create(
                name = province,
                location_code = province,
                location_type_id = self.prov.id,
                office_id = self.o.id,
                parent_location_id = self.top_lvl_location.id
            )

            # create a random dp for each indicator
            for indicator_id, values in indicator_ids_to_values.iteritems():
                idx = randint(0, len(values)-1)
                value_to_use = values[idx]
                dp = DataPointComputed.objects.create(
                    location_id = prov.id,
                    value = value_to_use,
                    campaign_id = campaign_id,
                    indicator_id = indicator_id,
                    document_id = document.id
                )
                if indicator_id == indicator_to_filter and value_to_use == indicator_val_to_filter:
                    dps_to_track.append(dp)

        ltc = LocationTreeCache()
        ltc.main()

        indicator_name_to_filter = Indicator.objects.get(id=indicator_to_filter);
        get_parameter = 'indicator__in={0}&campaign__in={1}&location_id__in={2}&location_depth=2&filter_indicator={3}&filter_value={4}&chart_type=TableChart'\
            .format(indicator_to_query, campaign_id, self.top_lvl_location.id, indicator_name_to_filter, indicator_val_to_filter)

        resp = self.api_client.get('/api/v1/datapoint/?' + get_parameter, \
            format='json', authentication=self.get_credentials())

        response_data = self.deserialize(resp)
        self.assertEqual(len(response_data['objects']), len(dps_to_track))
        # oof, nested for loop is okay since it's a small dataset
        # makes sure that all the campaign and location ids match
        for dp in dps_to_track:
            found_dp = False
            for resp_dp in response_data['objects']:
                same_campaign = int(resp_dp['campaign_id']) == dp.campaign_id
                same_location = int(resp_dp['location_id']) == dp.location_id
                if same_location and same_campaign:
                    found_dp = True
            if not found_dp:
                fail("the datapoints from the respnse do not match the system")
                break
        pass