def set_up(self):
self.d = date.today()
st = self.d - timedelta(days=1)
ed = self.d + timedelta(days=1)
self.u = User.objects.create_user('polio','*****@*****.**', 'polio')
o = Office.objects.create(name='NGA')
lt = LocationType.objects.create(name='country',admin_level=0)
ct = CampaignType.objects.create(name='NID')
self.ind_0 = Indicator.objects.create(name='number of VDPV cases',short_name='V')
self.ind_1 = Indicator.objects.create(name='number of WPV cases',short_name='W')
ind_tag = IndicatorTag.objects.create(tag_name='Polio')
self.tpl = Location.objects.create(name='NGA',location_code='NGA',\
office_id = o.id,location_type_id = lt.id)
self.doc = Document.objects.create(
doc_title = 'test',
created_by_id = self.u.id,
guid = 'test')
### SET UP CAMPAIGN DEFINITION ###
self.c = Campaign.objects.create(
office_id = o.id,\
top_lvl_location_id = self.tpl.id,
top_lvl_indicator_tag_id = ind_tag.id,
campaign_type_id = ct.id,
name = 'test',\
start_date = st,\
end_date = ed,\
)
ltc = LocationTreeCache()
ltc.main()
def setUp(self):
super(CampaignDataPointResourceTest, self).setUp()
# Create a user.
self.username = '******'
self.password = '******'
self.user = User.objects.create_user(self.username,\
'*****@*****.**', self.password)
self.get_credentials()
self.ts = TestSetupHelpers()
## create a metadata and data for us to use to test ##
self.create_metadata()
self.country_lt = LocationType.objects\
.create(name='Country',admin_level = 0)
self.region_lt = LocationType.objects\
.create(name='Region',admin_level = 1)
self.province_lt = LocationType.objects\
.create(name='Province',admin_level = 2)
self.top_lvl_location = Location.objects.get(name='Nigeria')
ltc = LocationTreeCache()
ltc.main()
LocationPermission.objects.create(user_id = self.user.id,\
top_lvl_location_id = self.top_lvl_location.id)
def setUp(self):
super(DataPointResourceTest, self).setUp()
# Create a user.
self.username = '******'
self.password = '******'
self.user = User.objects.create_user(self.username,\
'*****@*****.**', self.password)
self.lt = LocationType.objects.create(name='Country',admin_level = 0)
self.o = Office.objects.create(name = 'Earth')
self.top_lvl_location = Location.objects.create(
name = 'Nigeria',
location_code = 'Nigeria',
location_type_id = self.lt.id,
office_id = self.o.id,
)
ltc = LocationTreeCache()
ltc.main()
LocationPermission.objects.create(user_id = self.user.id,\
top_lvl_location_id = self.top_lvl_location.id)
self.get_credentials()
def setUp(self):
## instantiate the test client and all other methods ##
super(LocationResourceTest, self).setUp()
# Create a user.
self.username = '******'
self.password = '******'
self.user = User.objects\
.create_user(self.username, '*****@*****.**', self.password)
self.lt = LocationType.objects.create(name='test', admin_level=0)
self.not_allowed_to_see_location = Location.objects.create(
name='Somalia',
location_code='Somalia',
location_type_id=self.lt.id)
self.top_lvl_location = Location.objects.create(
name='Nigeria',
location_code='Nigeria',
location_type_id=self.lt.id)
self.sub_location = Location.objects.create(
name='Kano',
location_code='Kano',
location_type_id=self.lt.id,
parent_location_id=self.top_lvl_location.id)
### set the user permission ###
LocationPermission.objects.create(
user_id=self.user.id, top_lvl_location_id=self.top_lvl_location.id)
self.get_credentials()
ltr = LocationTreeCache()
ltr.main()
def setUp(self):
super(DocTransformResourceTest, self).setUp()
self.ts = TestSetupHelpers()
self.ts.load_some_metadata()
ltr = LocationTreeCache()
ltr.main()
self.mapped_location_id = self.ts.locations[0].id
self.mapped_location_id_2 = self.ts.locations[1].id
self.loc_map = SourceObjectMap.objects.create(
source_object_code = 'AF001039003000000000',
content_type = 'location',
mapped_by_id = self.ts.user.id,
master_object_id = self.mapped_location_id
)
source_campaign_string = '2016 March NID OPV'
self.mapped_campaign_id = self.ts.campaigns[0].id
self.campaign_map = SourceObjectMap.objects.create(
source_object_code = source_campaign_string,
content_type = 'campaign',
mapped_by_id = self.ts.user.id,
master_object_id = self.mapped_campaign_id
)
self.mapped_indicator_with_data = self.ts.locations[2].id
self.indicator_map = SourceObjectMap.objects.create(
source_object_code = 'Percent missed due to other reasons',
content_type = 'indicator',
mapped_by_id = self.ts.user.id,
master_object_id = self.mapped_indicator_with_data
)
def set_up(self):
self.d = date.today()
st = self.d - timedelta(days=1)
ed = self.d + timedelta(days=1)
self.u = User.objects.create_user(
'polio', '*****@*****.**', 'polio')
lt = LocationType.objects.create(name='country', admin_level=0)
ct = CampaignType.objects.create(name='NID')
self.ind_0 = Indicator.objects.create(
name='number of VDPV cases', short_name='V')
self.ind_1 = Indicator.objects.create(
name='number of WPV cases', short_name='W')
ind_tag = IndicatorTag.objects.create(tag_name='Polio')
self.tpl = Location.objects.create(name='NGA', location_code='NGA',
location_type_id=lt.id)
self.doc = Document.objects.create(
doc_title='test',
created_by_id=self.u.id,
guid='test')
### SET UP CAMPAIGN DEFINITION ###
self.c = Campaign.objects.create(
campaign_type_id=ct.id,
name='test',
start_date=st,
end_date=ed,
)
ltc = LocationTreeCache()
ltc.main()
def process_meta_data():
xl = pd.ExcelFile('migration_data/initial_data.xlsx')
all_sheets = xl.sheet_names
rhizome_app = get_app('rhizome')
auth_app = get_app('auth')
models_to_process = {}
all_models = get_models(rhizome_app) + get_models(auth_app)
for model in all_models:
## iterate through the models in the rhizome app and create a lookup
## for {'sheet_name': Model} .. for instance -> {'indicator': Indicator}
if model._meta.db_table in all_sheets:
models_to_process[model._meta.db_table] = model
for sheet in all_sheets:
## if the sheet has a cooresponding model, create a data frame out of
## the sheet anf bulk insert the data using the model_df_to_data fn
try:
model = models_to_process[sheet]
print 'processing sheet ' + sheet
model_df = xl.parse(sheet)
model_ids = model_df_to_data(model_df, model)
except KeyError:
pass
## once the locations are all created we need to ##
## cache them in the locaiton_tree table ##
ltc = LocationTreeCache()
ltc.main()
def setUp(self):
## instantiate the test client and all other methods ##
super(CampaignResourceTest, self).setUp()
self.ts = TestSetupHelpers()
self.lt = self.ts.create_arbitrary_location_type()
self.o = self.ts.create_arbitrary_office()
self.not_allowed_to_see_location = self.ts.create_arbitrary_location(
self.lt.id,
self.o.id)
self.top_lvl_location = self.ts.create_arbitrary_location(
self.lt.id,
self.o.id,
location_code ='Nigeria',
location_name='Nigeria')
self.sub_location = self.ts.create_arbitrary_location(
self.lt.id,
self.o.id,
location_name = 'Kano',
location_code = 'Kano',
parent_location_id = self.top_lvl_location.id)
self.it = IndicatorTag.objects.create(tag_name='Polio')
self.ct = CampaignType.objects.create(name='NID')
self.can_see_campaign = self.ts.create_arbitrary_campaign(
office_id = self.ts.create_arbitrary_office(name='test1').id,
campaign_type_id = self.ct.id,
location_id = self.top_lvl_location.id,
indicator_tag_id = self.it.id,
name="can_see"
)
self.can_see_campaign_2 = self.ts.create_arbitrary_campaign(
office_id = self.ts.create_arbitrary_office(name='test2').id,
campaign_type_id = self.ct.id,
location_id = self.top_lvl_location.id,
indicator_tag_id = self.it.id,
name="can_see2"
)
self.can_not_see_campaign = self.ts.create_arbitrary_campaign(
office_id = self.o.id,
campaign_type_id = self.ct.id,
location_id = self.not_allowed_to_see_location.id,
indicator_tag_id = self.it.id,
)
### set the user permission ###
LocationPermission.objects.create(user_id = self.ts.user.id,\
top_lvl_location_id = self.top_lvl_location.id)
self.ts.get_credentials(self)
ltr = LocationTreeCache()
ltr.main()
def setUp(self):
super(DataPointResourceTest, self).setUp()
# Create a user.
self.username = '******'
self.password = '******'
self.user = User.objects.create_user(self.username,\
'*****@*****.**', self.password)
self.lt = LocationType.objects.create(name='Country',admin_level = 0)
self.distr, created = \
LocationType.objects.get_or_create(name='District',admin_level = 3)
self.prov, created = \
LocationType.objects.get_or_create(name='Province',admin_level = 2)
self.region, created = \
LocationType.objects.get_or_create(name='Region', admin_level = 1)
self.o = Office.objects.create(name = 'Earth')
self.top_lvl_location = Location.objects.create(
name = 'Nigeria',
location_code = 'Nigeria',
id=1234,
location_type_id = self.lt.id,
office_id = self.o.id,
)
ltc = LocationTreeCache()
ltc.main()
LocationPermission.objects.create(user_id = self.user.id,\
top_lvl_location_id = self.top_lvl_location.id)
self.get_credentials()
def process_meta_data():
xl = pd.ExcelFile('migration_data/initial_data.xlsx')
all_sheets = xl.sheet_names
rhizome_app = get_app('rhizome')
auth_app = get_app('auth')
models_to_process = {}
all_models = get_models(rhizome_app) + get_models(auth_app)
for model in all_models:
## iterate through the models in the rhizome app and create a lookup
## for {'sheet_name': Model} .. for instance -> {'indicator': Indicator}
if model._meta.db_table in all_sheets:
models_to_process[model._meta.db_table] = model
for sheet in all_sheets:
## if the sheet has a cooresponding model, create a data frame out of
## the sheet anf bulk insert the data using the model_df_to_data fn
try:
model = models_to_process[sheet]
print 'processing sheet ' + sheet
model_df = xl.parse(sheet)
model_ids = model_df_to_data(model_df,model)
except KeyError:
pass
## once the locations are all created we need to ##
## cache them in the locaiton_tree table ##
ltc = LocationTreeCache()
ltc.main()
def create_metadata(self):
'''
Creating the Indicator, location, Campaign, meta data needed for the
system to aggregate / caclulate.
'''
top_lvl_tag = IndicatorTag.objects.create(id=1, tag_name='Polio')
campaign_df = read_csv('rhizome/tests/_data/campaigns.csv')
campaign_df['start_date'] = to_datetime(campaign_df['start_date'])
campaign_df['end_date'] = to_datetime(campaign_df['end_date'])
location_df = read_csv('rhizome/tests/_data/locations.csv')
indicator_df = read_csv('rhizome/tests/_data/indicators.csv')
campaign_type = CampaignType.objects.create(id=1, name="test")
locations = self.model_df_to_data(location_df, Location)
campaigns = self.model_df_to_data(campaign_df, Campaign)
indicators = self.model_df_to_data(indicator_df, Indicator)
self.user_id = User.objects.create_user(
'test', '*****@*****.**', 'test').id
self.mapped_location_id = locations[0].id
loc_map = SourceObjectMap.objects.create(
source_object_code='AF001039003000000000',
content_type='location',
mapped_by_id=self.user_id,
master_object_id=self.mapped_location_id
)
source_campaign_string = '2016 March NID OPV'
self.mapped_campaign_id = campaigns[0].id
campaign_map = SourceObjectMap.objects.create(
source_object_code=source_campaign_string,
content_type='campaign',
mapped_by_id=self.user_id,
master_object_id=self.mapped_campaign_id
)
self.mapped_indicator_id_0 = indicators[0].id
indicator_map = SourceObjectMap.objects.create(
source_object_code='Percent missed children_PCA',
content_type='indicator',
mapped_by_id=self.user_id,
master_object_id=self.mapped_indicator_id_0
)
self.mapped_indicator_with_data = locations[2].id
indicator_map = SourceObjectMap.objects.create(
source_object_code='Percent missed due to other reasons',
content_type='indicator',
mapped_by_id=self.user_id,
master_object_id=self.mapped_indicator_with_data
)
## make sure that the location tree is updated ##
ltc = LocationTreeCache()
ltc.main()
def setUp(self):
## instantiate the test client and all other methods ##
super(CampaignResourceTest, self).setUp()
self.ts = TestSetupHelpers()
self.lt = self.ts.create_arbitrary_location_type()
self.o = self.ts.create_arbitrary_office()
self.not_allowed_to_see_location = self.ts.create_arbitrary_location(
self.lt.id, self.o.id)
self.top_lvl_location = self.ts.create_arbitrary_location(
self.lt.id,
self.o.id,
location_code='Nigeria',
location_name='Nigeria')
self.sub_location = self.ts.create_arbitrary_location(
self.lt.id,
self.o.id,
location_name='Kano',
location_code='Kano',
parent_location_id=self.top_lvl_location.id)
self.it = IndicatorTag.objects.create(tag_name='Polio')
self.ct = CampaignType.objects.create(name='NID')
self.can_see_campaign = self.ts.create_arbitrary_campaign(
office_id=self.ts.create_arbitrary_office(name='test1').id,
campaign_type_id=self.ct.id,
location_id=self.top_lvl_location.id,
indicator_tag_id=self.it.id,
name="can_see")
self.can_see_campaign_2 = self.ts.create_arbitrary_campaign(
office_id=self.ts.create_arbitrary_office(name='test2').id,
campaign_type_id=self.ct.id,
location_id=self.top_lvl_location.id,
indicator_tag_id=self.it.id,
name="can_see2")
self.can_not_see_campaign = self.ts.create_arbitrary_campaign(
office_id=self.o.id,
campaign_type_id=self.ct.id,
location_id=self.not_allowed_to_see_location.id,
indicator_tag_id=self.it.id,
)
### set the user permission ###
LocationPermission.objects.create(user_id = self.ts.user.id,\
top_lvl_location_id = self.top_lvl_location.id)
self.ts.get_credentials(self)
ltr = LocationTreeCache()
ltr.main()
def create_metadata(self):
'''
Creating the Indicator, location, Campaign, meta data needed for the
system to aggregate / caclulate.
'''
top_lvl_tag = IndicatorTag.objects.create(id=1, tag_name='Polio')
campaign_df = read_csv('rhizome/tests/_data/campaigns.csv')
campaign_df['start_date'] = to_datetime(campaign_df['start_date'])
campaign_df['end_date'] = to_datetime(campaign_df['end_date'])
location_df = read_csv('rhizome/tests/_data/locations.csv')
indicator_df = read_csv('rhizome/tests/_data/indicators.csv')
campaign_type = CampaignType.objects.create(id=1, name="test")
locations = self.model_df_to_data(location_df, Location)
campaigns = self.model_df_to_data(campaign_df, Campaign)
indicators = self.model_df_to_data(indicator_df, Indicator)
self.user_id = User.objects.create_user(
'test', '*****@*****.**', 'test').id
self.mapped_location_id = locations[0].id
loc_map = SourceObjectMap.objects.create(
source_object_code='AF001039003000000000',
content_type='location',
mapped_by_id=self.user_id,
master_object_id=self.mapped_location_id
)
source_campaign_string = '2016 March NID OPV'
self.mapped_campaign_id = campaigns[0].id
campaign_map = SourceObjectMap.objects.create(
source_object_code=source_campaign_string,
content_type='campaign',
mapped_by_id=self.user_id,
master_object_id=self.mapped_campaign_id
)
self.mapped_indicator_id_0 = indicators[0].id
indicator_map = SourceObjectMap.objects.create(
source_object_code='Percent missed children_PCA',
content_type='indicator',
mapped_by_id=self.user_id,
master_object_id=self.mapped_indicator_id_0
)
self.mapped_indicator_with_data = locations[2].id
indicator_map = SourceObjectMap.objects.create(
source_object_code='Percent missed due to other reasons',
content_type='indicator',
mapped_by_id=self.user_id,
master_object_id=self.mapped_indicator_with_data
)
ltc = LocationTreeCache()
ltc.main()
def setUp(self):
super(DataPointResourceTest, self).setUp()
# Create a user.
self.username = '******'
self.password = '******'
self.user = User.objects.create_user(self.username,\
'*****@*****.**', self.password)
self.lt = LocationType.objects.create(name='Country',admin_level = 0)
self.province_lt = LocationType.objects.create(name='Province'\
,admin_level = 1)
self.district_lt = LocationType.objects.create(name='District'\
,admin_level = 2)
self.o = Office.objects.create(name = 'Earth')
self.ind = Indicator.objects.create(
name = 'Polio Cases',
short_name = 'Polio Cases',
data_format = 'date_int'
)
self.top_lvl_location = Location.objects.create(
name = 'Afghanistan',
location_code = 'Afghanistan',
id=1234,
location_type_id = self.lt.id,
office_id = self.o.id,
)
self.some_province = Location.objects.create(
name = 'Province',
location_code = 'Province',
id=432,
parent_location_id = self.top_lvl_location.id,
location_type_id = self.province_lt.id,
office_id = self.o.id,
)
self.some_district = Location.objects.create(
name = 'Achin',
location_code = 'Achin',
id=4321,
parent_location_id = self.some_province.id,
location_type_id = self.district_lt.id,
office_id = self.o.id,
)
ltc = LocationTreeCache()
ltc.main()
LocationPermission.objects.create(user_id = self.user.id,\
top_lvl_location_id = self.top_lvl_location.id)
self.get_credentials()
self.create_polio_cases()
def set_up(self):
data_df = read_csv('rhizome/tests/_data/calc_data.csv')
self.create_metadata()
self.user = User.objects.get(username="******")
self.test_df = data_df[data_df['is_raw'] == 1]
self.target_df = data_df[data_df['is_raw'] == 0]
ltr = LocationTreeCache()
ltr.main()
def create_metadata(self):
"""
Creating the Indicator, location, Campaign, meta data needed for the
system to aggregate / caclulate.
"""
top_lvl_tag = IndicatorTag.objects.create(id=1, tag_name="Polio")
campaign_df = read_csv("rhizome/tests/_data/campaigns.csv")
campaign_df["start_date"] = to_datetime(campaign_df["start_date"])
campaign_df["end_date"] = to_datetime(campaign_df["end_date"])
location_df = read_csv("rhizome/tests/_data/locations.csv")
indicator_df = read_csv("rhizome/tests/_data/indicators.csv")
campaign_type = CampaignType.objects.create(id=1, name="test")
locations = self.model_df_to_data(location_df, Location)
campaigns = self.model_df_to_data(campaign_df, Campaign)
indicators = self.model_df_to_data(indicator_df, Indicator)
self.user_id = User.objects.create_user("test", "*****@*****.**", "test").id
self.mapped_location_id = locations[0].id
loc_map = SourceObjectMap.objects.create(
source_object_code="AF001039003000000000",
content_type="location",
mapped_by_id=self.user_id,
master_object_id=self.mapped_location_id,
)
source_campaign_string = "2016 March NID OPV"
self.mapped_campaign_id = campaigns[0].id
campaign_map = SourceObjectMap.objects.create(
source_object_code=source_campaign_string,
content_type="campaign",
mapped_by_id=self.user_id,
master_object_id=self.mapped_campaign_id,
)
self.mapped_indicator_id_0 = indicators[0].id
indicator_map = SourceObjectMap.objects.create(
source_object_code="Percent missed children_PCA",
content_type="indicator",
mapped_by_id=self.user_id,
master_object_id=self.mapped_indicator_id_0,
)
self.mapped_indicator_with_data = locations[2].id
indicator_map = SourceObjectMap.objects.create(
source_object_code="Percent missed due to other reasons",
content_type="indicator",
mapped_by_id=self.user_id,
master_object_id=self.mapped_indicator_with_data,
)
ltc = LocationTreeCache()
ltc.main()
def run_agg(apps, schema_editor):
ltr = LocationTreeCache()
ltr.main()
# ensure that aggregation works by running the agg refresh in the migration itself.
campaigns = Campaign.objects.all()
for campaign in campaigns:
if DataPoint.objects.filter(campaign_id=campaign.id).exists():
agg = AggRefresh(campaign.id)
agg.main()
def set_up(self):
data_df = read_csv('rhizome/tests/_data/calc_data.csv')
self.create_metadata()
self.user = User.objects.get(username="******")
self.test_df = data_df[data_df['is_raw'] == 1]
self.target_df = data_df[data_df['is_raw'] == 0]
ltr = LocationTreeCache()
ltr.main()
def run_agg(apps, schema_editor): ltr = LocationTreeCache() ltr.main() # ensure that aggregation works by running the agg refresh in the migration itself. campaigns = Campaign.objects.all() for campaign in campaigns: if DataPoint.objects.filter(campaign_id = campaign.id).exists(): agg = AggRefresh(campaign.id) agg.main()
def setUp(self):
super(DataPointResourceTest, self).setUp()
# Create a user.
self.username = '******'
self.password = '******'
self.user = User.objects.create_user(self.username,\
'*****@*****.**', self.password)
self.lt = LocationType.objects.create(name='Country', admin_level=0)
self.province_lt = LocationType.objects.create(name='Province'\
,admin_level = 1)
self.district_lt = LocationType.objects.create(name='District'\
,admin_level = 2)
self.o = Office.objects.create(name='Earth')
self.ind = Indicator.objects.create(name='Polio Cases',
short_name='Polio Cases',
data_format='date_int')
self.top_lvl_location = Location.objects.create(
name='Afghanistan',
location_code='Afghanistan',
id=1234,
location_type_id=self.lt.id,
office_id=self.o.id,
)
self.some_province = Location.objects.create(
name='Province',
location_code='Province',
id=432,
parent_location_id=self.top_lvl_location.id,
location_type_id=self.province_lt.id,
office_id=self.o.id,
)
self.some_district = Location.objects.create(
name='Achin',
location_code='Achin',
id=4321,
parent_location_id=self.some_province.id,
location_type_id=self.district_lt.id,
office_id=self.o.id,
)
ltc = LocationTreeCache()
ltc.main()
LocationPermission.objects.create(user_id = self.user.id,\
top_lvl_location_id = self.top_lvl_location.id)
self.get_credentials()
self.create_polio_cases()
def setUp(self):
self.ts = TestSetupHelpers()
data_df = read_csv('rhizome/tests/_data/calc_data.csv')
self.create_metadata()
self.user = User.objects.get(username="******")
self.test_df = data_df[data_df['is_raw'] == 1]
self.target_df = data_df[data_df['is_raw'] == 0]
self.campaign_id = Campaign.objects.all()[0].id
self.top_lvl_location = Location.objects.filter(name='Nigeria')[0]
ltr = LocationTreeCache()
ltr.main()
def setUp(self):
self.ts = TestSetupHelpers()
data_df = read_csv('rhizome/tests/_data/calc_data.csv')
self.create_metadata()
self.user = User.objects.get(username="******")
self.test_df = data_df[data_df['is_raw'] == 1]
self.target_df = data_df[data_df['is_raw'] == 0]
self.campaign_id = Campaign.objects.all()[0].id
self.top_lvl_location = Location.objects.filter(name='Nigeria')[0]
ltr = LocationTreeCache()
ltr.main()
def setUp(self):
## instantiate the test client and all other methods ##
super(LocationResourceTest, self).setUp()
# Create a user.
self.username = '******'
self.password = '******'
self.user = User.objects\
.create_user(self.username,'*****@*****.**', self.password)
self.lt = LocationType.objects.create(name='test',admin_level = 0)
self.o = Office.objects.create(name = 'Earth')
self.not_allowed_to_see_location = Location.objects.create(
name = 'Somalia',
location_code = 'Somalia',
location_type_id = self.lt.id,
office_id = self.o.id,
)
self.top_lvl_location = Location.objects.create(
name = 'Nigeria',
location_code = 'Nigeria',
location_type_id = self.lt.id,
office_id = self.o.id,
)
self.sub_location = Location.objects.create(
name = 'Kano',
location_code = 'Kano',
location_type_id = self.lt.id,
office_id = self.o.id,
parent_location_id = self.top_lvl_location.id
)
### set the user permission ###
LocationPermission.objects.create(user_id = self.user.id,\
top_lvl_location_id = self.top_lvl_location.id)
self.get_credentials()
ltr = LocationTreeCache()
ltr.main()
def test_location_id_and_location_depth(self):
'''
When i pass location_id and depth_level, I should get data for
datapoints underneath the location_id requested at the specified
depth level
'''
# create Afghanistan, region, and provinces
afghanistan = Location.objects.create(
name='Afghanistan',
location_code='Afghanistan',
location_type_id=self.country_lt.id)
south = Location.objects.create(name='South',
location_code='South',
location_type_id=self.region_lt.id,
parent_location_id=afghanistan.id)
kandahar = Location.objects.create(
name='Kandahar',
location_code='Kandahar',
location_type_id=self.province_lt.id,
parent_location_id=south.id)
hilmand = Location.objects.create(name='Hilmand',
location_code='Hilmand',
location_type_id=self.province_lt.id,
parent_location_id=south.id)
ltc = LocationTreeCache()
ltc.main()
indicator = Indicator.objects.create(short_name='stuff', \
name='stuff', \
data_format='int',\
description='some stuff that we want to count', )
start_date = '2014-01-01'
end_date = '2014-01-01'
campaign = Campaign.objects.get(start_date=start_date)
kandahar_value = 27
hilmand_value = 31
# create some datapoints at province level
dp_kandahar = DataPointComputed.objects.create(
location_id=kandahar.id,
value=kandahar_value,
campaign_id=campaign.id,
indicator_id=indicator.id)
dp_hilmand = DataPointComputed.objects.create(
location_id=hilmand.id,
value=hilmand_value,
campaign_id=campaign.id,
indicator_id=indicator.id)
get_parameter = 'indicator__in={0}&campaign__in={1}&location_id={2}&location_depth=2'\
.format(indicator.id, campaign.id, afghanistan.id)
resp = self.api_client.get('/api/v1/campaign_datapoint/?' + get_parameter, \
format='json', authentication=self.get_credentials())
response_data = self.deserialize(resp)
self.assertHttpOK(resp)
self.assertEqual(len(response_data['objects']), 2)
# makes sure we're getting the right dp values, thus confirming that the provinces are returned
sum_of_values = 0
for indicator in response_data['objects']:
sum_of_values += float(indicator['value'])
self.assertEqual(sum_of_values, kandahar_value + hilmand_value)
def test_cache_location_tree(self):
'''
The point of this test is that it the cache_location_tree function
creates database rows that represents each parent / child relationship
between all locations, from the stored information.
That is, if we say that NYC is a sub-location of NYState, and NYState
is a sub-location of U.S.A. , then that means that we should have a row
in the LocationTree table rebresenting the fact that both Nystate AND
U.S.A. are parent_location's on New York City.
'''
location_batch = []
location_type_country = LocationType.objects\
.create(name='Country', admin_level=0, id=1)
location_type_state = LocationType.objects\
.create(name='State', admin_level=1, id=2)
location_type_city = LocationType.objects\
.create(name='City', admin_level=2, id=3)
location_type_city = LocationType.objects\
.create(name='Neighborhood', admin_level=3, id=4)
location_data = {
'id': [1, 2, 3, 4, 5, 6, 7, 8],
'location_type_id': [1, 2, 3, 2, 2, 2, 3, 4],
'name': ['U.S.A.', 'New York State', 'New York City', 'Texas',
'California', 'Maine', 'Portland, ME', 'Brooklyn'],
'parent_location_id': [None, 1, 2, 1, 1, 1, 6, 3]
}
df = DataFrame.from_dict(location_data)
df.set_index('id', inplace=True, drop=False, append=False)
no_nan_df = df.where((notnull(df)), None)
for ix, loc in no_nan_df.iterrows():
location_batch.append(Location(**{
'id': loc.id,
'name': loc.name,
'location_code': unicode(loc.name).replace(' ', ''),
'location_type_id': loc.location_type_id,
'parent_location_id': loc.parent_location_id
}))
Location.objects.bulk_create(location_batch)
## now use the following function to transform the location_tree ##
ltc = LocationTreeCache()
ltc.main()
location_tree_in_db = LocationTree.objects.all()\
.values_list('location_id', 'parent_location_id','lvl')
## every location should have a row in this table with
## lvl = 0 and itself as the parent_location_id
for loc in location_data['id']:
self.assertTrue((loc, loc, 0) in location_tree_in_db)
## new york city (3) has a parent of NY State (2) AND U.S.A (1)##
self.assertTrue((3, 1, 2) in location_tree_in_db)
self.assertTrue((3, 2, 1) in location_tree_in_db)
## Portland ME (7) has a parent of Maine (6) AND U.S.A (1) ##
self.assertTrue((7, 1, 2) in location_tree_in_db)
self.assertTrue((7, 6, 1) in location_tree_in_db)
## new york state and maine are direct children of USA ##
self.assertTrue((2, 1, 1) in location_tree_in_db)
self.assertTrue((6, 1, 1) in location_tree_in_db)
## Brooklkyn has, NYC, NYState and USA as parents ##
self.assertTrue((8, 3, 1) in location_tree_in_db)
self.assertTrue((8, 2, 2) in location_tree_in_db)
self.assertTrue((8, 1, 3) in location_tree_in_db)
def test_cache_location_tree(self):
'''
The point of this test is that it the cache_location_tree function
creates database rows that represents each parent / child relationship
between all locations, from the stored information.
That is, if we say that NYC is a sub-location of NYState, and NYState
is a sub-location of U.S.A. , then that means that we should have a row
in the LocationTree table rebresenting the fact that both Nystate AND
U.S.A. are parent_location's on New York City.
'''
location_batch = []
office = Office.objects.create(name='not important')
location_type_country = LocationType.objects.create(name='Country',\
admin_level=0,id=1)
location_type_state = LocationType.objects.create(name='State',\
admin_level=1,id=2)
location_type_city = LocationType.objects.create(name='City',\
admin_level=2,id=3)
location_type_city = LocationType.objects.create(name='Neighborhood',\
admin_level=3,id=4)
location_data = {
'id':[1,2,3,4,5,6,7,8],
'location_type_id':[1,2,3,2,2,2,3,4],
'name': ['U.S.A.','New York State','New York City','Texas',\
'California','Maine','Portland, ME','Brooklyn'],
'parent_location_id': [None,1,2,1,1,1,6,3]
}
df = DataFrame.from_dict(location_data)
df.set_index('id', inplace=True, drop=False, append=False)
no_nan_df = df.where((notnull(df)), None)
for ix, loc in no_nan_df.iterrows():
location_batch.append(Location(**{
'id':loc.id,
'name':loc.name,
'location_code': unicode(loc.name).replace(' ',''),
'office_id':office.id,
'location_type_id':loc.location_type_id,
'parent_location_id':loc.parent_location_id
}))
Location.objects.bulk_create(location_batch)
## now use the following function to transform the location_tree ##
ltc = LocationTreeCache()
ltc.main()
location_tree_in_db = LocationTree.objects.all()\
.values_list('location_id','parent_location_id')
## the ultimate parent, should have a with itself is parent ##
self.assertTrue((1,1) in location_tree_in_db)
## Brooklyn However should not be the parent of itself ##
self.assertFalse((8,8) in location_tree_in_db)
## new york city has a parent of NY State AND U.S.A ##
self.assertTrue((3,1) in location_tree_in_db)
self.assertTrue((3,2) in location_tree_in_db)
## Portland ME has a parent of Maine AND U.S.A ##
self.assertTrue((7,1) in location_tree_in_db)
self.assertTrue((7,6) in location_tree_in_db)
# ## new york state and maine are children of USA #
self.assertTrue((2,1) in location_tree_in_db)
self.assertTrue((6,1) in location_tree_in_db)
## Brooklkyn has, NYC, NYState and USA as parents ##
self.assertTrue((8,3) in location_tree_in_db)
self.assertTrue((8,2) in location_tree_in_db)
self.assertTrue((8,1) in location_tree_in_db)
def setUp(self):
## instantiate the test client and all other methods ##
super(CampaignResourceTest, self).setUp()
# Create a user.
self.username = '******'
self.password = '******'
self.user = User.objects\
.create_user(self.username,'*****@*****.**', self.password)
self.lt = LocationType.objects.create(name='test',admin_level = 0)
self.o = Office.objects.create(name = 'Earth')
self.not_allowed_to_see_location = Location.objects.create(
name = 'Somalia',
location_code = 'Somalia',
location_type_id = self.lt.id,
office_id = self.o.id,
)
self.top_lvl_location = Location.objects.create(
name = 'Nigeria',
location_code = 'Nigeria',
location_type_id = self.lt.id,
office_id = self.o.id,
)
self.sub_location = Location.objects.create(
name = 'Kano',
location_code = 'Kano',
location_type_id = self.lt.id,
office_id = self.o.id,
parent_location_id = self.top_lvl_location.id
)
self.it = IndicatorTag.objects.create(tag_name='Polio')
self.ct = CampaignType.objects.create(name='NID')
self.can_see_campaign = Campaign.objects.create(
start_date = '2016-01-01',
end_date = '2016-01-01',
office_id = self.o.id,
campaign_type_id = self.ct.id,
top_lvl_location_id = self.top_lvl_location.id,
top_lvl_indicator_tag_id = self.it.id
)
self.can_not_see_campaign = Campaign.objects.create(
start_date = '2016-02-01',
end_date = '2016-02-01',
office_id = self.o.id,
campaign_type_id = self.ct.id,
top_lvl_location_id = self.not_allowed_to_see_location.id,
top_lvl_indicator_tag_id = self.it.id
)
### set the user permission ###
LocationPermission.objects.create(user_id = self.user.id,\
top_lvl_location_id = self.top_lvl_location.id)
self.get_credentials()
ltr = LocationTreeCache()
ltr.main()
def build_location_meta(self):
self.existing_location_map = {
self.file_map['country_display']: self.top_lvl_location.id
}
## PROVINCE ##
## since we ingested the shapes first, we lookup the province name,
## then change it to what the ODK form has.. this allows us to attach
## the shapes to the location IDS from ODK so that they are familiar
## to the owners of the data
province_column = self.file_map['column_map']['province_column']
province_df = pd.DataFrame(self.source_sheet_df[province_column])
for ix, row in province_df.iterrows():
row_dict = row.to_dict()
province_name = row_dict[province_column]
location_obj, created = Location.objects.get_or_create(
name = province_name,
defaults = {
'location_code': province_name,
'parent_location_id': self.top_lvl_location.id,
'location_type_id': LocationType.objects\
.get(name = 'Province').id
})
self.existing_location_map[province_name] = location_obj.id
# DISTRICT ##
district_column = self.file_map['column_map']['district_column']
district_df = pd.DataFrame(\
self.source_sheet_df[[district_column,province_column]])
district_df.drop_duplicates(inplace=True)
self.process_location_df(district_df, 'District')
# ## ADMIN 3 ##
# settlement_column = self.file_map['column_map']['admn_2_column']
# settlement_df = pd.DataFrame(\
# self.source_sheet_df[[settlement_column,district_column]])
#
# settlement_df.drop_duplicates(inplace=True,subset=[settlement_column])
# self.process_location_df(settlement_df, 'Settlement')
#
# ## this wil lmake it so we can ingest data
# source_object_map_batch = [SourceObjectMap(**{
# 'master_object_id': loc.id,
# 'content_type': 'location',
# 'source_object_code': loc.location_code
# }) for loc in Location.objects.all()]
# # SourceObjectMap.objects.bulk_create(source_object_map_batch)
## now let me change the names of the locations
## so that they are familiar to the progam
# for k,v in self.location_lookup.iteritems():
#
# try:
# l = Location.objects.get(name=v)
# l.name = k
# l.location_code = k
# l.save()
# except Location.DoesNotExist: ## LOOK INTO THIS....
# pass
## populate the location tree @
ltc = LocationTreeCache()
ltc.main()
if len(LocationTree.objects.all()) == 0:
raise Exception('Empty Location Tree')
def test_location_type_and_depth(self):
# create Afghanistan, region, and provinces
afghanistan = Location.objects.create(
name='Afghanistan',
location_code ='Afghanistan',
location_type_id =self.lt.id,
office_id = self.o.id
)
south = Location.objects.create(
name='South',
location_code = 'South',
location_type_id=self.region.id,
office_id = self.o.id,
parent_location_id = afghanistan.id
)
kandahar = Location.objects.create(
name='Kandahar',
location_code = 'Kandahar',
location_type_id = self.prov.id,
office_id = self.o.id,
parent_location_id = south.id
)
hilmand = Location.objects.create(
name='Hilmand',
location_code = 'Hilmand',
location_type_id = self.prov.id,
office_id = self.o.id,
parent_location_id = south.id
)
ltc = LocationTreeCache()
ltc.main()
indicator = Indicator.objects.create(short_name='stuff', \
name='stuff', \
data_format='int',\
description='some stuff that we want to count', )
document = Document.objects.create(doc_title='I am Every Woman -- Whitney Houston')
start_date = '2016-01-01'
end_date = '2016-01-01'
campaign_type = CampaignType.objects\
.create(name='National Immunization Days (NID)')
ind_tag = IndicatorTag.objects.create(tag_name='Polio')
campaign = Campaign.objects.create(office=self.o,\
campaign_type=campaign_type,start_date=start_date,end_date=end_date,\
top_lvl_indicator_tag_id = ind_tag.id,\
top_lvl_location_id = afghanistan.id)
kandahar_value =27
hilmand_value =31
# create some datapoints at province level
dp_kandahar = DataPointComputed.objects.create(
location_id = kandahar.id,
value = kandahar_value,
campaign_id = campaign.id,
indicator_id = indicator.id,
document_id = document.id,
)
dp_hilmand = DataPointComputed.objects.create(
location_id = hilmand.id,
value = hilmand_value,
campaign_id = campaign.id,
indicator_id = indicator.id,
document_id = document.id
)
# TRY FOR LOCATION TYPE
# ++++++++++++++++++++++
get_parameter = 'indicator__in={0}&campaign__in={1}&location_id__in={2}&location_type={3}'\
.format(indicator.id, campaign.id, afghanistan.id, self.prov.id)
resp = self.api_client.get('/api/v1/datapoint/?' + get_parameter, \
format='json', authentication=self.get_credentials())
response_data = self.deserialize(resp)
self.assertHttpOK(resp)
self.assertEqual(len(response_data['objects']), 2)
# makes sure we're getting the right dp values, thus confirming that the provinces are returned
sum_of_values = 0
for return_indicator in response_data['objects']:
sum_of_values += float(return_indicator['value'])
self.assertEqual(sum_of_values, kandahar_value+hilmand_value)
# TRY FOR LOCATION DEPTH
# ++++++++++++++++++++++
get_parameter = 'indicator__in={0}&campaign__in={1}&location_id__in={2}&location_depth=2'\
.format(indicator.id, campaign.id, afghanistan.id)
resp = self.api_client.get('/api/v1/datapoint/?' + get_parameter, \
format='json', authentication=self.get_credentials())
response_data = self.deserialize(resp)
self.assertHttpOK(resp)
self.assertEqual(len(response_data['objects']), 2)
# makes sure we're getting the right dp values, thus confirming that the provinces are returned
sum_of_values = 0
for indicator in response_data['objects']:
sum_of_values += float(indicator['value'])
self.assertEqual(sum_of_values, kandahar_value+hilmand_value)
def test_indicator_filter(self):
campaign_id = 2
document = Document.objects.create(doc_title='some doc')
#make a couple different types of indicators, and indicators with different values
indicator_names_to_values = {"LPD Status":[1,2],
'LQAS':[0, 1, 2]
}
indicator_ids_to_values ={}
for indicator_name, values in indicator_names_to_values.iteritems():
indicator = Indicator.objects.create(short_name=indicator_name, \
name=indicator_name, \
description=indicator_name,
data_format ='class' )
indicator_ids_to_values[indicator.id] = values
some_provinces = ['Kandahar', 'Kunar', 'Hilmand', 'Nimroz', 'Sari-Pul', 'Kabul', 'Paktika', 'Ghazni']
ind_id_keys = indicator_ids_to_values.keys()
indicator_to_query = ind_id_keys[1]
# choose which indicator/value pair to filter by, and keep track of dps that match this as they're created
indicator_to_filter = ind_id_keys[0]
indicator_val_to_filter = indicator_ids_to_values[indicator_to_filter][0]
dps_to_track =[]
for province in some_provinces:
# create the province
prov = Location.objects.create(
name = province,
location_code = province,
location_type_id = self.prov.id,
office_id = self.o.id,
parent_location_id = self.top_lvl_location.id
)
# create a random dp for each indicator
for indicator_id, values in indicator_ids_to_values.iteritems():
idx = randint(0, len(values)-1)
value_to_use = values[idx]
dp = DataPointComputed.objects.create(
location_id = prov.id,
value = value_to_use,
campaign_id = campaign_id,
indicator_id = indicator_id,
document_id = document.id
)
if indicator_id == indicator_to_filter and value_to_use == indicator_val_to_filter:
dps_to_track.append(dp)
ltc = LocationTreeCache()
ltc.main()
indicator_name_to_filter = Indicator.objects.get(id=indicator_to_filter);
get_parameter = 'indicator__in={0}&campaign__in={1}&location_id__in={2}&location_depth=2&filter_indicator={3}&filter_value={4}&chart_type=TableChart'\
.format(indicator_to_query, campaign_id, self.top_lvl_location.id, indicator_name_to_filter, indicator_val_to_filter)
resp = self.api_client.get('/api/v1/datapoint/?' + get_parameter, \
format='json', authentication=self.get_credentials())
response_data = self.deserialize(resp)
self.assertEqual(len(response_data['objects']), len(dps_to_track))
# oof, nested for loop is okay since it's a small dataset
# makes sure that all the campaign and location ids match
for dp in dps_to_track:
found_dp = False
for resp_dp in response_data['objects']:
same_campaign = int(resp_dp['campaign_id']) == dp.campaign_id
same_location = int(resp_dp['location_id']) == dp.location_id
if same_location and same_campaign:
found_dp = True
if not found_dp:
fail("the datapoints from the respnse do not match the system")
break
pass
