def __init__(self, region, resolution, variant, is_custom=False, cache_dir="./cache", output_dir="./data", fast_mode=False):
common.Base.__init__(self, region, resolution, cache_dir)
self.output_dir = output_dir
self.fast_mode = fast_mode
self.variant = variant
self.is_custom = is_custom
# init the population (projections) modules
self.mye_api = myedata.MYEData(cache_dir)
self.npp_api = nppdata.NPPData(cache_dir)
if self.is_custom:
if variant not in customsnppdata.list_custom_projections(cache_dir):
raise ValueError("Requested custom SNPP %s is not in the cache directory (%s)" % (variant, cache_dir))
print("Using custom SNPP variant %s" % variant)
print("NOTE: assuming custom SNPP variant disables rescaling to national variant")
self.snpp_api = customsnppdata.CustomSNPPData(variant, cache_dir)
else:
self.snpp_api = snppdata.SNPPData(cache_dir)
# validation
if not is_custom and self.variant not in nppdata.NPPData.VARIANTS:
raise ValueError(self.variant + " is not a known projection variant")
if not isinstance(self.fast_mode, bool):
raise ValueError("fast mode should be boolean")
# TODO enable 2001 ref year?
# (down)load the census 2011 tables
self.__get_census_data()
def fetch_dummy_data_into_raw():
if not os.environ['NOMIS_API_KEY'] == "DUMMY":
print("This Function requires to be NOMIS_API_KEY == 'DUMMY' in env.\n"
"Currently set to {} ".format(os.environ['NOMIS_API_KEY']))
sys.exit()
NPPData.NPPData(test_data_dir)
MYEData.MYEData(test_data_dir)
SNPPData.SNPPData(test_data_dir)
SNHPData.SNHPData(test_data_dir)
def get_population_data(geogs):
# mye = MYEData.MYEData()
# mye2108 = mye.filter(geogs, 2018)
# print(mye2108.head())
snpp = SNPPData.SNPPData()
snpp_syoa = snpp.filter(geogs, 2020).drop("PROJECTED_YEAR_NAME", axis=1)
# got from single year of age to groups
return map_ages(snpp_syoa)
def setUp(self):
"""
Check env set up correctly for tests
(it's too late to override the env in this function unfortunately)
"""
self.mye = MYEData.MYEData(TEST_DATA_DIR)
self.npp = NPPData.NPPData(TEST_DATA_DIR)
self.snpp = SNPPData.SNPPData(TEST_DATA_DIR)
self.snhp = SNHPData.SNHPData(TEST_DATA_DIR)
# fix issue with test dataset
self.snpp.data[utils.EN].PROJECTED_YEAR_NAME = self.snpp.data[utils.EN].PROJECTED_YEAR_NAME.astype(int)
def setUp(self):
"""
Check env set up correctly for tests
(it's too late to override the env in this function unfortunately)
"""
self.mye = MYEData.MYEData("./tests/raw_data")
self.npp = NPPData.NPPData("./tests/raw_data")
self.snpp = SNPPData.SNPPData("./tests/raw_data")
if not self.npp.data_api.key == "DUMMY" or not self.snpp.data_api.key == "DUMMY":
print("Test requires NOMIS_API_KEY=DUMMY in env")
sys.exit()
def __init__(self, region, resolution, variant, cache_dir="./cache", output_dir="./data", fast_mode=False):
common.Base.__init__(self, region, resolution, cache_dir)
self.output_dir = output_dir
self.fast_mode = fast_mode
self.variant = variant
# init the population (projections) modules
self.mye_api = myedata.MYEData(cache_dir)
self.npp_api = nppdata.NPPData(cache_dir)
self.snpp_api = snppdata.SNPPData(cache_dir)
# validation
if not self.variant in nppdata.NPPData.VARIANTS:
raise ValueError(self.variant + " is not a known projection variant")
if not isinstance(self.fast_mode, bool):
raise ValueError("fast mode should be boolean")
# TODO enable 2001 ref year?
# (down)load the census 2011 tables
self.__get_census_data()
def setUp(self):
"""
Check env set up correctly for tests. It's too late to override the env in this function unfortunately.
"""
print(
"Warning: Some SNPP tests are disabled temporarily for the sake of development of the new dynamic "
"microsimulation but the code works")
# Build the test data objects from the raw_data directory.
self.mye = MYEData.MYEData(
TEST_DATA_DIR
) # Needs to be complete data for tests when upgrading to new estimates.
self.npp = NPPData.NPPData(
TEST_DATA_DIR) # Need to build the test version every migration.
self.snpp = SNPPData.SNPPData(
TEST_DATA_DIR) # Need to build the test version every migration.
self.snhp = SNHPData.SNHPData(TEST_DATA_DIR)
# fix issue with test dataset
self.snpp.data[utils.EN].PROJECTED_YEAR_NAME = self.snpp.data[
utils.EN].PROJECTED_YEAR_NAME.astype(int)
if not self.npp.data_api.key == "DUMMY" or not self.snpp.data_api.key == "DUMMY":
print("Test requires NOMIS_API_KEY=DUMMY in env")
sys.exit()
def __init__(self, params):
self.coverage = {
"EW": ukpoputils.EW,
"GB": ukpoputils.GB,
"UK": ukpoputils.UK
}.get(params["coverage"])
if not self.coverage:
raise RuntimeError("invalid coverage: %s" % params["coverage"])
self.cache_dir = params["cache_dir"]
# initialise data sources
self.census_ew = Nomisweb.Nomisweb(self.cache_dir)
self.census_sc = NRScotland.NRScotland(self.cache_dir)
self.census_ni = NISRA.NISRA(self.cache_dir)
# population projections
self.mye = MYEData.MYEData(self.cache_dir)
self.snpp = SNPPData.SNPPData(self.cache_dir)
self.npp = NPPData.NPPData(self.cache_dir)
# households
self.baseline = params["base_projection"]
if not os.path.isdir(params["output_dir"]):
raise ValueError("Output directory %s not found" %
params["output_dir"])
self.output_file = os.path.join(
params["output_dir"], "simim_%s_%s_%s" %
(params["model_type"], params["base_projection"],
os.path.basename(params["scenario"])))
self.custom_snpp_variant = pd.DataFrame()
self.snhp = SNHPData.SNHPData(self.cache_dir)
# holder for shapefile when requested
self.shapefile = None
import matplotlib.pyplot as plt
import ukpopulation.snppdata as SNPPData
import ukpopulation.snhpdata as SNHPData
#import ukpopulation.utils as utils
# initialise the population modules
snhp = SNHPData.SNHPData()
snpp = SNPPData.SNPPData()
lad = "E08000021" # Newcastle
start_year = 2016
end_year = snhp.max_year(lad)
# get the total
hh = snhp.aggregate(lad, range(start_year, end_year + 1))
p = snpp.aggregate(["C_AGE", "GENDER"], lad, range(start_year, end_year + 1))
# plot the data
fig, ax1 = plt.subplots()
ax1.plot(hh.PROJECTED_YEAR_NAME, hh.OBS_VALUE, "bo", label="households")
ax1.set_xlabel("Year")
ax1.set_ylabel("Households")
ax1.legend()
ax2 = ax1.twinx()
ax2.plot(p.PROJECTED_YEAR_NAME, p.OBS_VALUE, "ro", label="people")
ax2.set_xlabel("Year")
ax2.set_ylabel("People")
ax2.legend(loc=4)
plt.title(lad + " Households/People Projections")
plt.show()
def fetch_full_data_into_cache():
npp = NPPData.NPPData()
npp.force_load_variants(['hhh', 'ppp', 'lll'])
MYEData.MYEData()
SNPPData.SNPPData()
SNHPData.SNHPData()