def __init__(self, region, resolution, variant, is_custom=False, cache_dir="./cache", output_dir="./data", fast_mode=False):
common.Base.__init__(self, region, resolution, cache_dir)
self.output_dir = output_dir
self.fast_mode = fast_mode
self.variant = variant
self.is_custom = is_custom
# init the population (projections) modules
self.mye_api = myedata.MYEData(cache_dir)
self.npp_api = nppdata.NPPData(cache_dir)
if self.is_custom:
if variant not in customsnppdata.list_custom_projections(cache_dir):
raise ValueError("Requested custom SNPP %s is not in the cache directory (%s)" % (variant, cache_dir))
print("Using custom SNPP variant %s" % variant)
print("NOTE: assuming custom SNPP variant disables rescaling to national variant")
self.snpp_api = customsnppdata.CustomSNPPData(variant, cache_dir)
else:
self.snpp_api = snppdata.SNPPData(cache_dir)
# validation
if not is_custom and self.variant not in nppdata.NPPData.VARIANTS:
raise ValueError(self.variant + " is not a known projection variant")
if not isinstance(self.fast_mode, bool):
raise ValueError("fast mode should be boolean")
# TODO enable 2001 ref year?
# (down)load the census 2011 tables
self.__get_census_data()
def fetch_dummy_data_into_raw():
if not os.environ['NOMIS_API_KEY'] == "DUMMY":
print("This Function requires to be NOMIS_API_KEY == 'DUMMY' in env.\n"
"Currently set to {} ".format(os.environ['NOMIS_API_KEY']))
sys.exit()
NPPData.NPPData(test_data_dir)
MYEData.MYEData(test_data_dir)
SNPPData.SNPPData(test_data_dir)
SNHPData.SNHPData(test_data_dir)
def setUp(self):
"""
Check env set up correctly for tests
(it's too late to override the env in this function unfortunately)
"""
self.mye = MYEData.MYEData(TEST_DATA_DIR)
self.npp = NPPData.NPPData(TEST_DATA_DIR)
self.snpp = SNPPData.SNPPData(TEST_DATA_DIR)
self.snhp = SNHPData.SNHPData(TEST_DATA_DIR)
# fix issue with test dataset
self.snpp.data[utils.EN].PROJECTED_YEAR_NAME = self.snpp.data[utils.EN].PROJECTED_YEAR_NAME.astype(int)
def setUp(self):
"""
Check env set up correctly for tests
(it's too late to override the env in this function unfortunately)
"""
self.mye = MYEData.MYEData("./tests/raw_data")
self.npp = NPPData.NPPData("./tests/raw_data")
self.snpp = SNPPData.SNPPData("./tests/raw_data")
if not self.npp.data_api.key == "DUMMY" or not self.snpp.data_api.key == "DUMMY":
print("Test requires NOMIS_API_KEY=DUMMY in env")
sys.exit()
def __init__(self, region, resolution, variant, cache_dir="./cache", output_dir="./data", fast_mode=False):
common.Base.__init__(self, region, resolution, cache_dir)
self.output_dir = output_dir
self.fast_mode = fast_mode
self.variant = variant
# init the population (projections) modules
self.mye_api = myedata.MYEData(cache_dir)
self.npp_api = nppdata.NPPData(cache_dir)
self.snpp_api = snppdata.SNPPData(cache_dir)
# validation
if not self.variant in nppdata.NPPData.VARIANTS:
raise ValueError(self.variant + " is not a known projection variant")
if not isinstance(self.fast_mode, bool):
raise ValueError("fast mode should be boolean")
# TODO enable 2001 ref year?
# (down)load the census 2011 tables
self.__get_census_data()
def setUp(self):
"""
Check env set up correctly for tests. It's too late to override the env in this function unfortunately.
"""
print(
"Warning: Some SNPP tests are disabled temporarily for the sake of development of the new dynamic "
"microsimulation but the code works")
# Build the test data objects from the raw_data directory.
self.mye = MYEData.MYEData(
TEST_DATA_DIR
) # Needs to be complete data for tests when upgrading to new estimates.
self.npp = NPPData.NPPData(
TEST_DATA_DIR) # Need to build the test version every migration.
self.snpp = SNPPData.SNPPData(
TEST_DATA_DIR) # Need to build the test version every migration.
self.snhp = SNHPData.SNHPData(TEST_DATA_DIR)
# fix issue with test dataset
self.snpp.data[utils.EN].PROJECTED_YEAR_NAME = self.snpp.data[
utils.EN].PROJECTED_YEAR_NAME.astype(int)
if not self.npp.data_api.key == "DUMMY" or not self.snpp.data_api.key == "DUMMY":
print("Test requires NOMIS_API_KEY=DUMMY in env")
sys.exit()
def __init__(self, params):
self.coverage = {
"EW": ukpoputils.EW,
"GB": ukpoputils.GB,
"UK": ukpoputils.UK
}.get(params["coverage"])
if not self.coverage:
raise RuntimeError("invalid coverage: %s" % params["coverage"])
self.cache_dir = params["cache_dir"]
# initialise data sources
self.census_ew = Nomisweb.Nomisweb(self.cache_dir)
self.census_sc = NRScotland.NRScotland(self.cache_dir)
self.census_ni = NISRA.NISRA(self.cache_dir)
# population projections
self.mye = MYEData.MYEData(self.cache_dir)
self.snpp = SNPPData.SNPPData(self.cache_dir)
self.npp = NPPData.NPPData(self.cache_dir)
# households
self.baseline = params["base_projection"]
if not os.path.isdir(params["output_dir"]):
raise ValueError("Output directory %s not found" %
params["output_dir"])
self.output_file = os.path.join(
params["output_dir"], "simim_%s_%s_%s" %
(params["model_type"], params["base_projection"],
os.path.basename(params["scenario"])))
self.custom_snpp_variant = pd.DataFrame()
self.snhp = SNHPData.SNHPData(self.cache_dir)
# holder for shapefile when requested
self.shapefile = None
import matplotlib.pyplot as plt
import ukpopulation.myedata as MYEData
import ukpopulation.nppdata as NPPData
import ukpopulation.snppdata as SNPPData
import ukpopulation.utils as utils
# initialise the population modules
mye = MYEData.MYEData()
npp = NPPData.NPPData()
snpp = SNPPData.SNPPData()
lad = "E07000041" # Exeter
# 50 year horizon
years = range(2011, 2067)
(mye_years, proj_years) = utils.split_range(years, mye.max_year())
(snpp_years, npp_years) = utils.split_range(proj_years, snpp.max_year(lad))
pop_mye = mye.aggregate(["GENDER", "C_AGE"], lad, mye_years)
# get the total projected population for pop up to the SNPP horizon (2039)
pop = snpp.aggregate(["GENDER", "C_AGE"], lad, snpp_years)
# extrapolate for another ~25 years
pop_ex = snpp.extrapolagg(["GENDER", "C_AGE"], npp, lad, npp_years)
# plot the data
plt.plot(pop_mye.PROJECTED_YEAR_NAME, pop_mye.OBS_VALUE, "b^", label="MYE")
plt.plot(pop.PROJECTED_YEAR_NAME, pop.OBS_VALUE, "bo", label="SNPP")
plt.plot(pop_ex.PROJECTED_YEAR_NAME, pop_ex.OBS_VALUE, "b.", label="ext NPP")
plt.xlabel("Year")
plt.ylabel("Persons")
def fetch_full_data_into_cache():
npp = NPPData.NPPData()
npp.force_load_variants(['hhh', 'ppp', 'lll'])
MYEData.MYEData()
SNPPData.SNPPData()
SNHPData.SNHPData()
