def main(): api = Api.Nomisweb("/tmp/UKCensusAPI") print("Nomisweb census data geographical query example") print("See README.md for details on how to use this package") # Heres predefined query on a small geographical area table = "KS401EW" table_internal = "NM_618_1" query_params = {} query_params["CELL"] = "7...13" query_params["date"] = "latest" query_params["RURAL_URBAN"] = "0" query_params["select"] = "GEOGRAPHY_CODE,CELL,OBS_VALUE" query_params["geography"] = "1245710558...1245710560" query_params["MEASURES"] = "20100" ks401 = api.get_data(table, table_internal, query_params) # display the first ten rows print(ks401.head(10)) # Now add context - the desriptions of the values (7 to 13) in the CELL column api.contextify(table, "CELL", ks401) print(ks401.head(10)) ks401.to_csv("/tmp/contextified", sep="\t")
def __init__(self, cache_dir=None): if cache_dir is None: cache_dir = utils.default_cache_dir() self.cache_dir = cache_dir self.data_api = Api.Nomisweb(self.cache_dir) # store as a dictionary keyed by year (lazy retrieval) self.data = {}
def __init__(self, cache_dir=utils.default_cache_dir()): self.cache_dir = cache_dir self.data_api = Api.Nomisweb(self.cache_dir) self.data = {} self.data[utils.EN] = self.__do_england() self.data[utils.WA] = self.__do_wales() self.data[utils.SC] = self.__do_scotland() self.data[utils.NI] = self.__do_nireland()
def __init__(self, cache_dir = None): if cache_dir is None: cache_dir = utils.default_cache_dir() self.cache_dir = cache_dir self.data_api = Api.Nomisweb(self.cache_dir) # map of pandas dataframes keyed by variant code self.data = {} # load principal aggressively... self.data["ppp"] = self.__download_ppp()
def main(): api = Api.Nomisweb("/tmp/UKCensusAPI") print("Nomisweb census data geographical query example") print("See README.md for details on how to use this package") # In the previous example we had a predefined query using Leeds at MSOA resolution, # but we want to expand the geographical area and refine the resolution table = "KS401EW" table_internal = "NM_618_1" query_params = {} query_params["CELL"] = "7...13" query_params["date"] = "latest" query_params["RURAL_URBAN"] = "0" query_params["select"] = "GEOGRAPHY_CODE,CELL,OBS_VALUE" query_params[ "geography"] = "1245710558...1245710660,1245714998...1245714998,1245715007...1245715007,1245715021...1245715022" query_params["MEASURES"] = "20100" # Define the new coverage area in terms of local authorities coverage = ["Leeds", "Bradford"] # Define the new resolution resolution = Api.Nomisweb.OA # Convert the coverage area into nomis codes coverage_codes = api.get_lad_codes(coverage) # replace the geography value in the query query_params["geography"] = api.get_geo_codes(coverage_codes, resolution) # get the data ks401fine = api.get_data(table, table_internal, query_params) print(ks401fine.head(5)) # Now widen the coverage to England & Wales and coarsen the resolution to LA coverage_codes = [Api.Nomisweb.EnglandWales] resolution = Api.Nomisweb.LAD query_params["geography"] = api.get_geo_codes(coverage_codes, resolution) # get the data ks401broad = api.get_data(table, table_internal, query_params) print(ks401broad.head(5))
def __init__(self, params): self.coverage = { "EW": ukpoputils.EW, "GB": ukpoputils.GB, "UK": ukpoputils.UK }.get(params["coverage"]) if not self.coverage: raise RuntimeError("invalid coverage: %s" % params["coverage"]) self.cache_dir = params["cache_dir"] # initialise data sources self.census_ew = Nomisweb.Nomisweb(self.cache_dir) self.census_sc = NRScotland.NRScotland(self.cache_dir) self.census_ni = NISRA.NISRA(self.cache_dir) # population projections self.mye = MYEData.MYEData(self.cache_dir) self.snpp = SNPPData.SNPPData(self.cache_dir) self.npp = NPPData.NPPData(self.cache_dir) # households self.baseline = params["base_projection"] if not os.path.isdir(params["output_dir"]): raise ValueError("Output directory %s not found" % params["output_dir"]) self.output_file = os.path.join( params["output_dir"], "simim_%s_%s_%s" % (params["model_type"], params["base_projection"], os.path.basename(params["scenario"]))) self.custom_snpp_variant = pd.DataFrame() self.snhp = SNHPData.SNHPData(self.cache_dir) # holder for shapefile when requested self.shapefile = None
def get_census_data(geogs): api = CensusApi.Nomisweb("~/.ukcensusapi") nomis_lad_codes = api.get_lad_codes(geogs) nomis_msoa_codes = api.get_geo_codes( nomis_lad_codes, CensusApi.Nomisweb.GeoCodeLookup["MSOA11"]) # "C_ETHPUK11": { # "1": "White: Total", # "6": "Mixed/multiple ethnic group: Total", # "11": "Asian/Asian British: Total", # "17": "Black/African/Caribbean/Black British: Total", # "21": "Other ethnic group: Total", # }, table = "DC2101EW" query_params = { "date": "latest", "select": "GEOGRAPHY_CODE,C_SEX,C_AGE,C_ETHPUK11,OBS_VALUE", "C_SEX": "1,2", "C_AGE": "1...21", "C_ETHPUK11": "1,6,11,17,21", "MEASURES": "20100", } query_params["geography"] = nomis_msoa_codes dc2101ew_msoa = api.get_data(table, query_params) query_params["geography"] = nomis_lad_codes dc2101ew_lad = api.get_data(table, query_params) # ensure geogs are consistent assert dc2101ew_msoa["OBS_VALUE"].sum() == dc2101ew_lad["OBS_VALUE"].sum() return dc2101ew_msoa, dc2101ew_lad
def __init__(self, region, resolution, cache_dir="./cache"): self.api_ew = Api_ew.Nomisweb(cache_dir) self.api_sc = Api_sc.NRScotland(cache_dir) self.region = region # convert input string to enum self.resolution = resolution # Temporary workaround for unavailable Scottish data self.scotland = False if self.region[0] == "S": self.scotland = True print("Running in 'scotland mode'") # (down)load the census tables self.__get_census_data() # initialise table and index categories = [ "Area", "LC4402_C_TYPACCOM", "QS420_CELL", "LC4402_C_TENHUK11", "LC4408_C_AHTHUK11", "CommunalSize", "LC4404_C_SIZHUK11", "LC4404_C_ROOMS", "LC4405EW_C_BEDROOMS", "LC4408EW_C_PPBROOMHEW11", "LC4402_C_CENHEATHUK11", "LC4605_C_NSSEC", "LC4202_C_ETHHUK11", "LC4202_C_CARSNO" ] self.total_dwellings = sum(self.ks401.OBS_VALUE) + sum( self.communal.OBS_VALUE) # self.dwellings = pd.DataFrame(index=range(0, self.total_dwellings), columns=categories) self.dwellings = pd.DataFrame(columns=categories) self.index = 0 # generate indices self.type_index = self.lc4402.C_TYPACCOM.unique() self.tenure_index = self.lc4402.C_TENHUK11.unique() self.ch_index = self.lc4402.C_CENHEATHUK11.unique() self.comp_index = self.lc4408.C_AHTHUK11.unique()
""" import pandas as pd import ukpopulation.snppdata as SNPPData snpp = SNPPData.SNPPData() leeds=snpp.aggregate(["GENDER"], "E08000035", range(2018,2050), ages=range(16,91)) leeds.head() leeds.to_csv("PopulationProjection.csv") import ukcensusapi.Nomisweb as CensusApi import ukpopulation.snppdata as SNPPData otley = ["E02002185", "E02002187", "E02002332", "E02002333", "E02002336", "E02002337", "E02002338", "E02002339", "E02002340", "E02002342", "E02002343", "E02002345", "E02002346","E02002348", "E02002350", "E02002356", "E02002357"] leeds = "E08000035" year = 2018 census_api = CensusApi.Nomisweb("./cache") table = "KS102EW" query_params = { "date": "latest", "select": "GEOGRAPHY_CODE,CELL,OBS_VALUE", "RURAL_URBAN": "0", "CELL": "6...16", # age groups from 16 tp 90+ "MEASURES": "20100", "geography": "1245710558...1245710660,1245714998...1245714998,1245715007...1245715007,1245715021...1245715022" } # aggregate age groups and sum leedspop = census_api.get_data(table, query_params).groupby(["GEOGRAPHY_CODE"]).sum().drop("CELL", axis=1) # work out proportion of Leeds population for the 2 MSOAs leedspop["PROP"] = leedspop.OBS_VALUE / sum(leedspop.OBS_VALUE)
class Test(TestCase): """ Test harness """ cache_dir = "/tmp/UKCensusAPI" api_ew = Api_EW.Nomisweb(cache_dir, verbose=True) api_sc = Api_SC.NRScotland(cache_dir) api_ni = Api_NI.NISRA(cache_dir) query = Census.Query(cache_dir) def test_get_lad_codes(self): self.assertTrue(self.api_ew.get_lad_codes("Royston Vasey") == []) self.assertTrue(self.api_ew.get_lad_codes("Leeds") == [1946157127]) self.assertTrue( self.api_ew.get_lad_codes(["Leeds", "Bradford"]) == [1946157127, 1946157124]) def test_cache_dir_invalid(self): self.assertRaises(PermissionError, Api_EW.Nomisweb, "/home/invalid") self.assertRaises(PermissionError, Api_SC.NRScotland, "/bin") self.assertRaises(PermissionError, Api_NI.NISRA, "/bin/ls") # This overlaps test_getGeographyFromCodes def test_geo_codes_ew(self): result = self.api_ew.get_geo_codes( [Api_EW.Nomisweb.GeoCodeLookup["EnglandWales"]], Api_EW.Nomisweb.GeoCodeLookup["LAD"]) self.assertEqual(result, '1946157057...1946157404') result = self.api_ew.get_geo_codes( [1946157127], Api_EW.Nomisweb.GeoCodeLookup["OA11"]) self.assertEqual( result, '1254151943...1254154269,1254258198...1254258221,1254261711...1254261745,1254261853...1254261870,1254261894...1254261918,1254262125...1254262142,1254262341...1254262353,1254262394...1254262398,1254262498...1254262532,1254262620...1254262658,1254262922...1254262925' ) # test 2001 codes result = self.api_ew.get_geo_codes( [1946157127], Api_EW.Nomisweb.GeoCodeLookup["MSOA01"]) self.assertEqual(result, '1279265050...1279265157') def test_geo_codes_sc(self): lads = sorted([ 'S12000033', 'S12000034', 'S12000041', 'S12000035', 'S12000026', 'S12000005', 'S12000039', 'S12000006', 'S12000042', 'S12000008', 'S12000045', 'S12000010', 'S12000011', 'S12000036', 'S12000014', 'S12000015', 'S12000046', 'S12000017', 'S12000018', 'S12000019', 'S12000020', 'S12000021', 'S12000044', 'S12000023', 'S12000024', 'S12000038', 'S12000027', 'S12000028', 'S12000029', 'S12000030', 'S12000040', 'S12000013' ]) msoa_ab = sorted([ 'S02001275', 'S02001238', 'S02001237', 'S02001236', 'S02001278', 'S02001284', 'S02001247', 'S02001249', 'S02001246', 'S02001250', 'S02001261', 'S02001252', 'S02001251', 'S02001257', 'S02001258', 'S02001254', 'S02001256', 'S02001253', 'S02001248', 'S02001242', 'S02001240', 'S02001241', 'S02001239', 'S02001243', 'S02001259', 'S02001260', 'S02001264', 'S02001265', 'S02001268', 'S02001269', 'S02001267', 'S02001274', 'S02001266', 'S02001263', 'S02001262', 'S02001245', 'S02001270', 'S02001272', 'S02001273', 'S02001271', 'S02001244', 'S02001279', 'S02001282', 'S02001281', 'S02001280', 'S02001276', 'S02001277', 'S02001283', 'S02001255' ]) result = sorted(self.api_sc.get_geog("S92000003", "LAD")) self.assertTrue(np.array_equal(result, lads)) result = sorted(self.api_sc.get_geog("S12000033", "MSOA11")) self.assertTrue(np.array_equal(result, msoa_ab)) result = self.api_sc.get_geog("S12000033", "LSOA11") self.assertEqual(len(result), 283) result = self.api_sc.get_geog("S12000033", "OA11") self.assertEqual(len(result), 1992) def test_geo_codes_ni(self): # NI data lads = [ '95AA', '95BB', '95CC', '95DD', '95EE', '95FF', '95GG', '95HH', '95II', '95JJ', '95KK', '95LL', '95MM', '95NN', '95OO', '95PP', '95QQ', '95RR', '95SS', '95TT', '95UU', '95VV', '95WW', '95XX', '95YY', '95ZZ' ] msoa_95aa = [ '95AA01', '95AA02', '95AA03', '95AA04', '95AA05', '95AA06', '95AA07', '95AA08', '95AA09', '95AA10', '95AA11', '95AA12', '95AA13', '95AA14', '95AA15', '95AA16', '95AA17', '95AA18', '95AA19' ] lsoa_95aa = [ '95AA01S1', '95AA01S2', '95AA01S3', '95AA02W1', '95AA03W1', '95AA04W1', '95AA05W1', '95AA06S1', '95AA06S2', '95AA07W1', '95AA08W1', '95AA09W1', '95AA10W1', '95AA11S1', '95AA11S2', '95AA12W1', '95AA13S1', '95AA13S2', '95AA14W1', '95AA15S1', '95AA15S2', '95AA16W1', '95AA17W1', '95AA18W1', '95AA19W1' ] result = sorted(self.api_ni.get_geog("N92000002", "LAD")) self.assertTrue(np.array_equal(result, lads)) result = sorted(self.api_ni.get_geog("95AA", "MSOA11")) self.assertTrue(np.array_equal(result, msoa_95aa)) result = sorted(self.api_ni.get_geog("95AA", "LSOA11")) self.assertTrue(np.array_equal(result, lsoa_95aa)) result = sorted(self.api_ni.get_geog("95AA", "OA11")) self.assertEqual(len(result), 129) def test_get_metadata_ew(self): meta = self.api_ew.get_metadata("NONEXISTENT") self.assertFalse(meta) meta = self.api_ew.get_metadata("KS401EW") self.assertEqual( meta["description"], 'KS401EW - Dwellings, household spaces and accommodation type') self.assertEqual(meta["nomis_table"], 'NM_618_1') # test 2001 table meta = self.api_ew.get_metadata("UV070") self.assertEqual(meta["description"], 'UV070 - Communal Establishments') self.assertEqual(meta["nomis_table"], 'NM_1686_1') def test_get_metadata_sc(self): # Scotland meta = self.api_sc.get_metadata("KS401SC", "LAD") self.assertEqual(meta["table"], 'KS401SC') self.assertEqual(meta["geography"], 'LAD') self.assertTrue('KS401SC_0_CODE' in meta["fields"]) def test_get_metadata_ni(self): # NI meta = self.api_ni.get_metadata("QS401NI", "LSOA11") self.assertEqual(meta["table"], 'QS401NI') self.assertEqual(meta["geography"], 'SOA') self.assertTrue('QS401NI_0_CODE' in meta["fields"]) self.assertEqual(len(meta["fields"]['QS401NI_0_CODE']), 12) def test_get_url(self): table = "NM_618_1" query_params = {} query_params["CELL"] = "7...13" query_params["date"] = "latest" query_params["RURAL_URBAN"] = "0" query_params["select"] = "GEOGRAPHY_CODE,CELL,OBS_VALUE" query_params[ "geography"] = "1245710558...1245710660,1245714998...1245714998,1245715007...1245715007,1245715021...1245715022" query_params["MEASURES"] = "20100" self.assertEqual( self.api_ew.get_url(table, query_params), "https://www.nomisweb.co.uk/api/v01/dataset/NM_618_1.data.tsv?CELL=7...13&MEASURES=20100&RURAL_URBAN=0&date=latest&geography=1245710558...1245710660%2C1245714998...1245714998%2C1245715007...1245715007%2C1245715021...1245715022&select=GEOGRAPHY_CODE%2CCELL%2COBS_VALUE" ) def test_get_data_ew(self): table_name = "KS401EW" # table_internal = "NM_618_1" query_params = {} query_params["CELL"] = "7...13" query_params["date"] = "latest" query_params["RURAL_URBAN"] = "0" query_params["select"] = "GEOGRAPHY_CODE,CELL,OBS_VALUE" query_params["geography"] = "1245710558...1245710560" query_params["MEASURES"] = "20100" table = self.api_ew.get_data(table_name, query_params) self.assertEqual(table.shape, (21, 3)) self.assertEqual(sum(table.OBS_VALUE), 8214) def test_get_data_sc(self): table_name = "KS401SC" geography = "S12000033" # Aberdeen categories = {"KS401SC_0_CODE": range(8, 15)} table = self.api_sc.get_data(table_name, geography, "LAD", categories) self.assertEqual(table.shape, (7, 3)) self.assertEqual(sum(table.OBS_VALUE), 108153) table_name = "DC2101SC" geography = "S12000033" # Aberdeen categories = { "DC2101SC_0_CODE": 4, # White Irish "DC2101SC_1_CODE": [1, 2], # M+F "DC2101SC_2_CODE": [6, 7, 8, 9, 10, 11, 12] } # 18-49 table = self.api_sc.get_data(table_name, geography, "LAD", categories) self.assertEqual(table.shape, (14, 5)) self.assertEqual(sum(table.OBS_VALUE), 1732) def test_get_data_ni(self): table_name = "QS401NI" geography = "95AA" # Antrim categories = {"QS401NI_0_CODE": [1, 6, 8, 9, 10]} table = self.api_ni.get_data(table_name, geography, "LAD", categories) self.assertEqual(table.shape, (5, 3)) self.assertEqual(sum(table.OBS_VALUE), 52454) table_name = "QS202NI" geography = "95ZZ" # Strabane categories = {"QS202NI_0_CODE": [1, 2, 3, 4, 5, 6]} table = self.api_ni.get_data(table_name, geography, "MSOA11", categories) self.assertEqual(table.shape, (96, 3)) self.assertEqual(sum(table.OBS_VALUE), 14817) #'table': 'QS202NI', 'description': '', 'geography': 'SOA', 'fields': {'QS202NI_0_CODE': {0: 'All Household Reference Persons (HRPs)', 1: 'Ethnic group of HRP: Black', 2: 'Ethnic group of HRP: Chinese', 3: 'Ethnic group of HRP: Mixed', 4: 'Ethnic group of HRP: Other', 5: 'Ethnic group of HRP: Other Asian', 6: 'Ethnic group of HRP: White'}}} # OD data is structured differently def test_get_od_data(self): table = "WF01BEW" # table_internal = "NM_1228_1" query_params = {} query_params["date"] = "latest" query_params[ "select"] = "currently_residing_in_code,place_of_work_code,OBS_VALUE" # OD are 5 LSOAs in central Leeds query_params[ "currently_residing_in"] = "1249934756...1249934758,1249934760,1249934761" query_params[ "place_of_work"] = "1249934756...1249934758,1249934760,1249934761" query_params["MEASURES"] = "20100" table = self.api_ew.get_data(table, query_params) self.assertEqual(table.shape, (25, 3)) self.assertEqual(sum(table.OBS_VALUE), 1791) # Projection data doesnt explicitly have a table name - tests directly specifying nomis internal name def test_get_proj_data(self): table_internal = "NM_2002_1" query_params = { "gender": "1,2", "c_age": "101...191", "MEASURES": "20100", "select": "geography_code,gender,c_age,obs_value", "geography": "1879048193...1879048194", "date": "latestMINUS15" # 2001 } table = self.api_ew.get_data(table_internal, query_params) self.assertEqual(table.shape, (364, 4)) self.assertEqual(sum(table.OBS_VALUE), 592842) def test_get_and_add_descriptive_column(self): table_name = "KS401EW" query_params = {} query_params["CELL"] = "7...13" query_params["date"] = "latest" query_params["RURAL_URBAN"] = "0" query_params["select"] = "GEOGRAPHY_CODE,CELL,OBS_VALUE" query_params["geography"] = "1245710558...1245710560" query_params["MEASURES"] = "20100" table = self.api_ew.get_data(table_name, query_params) self.assertEqual(table.shape, (21, 3)) self.assertEqual(sum(table.OBS_VALUE), 8214) # first ensure table is unmodified if column doesnt exist old_cols = len(table.columns) self.api_ew.contextify(table_name, "NOT_THERE", table) self.assertTrue(len(table.columns) == old_cols) self.api_ew.contextify(table_name, "CELL", table) self.assertTrue( table.at[0, "CELL_NAME"] == "Whole house or bungalow: Detached") self.assertTrue( table.at[1, "CELL_NAME"] == "Whole house or bungalow: Semi-detached") self.assertTrue( table.at[2, "CELL_NAME"] == "Whole house or bungalow: Terraced (including end-terrace)") self.assertTrue( table.at[3, "CELL_NAME"] == "Flat, maisonette or apartment: Purpose-built block of flats or tenement" ) self.assertTrue( table.at[4, "CELL_NAME"] == "Flat, maisonette or apartment: Part of a converted or shared house (including bed-sits)" ) self.assertTrue( table.at[5, "CELL_NAME"] == "Flat, maisonette or apartment: In a commercial building") self.assertTrue(table.at[6, "CELL_NAME"] == "Caravan or other mobile or temporary structure") def test_get_geog_from_names(self): result = self.query.get_geog_from_names( ["Leeds"], Api_EW.Nomisweb.GeoCodeLookup["OA11"]) self.assertEqual( result, '1254151943...1254154269,1254258198...1254258221,1254261711...1254261745,1254261853...1254261870,1254261894...1254261918,1254262125...1254262142,1254262341...1254262353,1254262394...1254262398,1254262498...1254262532,1254262620...1254262658,1254262922...1254262925' ) # same, but query with ONS code result = self.query.get_geog_from_names( ["E08000035"], Api_EW.Nomisweb.GeoCodeLookup["OA11"]) self.assertEqual( result, '1254151943...1254154269,1254258198...1254258221,1254261711...1254261745,1254261853...1254261870,1254261894...1254261918,1254262125...1254262142,1254262341...1254262353,1254262394...1254262398,1254262498...1254262532,1254262620...1254262658,1254262922...1254262925' ) result = self.query.get_geog_from_names( ["Newcastle upon Tyne"], Api_EW.Nomisweb.GeoCodeLookup["LSOA11"]) self.assertEqual(result, '1249910667...1249910832,1249935220...1249935228') result = self.query.get_geog_from_names( ["Leeds", "Bradford"], Api_EW.Nomisweb.GeoCodeLookup["MSOA11"]) self.assertEqual( result, '1245710411...1245710471,1245710558...1245710660,1245714998...1245714998,1245715007...1245715007,1245715021...1245715022' ) def test_get_geog_from_codes(self): result = self.query.api.get_geo_codes( [Api_EW.Nomisweb.GeoCodeLookup["EnglandWales"]], Api_EW.Nomisweb.GeoCodeLookup["LAD"]) self.assertEqual(result, '1946157057...1946157404') # test example code def test_geoquery(self): import inst.examples.geoquery as eg_geo eg_geo.main() def test_contextify(self): import inst.examples.contextify as eg_cont eg_cont.main() # just checks code snippet runs ok (i.e. returns 0) def test_code_snippet(self): table = "KS401EW" meta = self.api_ew.get_metadata(table) query_params = {} query_params["CELL"] = "7...13" query_params["date"] = "latest" query_params["RURAL_URBAN"] = "0" query_params["select"] = "GEOGRAPHY_CODE,CELL,OBS_VALUE" query_params["geography"] = "1245710558...1245710560" query_params["MEASURES"] = "20100" self.query.write_code_snippets(table, meta, query_params) self.assertTrue( os.system("python3 " + str(self.api_ew.cache_dir / (table + ".py"))) == 0) # fails on travis because R isnt installed #self.assertTrue(os.system("Rscript " + self.api.cache_dir + table + ".R") == 0) # checks the logic to compress a list of nomis geo codes into a shorter form for url def test_shorten_codelist(self): n = list(range(1, 21)) for _ in range(0, 100): short = Api_EW._shorten(sample(n, len(n))) self.assertEqual(short, "1...20") del (n[3]) for _ in range(0, 100): short = Api_EW._shorten(sample(n, len(n))) self.assertEqual(short, "1...3,5...20") del (n[16]) for _ in range(0, 100): short = Api_EW._shorten(sample(n, len(n))) self.assertEqual(short, "1...3,5...17,19...20") del (n[16]) for _ in range(0, 100): short = Api_EW._shorten(sample(n, len(n))) self.assertEqual(short, "1...3,5...17,20")
def __init__(self, cache_dir): self.cache_dir = cache_dir self.api = ApiEW.Nomisweb(cache_dir)
class Test(TestCase): api = Api.Nomisweb("/tmp/UKCensusAPI") query = Census.Query(api) def test_get_lad_codes(self): self.assertTrue(self.api.get_lad_codes("Royston Vasey") == []) self.assertTrue(self.api.get_lad_codes("Leeds") == [1946157127]) self.assertTrue(self.api.get_lad_codes(["Leeds", "Bradford"]) == [1946157127, 1946157124]) # This overlaps test_getGeographyFromCodes def test_geo_codes(self): result = self.api.get_geo_codes([Api.Nomisweb.EnglandWales], Api.Nomisweb.LAD) self.assertEqual(result, '1946157057...1946157404') result = self.api.get_geo_codes([1946157127], Api.Nomisweb.OA) self.assertEqual(result, '1254151943...1254154269,1254258198...1254258221,1254261711...1254261745,1254261853...1254261870,1254261894...1254261918,1254262125...1254262142,1254262341...1254262353,1254262394...1254262398,1254262498...1254262532,1254262620...1254262658,1254262922...1254262925') def test_get_metadata(self): meta = self.api.get_metadata("NONEXISTENT") self.assertFalse(meta) meta = self.api.get_metadata("KS401EW") self.assertEqual(meta["description"], 'KS401EW - Dwellings, household spaces and accommodation type') self.assertEqual(meta["nomis_table"], 'NM_618_1') def test_get_url(self): table = "NM_618_1" query_params = {} query_params["CELL"] = "7...13" query_params["date"] = "latest" query_params["RURAL_URBAN"] = "0" query_params["select"] = "GEOGRAPHY_CODE,CELL,OBS_VALUE" query_params["geography"] = "1245710558...1245710660,1245714998...1245714998,1245715007...1245715007,1245715021...1245715022" query_params["MEASURES"] = "20100" self.assertEqual(self.api.get_url(table, query_params), "https://www.nomisweb.co.uk/api/v01/dataset/NM_618_1.data.tsv?CELL=7...13&MEASURES=20100&RURAL_URBAN=0&date=latest&geography=1245710558...1245710660%2C1245714998...1245714998%2C1245715007...1245715007%2C1245715021...1245715022&select=GEOGRAPHY_CODE%2CCELL%2COBS_VALUE") def test_get_data(self): table = "KS401EW" table_internal = "NM_618_1" query_params = {} query_params["CELL"] = "7...13" query_params["date"] = "latest" query_params["RURAL_URBAN"] = "0" query_params["select"] = "GEOGRAPHY_CODE,CELL,OBS_VALUE" query_params["geography"] = "1245710558...1245710560" query_params["MEASURES"] = "20100" table = self.api.get_data(table, table_internal, query_params) self.assertEqual(table.shape, (21, 3)) self.assertEqual(sum(table.OBS_VALUE), 8214) def test_get_and_add_descriptive_column(self): table_name = "KS401EW" meta = self.api.load_metadata(table_name) query_params = {} query_params["CELL"] = "7...13" query_params["date"] = "latest" query_params["RURAL_URBAN"] = "0" query_params["select"] = "GEOGRAPHY_CODE,CELL,OBS_VALUE" query_params["geography"] = "1245710558...1245710560" query_params["MEASURES"] = "20100" table = self.api.get_data(table_name, meta["nomis_table"], query_params) self.assertEqual(table.shape, (21, 3)) self.assertEqual(sum(table.OBS_VALUE), 8214) # first ensure table is unmodified if column doesnt exist old_cols = len(table.columns) self.api.contextify(table_name, "NOT_THERE", table) self.assertTrue(len(table.columns) == old_cols) self.api.contextify(table_name, "CELL", table) self.assertTrue(table.at[0,"CELL_NAME"] == "Whole house or bungalow: Detached") self.assertTrue(table.at[1,"CELL_NAME"] == "Whole house or bungalow: Semi-detached") self.assertTrue(table.at[2,"CELL_NAME"] == "Whole house or bungalow: Terraced (including end-terrace)") self.assertTrue(table.at[3,"CELL_NAME"] == "Flat, maisonette or apartment: Purpose-built block of flats or tenement") self.assertTrue(table.at[4,"CELL_NAME"] == "Flat, maisonette or apartment: Part of a converted or shared house (including bed-sits)") self.assertTrue(table.at[5,"CELL_NAME"] == "Flat, maisonette or apartment: In a commercial building") self.assertTrue(table.at[6,"CELL_NAME"] == "Caravan or other mobile or temporary structure") def test_get_geog_from_names(self): result = self.query.get_geog_from_names(["Leeds"], Api.Nomisweb.OA) self.assertEqual(result, '1254151943...1254154269,1254258198...1254258221,1254261711...1254261745,1254261853...1254261870,1254261894...1254261918,1254262125...1254262142,1254262341...1254262353,1254262394...1254262398,1254262498...1254262532,1254262620...1254262658,1254262922...1254262925') result = self.query.get_geog_from_names(["Newcastle upon Tyne"], Api.Nomisweb.LSOA) self.assertEqual(result, '1249910667...1249910832,1249935220...1249935228') result = self.query.get_geog_from_names(["Leeds", "Bradford"], Api.Nomisweb.MSOA) self.assertEqual(result, '1245710411...1245710471,1245710558...1245710660,1245714998...1245714998,1245715007...1245715007,1245715021...1245715022') def test_get_geog_from_codes(self): result = self.query.api.get_geo_codes([Api.Nomisweb.EnglandWales], Api.Nomisweb.LAD) self.assertEqual(result, '1946157057...1946157404') # test example code def test_geoquery(self): import inst.examples.geoquery as eg_geo eg_geo.main() def test_contextify(self): import inst.examples.contextify as eg_cont eg_cont.main() # just checks code snippet runs ok (i.e. returns 0) def test_code_snippet(self): table = "KS401EW" meta = self.api.get_metadata(table) query_params = {} query_params["CELL"] = "7...13" query_params["date"] = "latest" query_params["RURAL_URBAN"] = "0" query_params["select"] = "GEOGRAPHY_CODE,CELL,OBS_VALUE" query_params["geography"] = "1245710558...1245710560" query_params["MEASURES"] = "20100" self.query.write_code_snippets(table, meta, query_params) self.assertTrue(os.system("python3 " + self.api.cache_dir + table + ".py") == 0)
#!/usr/bin/env python3 # Disable "Invalid constant name" # pylint: disable=C0103 # -*- coding: utf-8 -*- """ interactive census table query """ import ukcensusapi.Nomisweb as CensusApi import ukcensusapi.Query as Census # intialise the API using current directory as the cache directory api = CensusApi.Nomisweb("/tmp/UKCensusAPI") # initialise the census query census = Census.Query(api) # run the interactive query census.table()
def read_cell_range(worksheet, topleft, bottomright): data_rows = [] for row in worksheet[topleft:bottomright]: data_cols = [] for cell in row: data_cols.append(cell.value) data_rows.append(data_cols) return np.array(data_rows) # England SNPP by gender by SYOA from UKCensusAPI cache_dir = "./raw_data" print("Collating SNPP data for England...") data_api = Api.Nomisweb(cache_dir) # need to do this in 2 batches as entire table has >1000000 rows table_internal = "NM_2006_1" # 2014-based SNPP query_params = { "gender": "1,2", "c_age": "101...191", "MEASURES": "20100", "date": "latest", # 2014-based "projected_year": "2014...2027", "select": "geography_code,projected_year_name,gender,c_age,obs_value", "geography": "1946157057...1946157382" } snpp_e = data_api.get_data(table_internal, query_params) query_params["projected_year"] = "2028...2039"
def __init__(self, region, resolution, cache_dir): self.region = region self.resolution = resolution self.data_api_en = Api_ew.Nomisweb(cache_dir) self.data_api_sc = Api_sc.NRScotland(cache_dir)