def test_filter_records_exclusion_analysis_with_incorrect_columns( data: pd.DataFrame): first_country_code = data.loc[0, COLUMN_NAME] census_data = data with pytest.raises(ValueError): census_data = filter.filter_records(census_data, field=COLUMN_NAME, value_list={first_country_code}, exclude_matching=False, exclusion_analysis=True) census_data = filter.filter_records(census_data, field=COLUMN_NAME, value_list={first_country_code}, exclude_matching=True, exclusion_analysis=True)
def test_filter_records_exclusion(data: pd.DataFrame): first_country_code = data.loc[0, COLUMN_NAME] census_data = data census_data = filter.filter_records(census_data, field=COLUMN_NAME, value_list={first_country_code}, exclude_matching=False, exclusion_analysis=False) expected_outcome = data.loc[data[COLUMN_NAME] == first_country_code] assert census_data.equals(expected_outcome)
import time from incognita.data.scout_census import load_census_data from incognita.logger import logger from incognita.reports.history_summary import HistorySummary from incognita.utility import filter from incognita.utility import timing if __name__ == "__main__": start_time = time.time() logger.info( f"Starting at {time.strftime('%H:%M:%S', time.localtime(start_time))}") census_ids = {15, 16, 17, 18, 19, 20} census_data = load_census_data() census_data = filter.filter_records(census_data, "Census_ID", census_ids) census_data = filter.filter_records( census_data, "X_name", {"England", "Scotland", "Wales", "Northern Ireland"}) # If filtering on IMD, remove NA values # census_data = filter.filter_records(census_data, "imd_decile", ["nan"], exclude_matching=True) # census_data = filter.filter_records(census_data, "imd_decile", [1, 2, 3]) section_history = HistorySummary(census_data) section_history.new_section_history_summary( sorted(census_ids), report_name="opened_section_data") timing.close(start_time)
import pandas as pd from incognita.data.scout_census import load_census_data from incognita.geographies import district_boundaries from incognita.logger import logger from incognita.utility import config from incognita.utility import filter from incognita.utility import timing if __name__ == "__main__": start_time = time.time() logger.info( f"Starting at {time.strftime('%H:%M:%S', time.localtime(start_time))}") census_data = load_census_data() census_data = filter.filter_records(census_data, "Census_ID", {20}) # Remove Jersey, Guernsey, and Isle of Man as they have invalid lat/long coordinates for their postcodes census_data = filter.filter_records( census_data, "C_name", {"Bailiwick of Guernsey", "Isle of Man", "Jersey"}, exclude_matching=True) # low resolution shape data world_low_res = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) uk_shape = world_low_res.loc[world_low_res.name == "United Kingdom", "geometry"].array.data[0] # # high resolution shape data # uk_shape = gpd.read_file(r"S:\Development\incognita\data\UK Shape\GBR_adm0.shp")["geometry"].array.data[0] logger.info("UK outline shapefile loaded.") district_polygons = district_boundaries.create_district_boundaries(
from incognita.maps.map import Map from incognita.reports.reports import Reports from incognita.utility import filter from incognita.utility import timing if __name__ == "__main__": start_time = time.time() logger.info( f"Starting at {time.strftime('%H:%M:%S', time.localtime(start_time))}") countries = {"England", "Wales"} country_codes = {"E92000001", "W92000004"} # setup data census_data = load_census_data() census_data = filter.filter_records(census_data, "Census_ID", {20}) census_data = filter.filter_records(census_data, "X_name", countries) census_data = filter.filter_records(census_data, "type", {"Colony", "Pack", "Troop", "Unit"}) census_data = filter.filter_records(census_data, "ctry", country_codes) census_data = filter.filter_records(census_data, "postcode_is_valid", {True}, exclusion_analysis=True) lsoa = Reports("LSOA", census_data) lsoa.filter_boundaries("ctry", country_codes) lsoa_boundary_report = lsoa.create_boundary_report( {"Section numbers", "6 to 17 numbers"}, report_name="lsoa_ew") # iz = Reports("Intermediate Zone", census_data) # iz.filter_boundaries(field="ctry", value_list={"S92000003"})
from incognita.logger import logger from incognita.maps.map import Map from incognita.reports.reports import Reports from incognita.utility import filter from incognita.utility import timing if __name__ == "__main__": start_time = time.time() logger.info( f"Starting at {time.strftime('%H:%M:%S', time.localtime(start_time))}") county_name = "Birmingham" census_id = 21 census_data = load_census_data() census_data = filter.filter_records(census_data, "Census_ID", {census_id}) # 16, 17, 18, 19, 20 census_data = filter.filter_records( census_data, "C_name", {county_name}) # "Shropshire", "West Mercia" census_data = filter.filter_records(census_data, "postcode_is_valid", {True}) reports = Reports("LSOA", census_data) reports.filter_boundaries("C_name", {county_name}, "oslaua") boundary_report = reports.create_boundary_report( {"Section numbers"}, report_name=f"{county_name} by LSOA" ) # TODO: before postcode filtering # boundary_report = reports.create_boundary_report({"Section numbers"}, historical=True, report_name=f"{county_name}_by_lsoa") # TODO: before postcode filtering # Create map object mapper = Map(map_name=f"{county_name}", map_title=f"{county_name} Sections by IMD Decile")
from incognita.data.scout_census import load_census_data from incognita.logger import logger from incognita.maps.map import Map from incognita.reports.reports import Reports from incognita.utility import filter from incognita.utility import timing if __name__ == "__main__": start_time = time.time() logger.info( f"Starting at {time.strftime('%H:%M:%S', time.localtime(start_time))}") census_id = 20 census_data = load_census_data() census_data = filter.filter_records(census_data, "postcode_is_valid", {True}) census_data = filter.filter_records(census_data, "Census_ID", {census_id}) # Remove Jersey, Guernsey, and Isle of Man as they don't have lat long coordinates in their postcodes census_data = filter.filter_records( census_data, "C_name", {"Bailiwick of Guernsey", "Isle of Man", "Jersey"}, exclude_matching=True) # Generate boundary report reports = Reports("Local Authority", census_data) boundary_report = reports.create_boundary_report( {"awards"}, report_name="laua_awards_report") # Create map object mapper = Map(map_name="UK_QSA_awards")
from incognita.logger import logger from incognita.maps.map import Map from incognita.reports.reports import Reports from incognita.utility import filter from incognita.utility import timing if __name__ == "__main__": start_time = time.time() logger.info(f"Starting at {time.strftime('%H:%M:%S', time.localtime(start_time))}") county_name = "North Yorkshire" census_id = 20 # setup data census_data = load_census_data() census_data = filter.filter_records(census_data, "Census_ID", {census_id}) census_data = filter.filter_records(census_data, "X_name", {"England", "Scotland", "Wales", "Northern Ireland"}) census_data = filter.filter_records(census_data, "C_name", {"Bailiwick of Guernsey", "Isle of Man", "Jersey"}, exclude_matching=True) census_data = filter.filter_records(census_data, "type", {"Colony", "Pack", "Troop", "Unit"}) census_data = filter.filter_records(census_data, "C_name", {county_name}) census_data = filter.filter_records(census_data, "postcode_is_valid", {True}, exclusion_analysis=True) # # % 6-17 pcon uptake from Jan-2020 Scout Census with May 2019 ONS # pcon_reports = Reports("Constituency", census_data) # pcon_reports.filter_boundaries("C_name", {county_name}, "pcon") # pcon_boundary_report = pcon_reports.create_boundary_report({"Section numbers", "6 to 17 numbers"}, report_name=f"{county_name} - westminster constituencies") # pcon_reports.create_uptake_report(pcon_boundary_report, report_name=f"{county_name} - westminster constituencies (uptake)") # # # 6-17 IMD from Jan-2020 Scout Census with May 2019 ONS # imd_reports = Reports("LSOA", census_data) # imd_reports.filter_boundaries("C_name", {county_name}, "pcon")
from incognita.reports.reports import Reports from incognita.utility import config from incognita.utility import filter from incognita.utility import timing if __name__ == "__main__": start_time = time.time() logger.info( f"Starting at {time.strftime('%H:%M:%S', time.localtime(start_time))}") county_name = "Central Yorkshire" la_code = "E08000035" # Leeds LA code census_id = 20 census_data = load_census_data() census_data = filter.filter_records(census_data, "Census_ID", {census_id}) census_data = filter.filter_records(census_data, "oslaua", {la_code}) census_data = filter.filter_records(census_data, "postcode_is_valid", {True}, exclusion_analysis=True) # Generate boundary report reports = Reports("LSOA", census_data) reports.filter_boundaries("oslaua", {la_code}) # Leeds LA code boundary_report = reports.create_boundary_report( {"Section numbers"}, report_name="leeds_sections") # Create map object mapper = Map(map_name="Leeds") # Plot
import time from incognita.data.scout_census import load_census_data from incognita.logger import logger from incognita.reports.history_summary import HistorySummary from incognita.utility import filter from incognita.utility import timing if __name__ == "__main__": start_time = time.time() logger.info( f"Starting at {time.strftime('%H:%M:%S', time.localtime(start_time))}") census_data = load_census_data() census_data = filter.filter_records(census_data, "imd_decile", {1, 2}) census_data = filter.filter_records(census_data, "C_name", {"Shropshire"}) # # Read group list - with a column headed "G_ID" # groups = pd.read_csv(r"Output\yuf_groups.csv") # group_ids = groups["G_ID"].drop_duplicates().dropna().to_list() # census_data = filter.filter_records(census_data, "G_ID", {group_ids}) years = [ 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 ] history_summary = HistorySummary(census_data) history_summary.group_history_summary(years, report_name="group_history_report") timing.close(start_time)
import matplotlib.cm from matplotlib.colors import rgb2hex from incognita.data.scout_census import load_census_data from incognita.logger import logger from incognita.utility import config from incognita.utility import filter if __name__ == "__main__": start_time = time.time() logger.info(f"Starting at {time.strftime('%H:%M:%S', time.localtime(start_time))}") county_name = "Central Yorkshire" census_data = load_census_data() census_data = filter.filter_records(census_data, "Census_ID", {20}) county_id = census_data.loc[census_data["C_name"] == county_name, "C_ID"].array[0] gdf = gpd.read_file(config.SETTINGS.folders.boundaries / "districts-borders-uk.geojson") gdf = gdf[gdf["C_ID"] == county_id].copy() # https://matplotlib.org/stable/tutorials/colors/colormaps.html # cmap = matplotlib.cm.get_cmap('Set1') cmap = matplotlib.cm.get_cmap("tab20") # cmap = matplotlib.cm.get_cmap('hsv') cmap_indicies = (i / (len(gdf.index) - 1) for i in range(len(gdf.index))) # qualitative colours # cmap_indicies = (random.random() for _ in range(len(gdf.index))) # qualitative colours gdf["fill"] = [rgb2hex(cmap(i)) for i in cmap_indicies] gdf["stroke-width"] = 0.2
from incognita.data.scout_census import load_census_data from incognita.logger import logger from incognita.reports.reports import Reports from incognita.utility import filter from incognita.utility import timing if __name__ == "__main__": start_time = time.time() logger.info( f"Starting at {time.strftime('%H:%M:%S', time.localtime(start_time))}") county_name = "Gt. London South" census_id = 20 census_data = load_census_data() census_data = filter.filter_records(census_data, "Census_ID", {census_id}) census_data = filter.filter_records(census_data, "C_name", {county_name}) census_data = filter.filter_records(census_data, "postcode_is_valid", {True}, exclusion_analysis=True) reports = Reports("IMD Decile", census_data) report_options = { "Groups", "Number of Sections", "Section numbers", "waiting list total" } boundary_report = reports.create_boundary_report( report_options, report_name=f"{county_name} - {census_id} IMD report") timing.close(start_time)