def test_filter_records_exclusion_analysis_with_incorrect_columns(
data: pd.DataFrame):
first_country_code = data.loc[0, COLUMN_NAME]
census_data = data
with pytest.raises(ValueError):
census_data = filter.filter_records(census_data,
field=COLUMN_NAME,
value_list={first_country_code},
exclude_matching=False,
exclusion_analysis=True)
census_data = filter.filter_records(census_data,
field=COLUMN_NAME,
value_list={first_country_code},
exclude_matching=True,
exclusion_analysis=True)
def test_filter_records_exclusion(data: pd.DataFrame):
first_country_code = data.loc[0, COLUMN_NAME]
census_data = data
census_data = filter.filter_records(census_data,
field=COLUMN_NAME,
value_list={first_country_code},
exclude_matching=False,
exclusion_analysis=False)
expected_outcome = data.loc[data[COLUMN_NAME] == first_country_code]
assert census_data.equals(expected_outcome)
import time
from incognita.data.scout_census import load_census_data
from incognita.logger import logger
from incognita.reports.history_summary import HistorySummary
from incognita.utility import filter
from incognita.utility import timing
if __name__ == "__main__":
start_time = time.time()
logger.info(
f"Starting at {time.strftime('%H:%M:%S', time.localtime(start_time))}")
census_ids = {15, 16, 17, 18, 19, 20}
census_data = load_census_data()
census_data = filter.filter_records(census_data, "Census_ID", census_ids)
census_data = filter.filter_records(
census_data, "X_name",
{"England", "Scotland", "Wales", "Northern Ireland"})
# If filtering on IMD, remove NA values
# census_data = filter.filter_records(census_data, "imd_decile", ["nan"], exclude_matching=True)
# census_data = filter.filter_records(census_data, "imd_decile", [1, 2, 3])
section_history = HistorySummary(census_data)
section_history.new_section_history_summary(
sorted(census_ids), report_name="opened_section_data")
timing.close(start_time)
import pandas as pd
from incognita.data.scout_census import load_census_data
from incognita.geographies import district_boundaries
from incognita.logger import logger
from incognita.utility import config
from incognita.utility import filter
from incognita.utility import timing
if __name__ == "__main__":
start_time = time.time()
logger.info(
f"Starting at {time.strftime('%H:%M:%S', time.localtime(start_time))}")
census_data = load_census_data()
census_data = filter.filter_records(census_data, "Census_ID", {20})
# Remove Jersey, Guernsey, and Isle of Man as they have invalid lat/long coordinates for their postcodes
census_data = filter.filter_records(
census_data,
"C_name", {"Bailiwick of Guernsey", "Isle of Man", "Jersey"},
exclude_matching=True)
# low resolution shape data
world_low_res = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres"))
uk_shape = world_low_res.loc[world_low_res.name == "United Kingdom",
"geometry"].array.data[0]
# # high resolution shape data
# uk_shape = gpd.read_file(r"S:\Development\incognita\data\UK Shape\GBR_adm0.shp")["geometry"].array.data[0]
logger.info("UK outline shapefile loaded.")
district_polygons = district_boundaries.create_district_boundaries(
from incognita.maps.map import Map
from incognita.reports.reports import Reports
from incognita.utility import filter
from incognita.utility import timing
if __name__ == "__main__":
start_time = time.time()
logger.info(
f"Starting at {time.strftime('%H:%M:%S', time.localtime(start_time))}")
countries = {"England", "Wales"}
country_codes = {"E92000001", "W92000004"}
# setup data
census_data = load_census_data()
census_data = filter.filter_records(census_data, "Census_ID", {20})
census_data = filter.filter_records(census_data, "X_name", countries)
census_data = filter.filter_records(census_data, "type",
{"Colony", "Pack", "Troop", "Unit"})
census_data = filter.filter_records(census_data, "ctry", country_codes)
census_data = filter.filter_records(census_data,
"postcode_is_valid", {True},
exclusion_analysis=True)
lsoa = Reports("LSOA", census_data)
lsoa.filter_boundaries("ctry", country_codes)
lsoa_boundary_report = lsoa.create_boundary_report(
{"Section numbers", "6 to 17 numbers"}, report_name="lsoa_ew")
# iz = Reports("Intermediate Zone", census_data)
# iz.filter_boundaries(field="ctry", value_list={"S92000003"})
from incognita.logger import logger
from incognita.maps.map import Map
from incognita.reports.reports import Reports
from incognita.utility import filter
from incognita.utility import timing
if __name__ == "__main__":
start_time = time.time()
logger.info(
f"Starting at {time.strftime('%H:%M:%S', time.localtime(start_time))}")
county_name = "Birmingham"
census_id = 21
census_data = load_census_data()
census_data = filter.filter_records(census_data, "Census_ID",
{census_id}) # 16, 17, 18, 19, 20
census_data = filter.filter_records(
census_data, "C_name", {county_name}) # "Shropshire", "West Mercia"
census_data = filter.filter_records(census_data, "postcode_is_valid",
{True})
reports = Reports("LSOA", census_data)
reports.filter_boundaries("C_name", {county_name}, "oslaua")
boundary_report = reports.create_boundary_report(
{"Section numbers"}, report_name=f"{county_name} by LSOA"
) # TODO: before postcode filtering
# boundary_report = reports.create_boundary_report({"Section numbers"}, historical=True, report_name=f"{county_name}_by_lsoa") # TODO: before postcode filtering
# Create map object
mapper = Map(map_name=f"{county_name}",
map_title=f"{county_name} Sections by IMD Decile")
from incognita.data.scout_census import load_census_data
from incognita.logger import logger
from incognita.maps.map import Map
from incognita.reports.reports import Reports
from incognita.utility import filter
from incognita.utility import timing
if __name__ == "__main__":
start_time = time.time()
logger.info(
f"Starting at {time.strftime('%H:%M:%S', time.localtime(start_time))}")
census_id = 20
census_data = load_census_data()
census_data = filter.filter_records(census_data, "postcode_is_valid",
{True})
census_data = filter.filter_records(census_data, "Census_ID", {census_id})
# Remove Jersey, Guernsey, and Isle of Man as they don't have lat long coordinates in their postcodes
census_data = filter.filter_records(
census_data,
"C_name", {"Bailiwick of Guernsey", "Isle of Man", "Jersey"},
exclude_matching=True)
# Generate boundary report
reports = Reports("Local Authority", census_data)
boundary_report = reports.create_boundary_report(
{"awards"}, report_name="laua_awards_report")
# Create map object
mapper = Map(map_name="UK_QSA_awards")
from incognita.logger import logger
from incognita.maps.map import Map
from incognita.reports.reports import Reports
from incognita.utility import filter
from incognita.utility import timing
if __name__ == "__main__":
start_time = time.time()
logger.info(f"Starting at {time.strftime('%H:%M:%S', time.localtime(start_time))}")
county_name = "North Yorkshire"
census_id = 20
# setup data
census_data = load_census_data()
census_data = filter.filter_records(census_data, "Census_ID", {census_id})
census_data = filter.filter_records(census_data, "X_name", {"England", "Scotland", "Wales", "Northern Ireland"})
census_data = filter.filter_records(census_data, "C_name", {"Bailiwick of Guernsey", "Isle of Man", "Jersey"}, exclude_matching=True)
census_data = filter.filter_records(census_data, "type", {"Colony", "Pack", "Troop", "Unit"})
census_data = filter.filter_records(census_data, "C_name", {county_name})
census_data = filter.filter_records(census_data, "postcode_is_valid", {True}, exclusion_analysis=True)
# # % 6-17 pcon uptake from Jan-2020 Scout Census with May 2019 ONS
# pcon_reports = Reports("Constituency", census_data)
# pcon_reports.filter_boundaries("C_name", {county_name}, "pcon")
# pcon_boundary_report = pcon_reports.create_boundary_report({"Section numbers", "6 to 17 numbers"}, report_name=f"{county_name} - westminster constituencies")
# pcon_reports.create_uptake_report(pcon_boundary_report, report_name=f"{county_name} - westminster constituencies (uptake)")
#
# # 6-17 IMD from Jan-2020 Scout Census with May 2019 ONS
# imd_reports = Reports("LSOA", census_data)
# imd_reports.filter_boundaries("C_name", {county_name}, "pcon")
from incognita.reports.reports import Reports
from incognita.utility import config
from incognita.utility import filter
from incognita.utility import timing
if __name__ == "__main__":
start_time = time.time()
logger.info(
f"Starting at {time.strftime('%H:%M:%S', time.localtime(start_time))}")
county_name = "Central Yorkshire"
la_code = "E08000035" # Leeds LA code
census_id = 20
census_data = load_census_data()
census_data = filter.filter_records(census_data, "Census_ID", {census_id})
census_data = filter.filter_records(census_data, "oslaua", {la_code})
census_data = filter.filter_records(census_data,
"postcode_is_valid", {True},
exclusion_analysis=True)
# Generate boundary report
reports = Reports("LSOA", census_data)
reports.filter_boundaries("oslaua", {la_code}) # Leeds LA code
boundary_report = reports.create_boundary_report(
{"Section numbers"}, report_name="leeds_sections")
# Create map object
mapper = Map(map_name="Leeds")
# Plot
import time
from incognita.data.scout_census import load_census_data
from incognita.logger import logger
from incognita.reports.history_summary import HistorySummary
from incognita.utility import filter
from incognita.utility import timing
if __name__ == "__main__":
start_time = time.time()
logger.info(
f"Starting at {time.strftime('%H:%M:%S', time.localtime(start_time))}")
census_data = load_census_data()
census_data = filter.filter_records(census_data, "imd_decile", {1, 2})
census_data = filter.filter_records(census_data, "C_name", {"Shropshire"})
# # Read group list - with a column headed "G_ID"
# groups = pd.read_csv(r"Output\yuf_groups.csv")
# group_ids = groups["G_ID"].drop_duplicates().dropna().to_list()
# census_data = filter.filter_records(census_data, "G_ID", {group_ids})
years = [
2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021
]
history_summary = HistorySummary(census_data)
history_summary.group_history_summary(years,
report_name="group_history_report")
timing.close(start_time)
import matplotlib.cm
from matplotlib.colors import rgb2hex
from incognita.data.scout_census import load_census_data
from incognita.logger import logger
from incognita.utility import config
from incognita.utility import filter
if __name__ == "__main__":
start_time = time.time()
logger.info(f"Starting at {time.strftime('%H:%M:%S', time.localtime(start_time))}")
county_name = "Central Yorkshire"
census_data = load_census_data()
census_data = filter.filter_records(census_data, "Census_ID", {20})
county_id = census_data.loc[census_data["C_name"] == county_name, "C_ID"].array[0]
gdf = gpd.read_file(config.SETTINGS.folders.boundaries / "districts-borders-uk.geojson")
gdf = gdf[gdf["C_ID"] == county_id].copy()
# https://matplotlib.org/stable/tutorials/colors/colormaps.html
# cmap = matplotlib.cm.get_cmap('Set1')
cmap = matplotlib.cm.get_cmap("tab20")
# cmap = matplotlib.cm.get_cmap('hsv')
cmap_indicies = (i / (len(gdf.index) - 1) for i in range(len(gdf.index))) # qualitative colours
# cmap_indicies = (random.random() for _ in range(len(gdf.index))) # qualitative colours
gdf["fill"] = [rgb2hex(cmap(i)) for i in cmap_indicies]
gdf["stroke-width"] = 0.2
from incognita.data.scout_census import load_census_data
from incognita.logger import logger
from incognita.reports.reports import Reports
from incognita.utility import filter
from incognita.utility import timing
if __name__ == "__main__":
start_time = time.time()
logger.info(
f"Starting at {time.strftime('%H:%M:%S', time.localtime(start_time))}")
county_name = "Gt. London South"
census_id = 20
census_data = load_census_data()
census_data = filter.filter_records(census_data, "Census_ID", {census_id})
census_data = filter.filter_records(census_data, "C_name", {county_name})
census_data = filter.filter_records(census_data,
"postcode_is_valid", {True},
exclusion_analysis=True)
reports = Reports("IMD Decile", census_data)
report_options = {
"Groups", "Number of Sections", "Section numbers", "waiting list total"
}
boundary_report = reports.create_boundary_report(
report_options, report_name=f"{county_name} - {census_id} IMD report")
timing.close(start_time)
