def test_employment_age_distribution(do_show, do_save, create_sample_pop_e2e,
get_fig_dir_by_module):
sp.logger.info(
"Test employment age distribution vs the employment_rates_by_age.dat")
plotting_kwargs = sc.objdict(do_show=do_show,
do_save=do_save,
figdir=get_fig_dir_by_module)
actual_employment_age_count = create_sample_pop_e2e.count_employment_by_age(
)
total_employee = sum(actual_employment_age_count.values())
expected_employment_age_dist = sp.norm_dic(
sp.get_employment_rates(**create_sample_pop_e2e.loc_pars))
expected_employment_age_count = {
i: round(expected_employment_age_dist[i] * total_employee)
for i in expected_employment_age_dist
}
# generate list of ages based on the actual count
generated_actual = sum([[i] * actual_employment_age_count[i]
for i in actual_employment_age_count], [])
generated_expected = sum([[i] * expected_employment_age_count[i]
for i in expected_employment_age_count], [])
# run statistical tests for employment by age distribution
# TODO: Need to refine the data for fair comparison
sp.statistic_test(expected=generated_expected,
actual=generated_actual,
test=st.kstest,
verbose=True)
# plot enrollment by age
create_sample_pop_e2e.plot_employment_rates_by_age(**plotting_kwargs)
def test_get_uids_potential_workers(location='seattle_metro',
state_location='Washington',
country_location='usa'):
n = 10000
homes = sp.get_head_age_by_size_distr(datadir,
state_location,
country_location,
file_path=None,
household_size_1_included=False,
use_default=True)
homes_by_uids, age_by_uid_dic = sp.assign_uids_by_homes(homes, id_len=16)
uids_in_school, uids_in_school_by_age, ages_in_school_count = sp.get_uids_in_school(
datadir,
n,
location,
state_location,
country_location,
age_by_uid_dic,
homes_by_uids,
use_default=False)
employment_rates = sp.get_employment_rates(
datadir,
location=location,
state_location=state_location,
country_location=country_location,
use_default=True)
potential_worker_uids, potential_worker_uids_by_age, potential_worker_ages_left_count = sp.get_uids_potential_workers(
uids_in_school, employment_rates, age_by_uid_dic)
assert potential_worker_ages_left_count is not None
return potential_worker_uids, potential_worker_uids_by_age, employment_rates, age_by_uid_dic
def test_assign_teachers_to_work(location='seattle_metro',
state_location='Washington',
country_location='usa',
folder_name='contact_networks',
n=10000):
# Assign students to school
gen_schools, gen_school_uids = test_send_students_to_school()
employment_rates = sp.get_employment_rates(
datadir,
location=location,
state_location=state_location,
country_location=country_location,
use_default=True)
age_by_uid_dic = sp.read_in_age_by_uid(datadir, location, state_location,
country_location, folder_name, n)
uids_in_school = sp.get_uids_in_school(datadir,
n,
location,
state_location,
country_location,
folder_name=folder_name,
use_default=True)
potential_worker_uids, potential_worker_uids_by_age, \
potential_worker_ages_left_count = sp.get_uids_potential_workers(uids_in_school, employment_rates, age_by_uid_dic)
workers_by_age_to_assign_count = sp.get_workers_by_age_to_assign(
employment_rates, potential_worker_ages_left_count, age_by_uid_dic)
# Assign teachers and update school lists
syn_schools, syn_school_uids, potential_worker_uids, potential_worker_uids_by_age, \
workers_by_age_to_assign_count = sp.assign_teachers_to_work(gen_schools, gen_school_uids, employment_rates,
workers_by_age_to_assign_count,
potential_worker_uids, potential_worker_uids_by_age,
potential_worker_ages_left_count,
student_teacher_ratio=30, teacher_age_min=25,
teacher_age_max=75, verbose=False)
for n in range(len(syn_schools)):
print(syn_schools[n])
assert syn_schools[n] is not None
assert syn_school_uids[n] is not None
assert syn_schools == gen_schools
assert syn_school_uids == gen_school_uids
assert potential_worker_uids == potential_worker_uids
assert potential_worker_uids_by_age == potential_worker_uids_by_age
assert workers_by_age_to_assign_count == workers_by_age_to_assign_count
def test_get_uids_potential_workers(location='seattle_metro', state_location='Washington',
country_location='usa', folder_name='contact_networks'):
Nhomes = 10000
uids_in_school = sp.get_uids_in_school(datadir, Nhomes, location,
state_location,
country_location,
folder_name=folder_name,
use_default=True)
employment_rates = sp.get_employment_rates(datadir, location=location, state_location=state_location,
country_location=country_location, use_default=True)
age_by_uid_dic = sp.read_in_age_by_uid(datadir, location, state_location, country_location, folder_name, Nhomes)
potential_worker_uids, potential_worker_uids_by_age, potential_worker_ages_left_count = sp.get_uids_potential_workers(
uids_in_school, employment_rates, age_by_uid_dic)
assert potential_worker_ages_left_count is not None
return potential_worker_uids, potential_worker_uids_by_age, employment_rates, age_by_uid_dic
def test_generate_workplace_sizes(location='seattle_metro', state_location='Washington',
country_location='usa', folder_name='contact_networks'):
Npeople = 10000
uids_in_school, uids_in_school_by_age, ages_in_school_count = sp.get_uids_in_school(datadir, Npeople, location,
state_location,
country_location,
folder_name=folder_name,
use_default=True)
school_size_distr_by_bracket = sp.get_school_size_distr_by_brackets(datadir, location, state_location,
country_location)
school_size_brackets = sp.get_school_size_brackets(datadir, location, state_location, country_location)
school_sizes = sp.generate_school_sizes(school_size_distr_by_bracket, school_size_brackets, uids_in_school)
age_brackets_filepath = sp.get_census_age_brackets_path(datadir, state_location, country_location)
age_brackets = sp.get_age_brackets_from_df(age_brackets_filepath)
age_by_brackets_dic = sp.get_age_by_brackets_dic(age_brackets)
contact_matrix_dic = sp.get_contact_matrix_dic(datadir, sheet_name='United States of America')
# Need to instead get syn_schools now
syn_schools, syn_school_uids = sp.send_students_to_school(school_sizes, uids_in_school, uids_in_school_by_age,
ages_in_school_count, age_brackets, age_by_brackets_dic,
contact_matrix_dic)
employment_rates = sp.get_employment_rates(datadir, location=location, state_location=state_location,
country_location=country_location, use_default=True)
age_by_uid_dic = sp.read_in_age_by_uid(datadir, location, state_location, country_location, folder_name, Npeople)
potential_worker_uids, potential_worker_uids_by_age, potential_worker_ages_left_count = sp.get_uids_potential_workers(
syn_school_uids, employment_rates, age_by_uid_dic)
workers_by_age_to_assign_count = sp.get_workers_by_age_to_assign(employment_rates, potential_worker_ages_left_count,
age_by_uid_dic)
workplace_size_brackets = sp.get_workplace_size_brackets(datadir, location, state_location, country_location,
use_default=True)
workplace_size_distr_by_brackets = sp.get_workplace_size_distr_by_brackets(datadir,
state_location=state_location,
country_location=country_location,
use_default=True)
workplace_sizes = sp.generate_workplace_sizes(workplace_size_distr_by_brackets, workplace_size_brackets,
workers_by_age_to_assign_count)
return workers_by_age_to_assign_count, workplace_size_brackets, workplace_size_distr_by_brackets, workplace_sizes
def test_generate_workplace_sizes(location='seattle_metro',
state_location='Washington',
country_location='usa'):
Npeople = 10000
uids_in_school, uids_in_school_by_age, uids_in_school_count = sp.get_uids_in_school(
datadir,
Npeople,
location,
state_location,
country_location,
use_default=True)
employment_rates = sp.get_employment_rates(
datadir,
location=location,
state_location=state_location,
country_location=country_location,
use_default=True)
age_by_uid_dic = sp.read_in_age_by_uid(datadir, location, state_location,
country_location, Npeople)
potential_worker_uids, potential_worker_uids_by_age, potential_worker_ages_left_count = sp.get_uids_potential_workers(
uids_in_school, employment_rates, age_by_uid_dic)
workers_by_age_to_assign_count = sp.get_workers_by_age_to_assign(
employment_rates, potential_worker_ages_left_count, age_by_uid_dic)
workplace_size_brackets = sp.get_workplace_size_brackets(datadir,
location,
state_location,
country_location,
use_default=True)
workplace_size_distr_by_brackets = sp.get_workplace_size_distr_by_brackets(
datadir,
state_location=state_location,
country_location=country_location,
use_default=True)
workplace_sizes = sp.generate_workplace_sizes(
workplace_size_distr_by_brackets, workplace_size_brackets,
workers_by_age_to_assign_count)
print(workplace_sizes)
uids_in_school)
# print(gen_school_sizes)
# print(age_brackets)
# print(age_by_brackets_dic[34])
gen_schools, gen_school_uids = sp.send_students_to_school(
gen_school_sizes, uids_in_school, uids_in_school_by_age,
ages_in_school_count, age_brackets, age_by_brackets_dic,
contact_matrix_dic)
# for s in range(5):
# print(Counter(gen_schools[s]))
# print(gen_schools[s])
emp_rates = sp.get_employment_rates(datadir, location, state_location,
country_location)
# print(emp_rates)
potential_worker_uids, potential_worker_uids_by_age, potential_worker_ages_left_count = sp.get_uids_potential_workers(
uids_in_school, uids_in_school_by_age, age_by_uid_dic)
workers_by_age_to_assign_count = sp.get_workers_by_age_to_assign(
emp_rates, potential_worker_ages_left_count, uids_by_age_dic)
# print(len(potential_worker_uids))
gen_schools, gen_school_uids, potential_worker_uids, potential_worker_uids_by_age, workers_by_age_to_assign_count = sp.assign_teachers_to_work(
gen_schools, gen_school_uids, emp_rates,
workers_by_age_to_assign_count, potential_worker_uids,
potential_worker_uids_by_age, potential_worker_ages_left_count)
# print(len(potential_worker_uids))
# for a in potential_worker_uids_by_age:
# print(a, len(potential_worker_uids_by_age[a]))
def test_get_uids_potential_workers(location='seattle_metro',
state_location='Washington',
country_location='usa'):
n = 10000
homes = sprw.read_setting_groups(datadir,
location,
state_location,
country_location,
folder_name,
'households',
n,
with_ages=True)
homes_by_uids, age_by_uid_dic = sp.assign_uids_by_homes(homes)
uids_in_school, uids_in_school_by_age, ages_in_school_count = sp.get_uids_in_school(
datadir,
n,
location,
state_location,
country_location,
age_by_uid_dic,
homes_by_uids,
use_default=False)
employment_rates = sp.get_employment_rates(
datadir,
location=location,
state_location=state_location,
country_location=country_location,
use_default=True)
school_size_distr_by_bracket = sp.get_school_size_distr_by_brackets(
datadir, location, state_location, country_location)
school_size_brackets = sp.get_school_size_brackets(datadir, location,
state_location,
country_location)
school_sizes = sp.generate_school_sizes(school_size_distr_by_bracket,
school_size_brackets,
uids_in_school)
age_brackets_filepath = sp.get_census_age_brackets_path(
datadir, state_location, country_location)
age_brackets = sp.get_age_brackets_from_df(age_brackets_filepath)
age_by_brackets_dic = sp.get_age_by_brackets_dic(age_brackets)
contact_matrix_dic = sp.get_contact_matrix_dic(
datadir, sheet_name='United States of America')
syn_schools, syn_school_uids, syn_school_types = sp.send_students_to_school(
school_sizes,
uids_in_school,
uids_in_school_by_age,
ages_in_school_count,
age_brackets,
age_by_brackets_dic,
contact_matrix_dic,
verbose=False)
potential_worker_uids, potential_worker_uids_by_age, potential_worker_ages_left_count = sp.get_uids_potential_workers(
syn_school_uids, employment_rates, age_by_uid_dic)
assert potential_worker_ages_left_count is not None
return potential_worker_uids, potential_worker_uids_by_age, employment_rates, age_by_uid_dic
def check_employment_age_distribution(pop,
n,
datadir,
figdir,
location=None,
state_location=None,
country_location=None,
file_path=None,
use_default=False,
test_prefix="",
skip_stat_check=False,
do_close=True):
"""
Check the population employment by age distribution against the reference data
Args:
pop : population dictionary
n : population size
datadir : root data directory which has resides the reference data
figdir : directory where to result files are saved
location : name of the location
state_location : name of the state the location is in
country_location : name of the country the location is in
file_path : file path to user specified gender by age bracket distribution data
use_default : if True, try to first use the other parameters to find data specific to the location
under study, otherwise returns default data drawing from Seattle, Washington.
test_prefix : used for prefix of the plot title
skip_stat_check : skip the statistics check for distribution
do_close : close the image immediately if set to True
Returns:
None.
Plots will be save to figdir if provided
"""
figdir = os.path.join(figdir, "employment")
er = sp.get_employment_rates(datadir=datadir,
location=location,
state_location=state_location,
country_location=country_location,
file_path=file_path,
use_default=use_default)
brackets = sp.get_census_age_brackets(datadir=datadir,
state_location=state_location,
country_location=country_location)
ageindex = sp.get_age_by_brackets_dic(brackets)
age_dist = sp.read_age_bracket_distr(datadir=datadir,
location=location,
state_location=state_location,
country_location=country_location,
file_path=file_path,
use_default=use_default)
# counting the actual population by age with employment including teachers and staffs
actual_employed_age_dist, actual_unemployed_age_dist = \
utilities.get_ids_count_by_param(pop,
condition_name=['wpid', 'sc_teacher', 'sc_staff'],
param='age')
utilities.plot_array([
actual_employed_age_dist[k] for k in sorted(actual_employed_age_dist)
],
datadir=figdir,
names=[k for k in sorted(actual_employed_age_dist)],
expect_label='employed by age count',
xlabel_rotation=90,
testprefix="employeed count by age " + test_prefix)
utilities.plot_array([
actual_unemployed_age_dist[k]
for k in sorted(actual_unemployed_age_dist)
],
datadir=figdir,
names=[k for k in sorted(actual_unemployed_age_dist)],
expect_label='unemployed by age count',
xlabel_rotation=90,
testprefix="unemployed count by age " + test_prefix)
sorted_actual_employed_rate = {}
actual_employed_rate = utilities.calc_rate(actual_employed_age_dist,
actual_unemployed_age_dist)
for i in er.keys():
if i in actual_employed_rate:
sorted_actual_employed_rate[i] = actual_employed_rate[i]
else:
sorted_actual_employed_rate[i] = 0
actual_values = np.array(list(sorted_actual_employed_rate.values()))
expected_values = np.array(list(er.values()))
if not skip_stat_check:
utilities.statistic_test(expected_values,
actual_values,
test="x",
comments="employment rate distribution check")
# plotting fill 0 to under age 16 for better display
filled_count = min(er.keys())
expected_values = np.insert(expected_values, 0, np.zeros(filled_count))
actual_values = np.insert(actual_values, 0, np.zeros(filled_count))
names = [i for i in range(0, max(er.keys()) + 1)]
# somehow double stacks for age 100
utilities.plot_array(
expected_values,
actual_values,
names=None,
datadir=figdir,
testprefix="employment rate distribution " + test_prefix,
do_close=do_close,
)
# check if total employment match
expected_employed_brackets = {k: 0 for k in brackets}
actual_employed_brackets = {k: 0 for k in brackets}
for i in names:
expected_employed_brackets[ageindex[i]] += expected_values[i]
if i in actual_employed_age_dist:
actual_employed_brackets[
ageindex[i]] += actual_employed_age_dist[i]
for i in expected_employed_brackets:
expected_employed_brackets[i] = expected_employed_brackets[i] / len(
brackets[i]) * age_dist[i] * n
expected_total = np.array(list(expected_employed_brackets.values()))
actual_total = np.array(list(actual_employed_brackets.values()))
utilities.plot_array(expected_total,
actual_total,
names=brackets.keys(),
datadir=figdir,
testprefix="employment total " + test_prefix,
do_close=do_close)
expected_etotal = np.round(np.sum(expected_total))
actual_etotal = np.round(np.sum(actual_total))
utilities.check_error_percentage(n,
expected_etotal,
actual_etotal,
name="employee")