def initialize_app(app):
from utils.perf import PerfCounter
app.layout = generate_layout
page_contents = []
pc = PerfCounter('pages')
pc.display('Rendering all')
# Generate all pages for checking input and output callbacks
pages = []
for page in all_pages.values():
if isinstance(page, Page):
pass
elif issubclass(page, Page):
page = page()
else:
raise Exception('Invalid page: %s' % page)
pc.display(page.path)
page_contents.append(page.render())
page_contents.append(dcc.Store(id=page.make_id('path-store')))
pages.append(page)
pc.display('done')
global _all_page_contents
_all_page_contents = page_contents
register_callbacks(app, pages)
def display_page(current_path, href):
if current_path is None:
return [None]
pc = PerfCounter('Page %s' % current_path)
pc.display('start')
if current_path.endswith('/') and len(current_path) > 1:
current_path = current_path.lstrip('/')
page_or_class = all_pages.get(current_path)
if not page_or_class:
return [html.H2('Sivua ei löydy')]
if isinstance(page_or_class, Page):
page = page_or_class
elif issubclass(page_or_class, Page):
page = page_or_class()
else:
return html.H2('Sisäinen virhe')
ret = [
page.render(),
dcc.Store(id=page.make_id('path-store'), data=page.path)
]
pc.display('finished')
return [ret]
def display_page(current_path):
pc = PerfCounter('Page %s' % current_path)
pc.display('start')
if current_path.endswith('/') and len(current_path) > 1:
current_path = current_path.lstrip('/')
current_page = None
for page_path, page in all_pages.items():
if current_path == page_path:
current_page = page
break
if not current_page:
return [html.Div()]
ret = current_page.render()
pc.display('finished')
return [ret]
def wrap_calc_func(*args, **kwargs):
should_profile = os.environ.get('PROFILE_CALC', '').lower() in ('1', 'true', 'yes')
only_if_in_cache = kwargs.pop('only_if_in_cache', False)
skip_cache = kwargs.pop('skip_cache', False)
var_store = kwargs.pop('variable_store', None)
if should_profile:
pc = PerfCounter('%s.%s' % (func.__module__, func.__name__))
pc.display('enter')
hash_data = _get_func_hash_data(func, None)
cache_key = _calculate_cache_key(func, hash_data, var_store=var_store)
assert 'variables' not in kwargs
assert 'datasets' not in kwargs
unknown_kwargs = set(kwargs.keys()) - set(['step_callback'])
if not args and not unknown_kwargs and not skip_cache:
should_cache_func = True
else:
should_cache_func = False
if should_cache_func:
ret = cache.get(cache_key)
if ret is not None: # calcfuncs must not return None
if should_profile:
pc.display('cache hit (%s)' % cache_key)
return ret
if only_if_in_cache:
if should_profile:
pc.display('cache miss so leaving as requested (%s)' % cache_key)
return None
if variables is not None:
kwargs['variables'] = {x: get_variable(y, var_store=var_store) for x, y in variables.items()}
if datasets is not None:
datasets_to_load = set(list(datasets.values())) - set(_dataset_cache.keys())
if datasets_to_load:
loaded_datasets = []
for dataset_name in datasets_to_load:
if should_profile:
ds_pc = PerfCounter('dataset %s' % dataset_name)
df = load_datasets(dataset_name)
if should_profile:
ds_pc.display('loaded')
del ds_pc
loaded_datasets.append(df)
for dataset_name, dataset in zip(datasets_to_load, loaded_datasets):
_dataset_cache[dataset_name] = dataset
kwargs['datasets'] = {ds_name: _dataset_cache[ds_url] for ds_name, ds_url in datasets.items()}
ret = func(*args, **kwargs)
if should_profile:
pc.display('func ret')
if should_cache_func:
assert ret is not None
cache.set(cache_key, ret, timeout=3600)
return ret
def wrap_calc_func(*args, **kwargs):
should_profile = os.environ.get('PROFILE_CALC',
'').lower() in ('1', 'true', 'yes')
if should_profile:
pc = PerfCounter('%s.%s' % (func.__module__, func.__name__))
pc.display('enter')
hash_data = _get_func_hash_data(func, None)
cache_key = _calculate_cache_key(func, hash_data)
assert 'variables' not in kwargs
assert 'datasets' not in kwargs
if not args and not kwargs:
should_cache_func = True
else:
should_cache_func = False
if should_cache_func:
ret = cache.get(cache_key)
if ret is not None: # calcfuncs must not return None
if should_profile:
pc.display('cache hit')
return ret
if variables is not None:
kwargs['variables'] = {
x: get_variable(y)
for x, y in variables.items()
}
if datasets is not None:
datasets_to_load = set(list(datasets.values())) - set(
_dataset_cache.keys())
if datasets_to_load:
loaded_datasets = []
for dataset_name in datasets_to_load:
if should_profile:
ds_pc = PerfCounter('dataset %s' % dataset_name)
df = load_datasets(dataset_name)
if should_profile:
ds_pc.display('loaded')
del ds_pc
loaded_datasets.append(df)
for dataset_name, dataset in zip(datasets_to_load,
loaded_datasets):
_dataset_cache[dataset_name] = dataset
kwargs['datasets'] = {
ds_name: _dataset_cache[ds_url]
for ds_name, ds_url in datasets.items()
}
ret = func(*args, **kwargs)
if should_profile:
pc.display('func ret')
if should_cache_func:
assert ret is not None
cache.set(cache_key, ret, timeout=600)
return ret
def simulate_individuals(variables,
step_callback=None,
callback_day_interval=1):
pc = PerfCounter()
age_structure = get_population_for_area().sum(axis=1)
ipc = get_initial_population_condition()
age_to_group = make_age_groups()
age_groups = list(np.unique(age_to_group))
pop_params = dict(
age_structure=age_structure,
contacts_per_day=get_contacts_per_day(),
initial_population_condition=ipc,
age_groups=dict(
labels=age_groups,
age_indices=[age_groups.index(x) for x in age_to_group]),
imported_infection_ages=variables['imported_infection_ages'],
)
df = get_contacts_per_day()
hc_params = dict(hospital_beds=variables['hospital_beds'],
icu_units=variables['icu_units'])
disease_params = create_disease_params(variables)
context = model.Context(population_params=pop_params,
healthcare_params=hc_params,
disease_params=disease_params,
start_date=variables['start_date'],
random_seed=variables['random_seed'])
start_date = date.fromisoformat(variables['start_date'])
ivs = get_active_interventions(variables)
for iv in ivs:
context.add_intervention(iv)
pc.measure()
days = variables['simulation_days']
date_index = pd.date_range(start_date, periods=days)
df = pd.DataFrame(
columns=POP_ATTRS + STATE_ATTRS + EXPOSURES_ATTRS +
['us_per_infected'],
index=date_index,
)
ag_array = np.empty((days, len(POP_ATTRS), len(age_groups)), dtype='i')
for day in range(days):
s = context.generate_state()
today_date = (start_date + timedelta(days=day)).isoformat()
for idx, attr in enumerate(POP_ATTRS):
ag_array[day, idx, :] = s[attr]
rec = {attr: s[attr].sum() for attr in POP_ATTRS}
for state_attr in STATE_ATTRS:
rec[state_attr] = s[state_attr]
for place, nr in s['daily_contacts'].items():
key = 'exposures_%s' % place
assert key in df.columns
rec[key] = nr
rec['us_per_infected'] = pc.measure(
) * 1000 / rec['infected'] if rec['infected'] else 0
if False:
st = '\n%-15s' % today_date
for ag in age_groups:
st += '%8s' % ag
print(st)
for attr in ('all_detected', 'in_ward', 'dead', 'cum_icu'):
st = '%-15s' % attr
t = s[attr].sum()
for val in s[attr]:
st += '%8.2f' % ((val / t) * 100)
print(st)
if False:
dead = context.get_population_stats('dead')
all_infected = context.get_population_stats('all_infected')
detected = context.get_population_stats('all_detected')
age_groups = pd.interval_range(0, 80, freq=10, closed='left')
age_groups = age_groups.append(
pd.Index([pd.Interval(80, 100, closed='left')]))
s = pd.Series(dead)
dead_by_age = s.groupby(pd.cut(s.index, age_groups)).sum()
dead_by_age.name = 'dead'
s = pd.Series(all_infected)
infected_by_age = s.groupby(pd.cut(s.index, age_groups)).sum()
infected_by_age.scenario_name = 'infected'
s = pd.Series(detected)
detected_by_age = s.groupby(pd.cut(s.index, age_groups)).sum()
detected_by_age.name = 'detected'
print(dead_by_age / sum(dead_by_age) * 100)
print(infected_by_age / sum(infected_by_age) * 100)
print(detected_by_age / sum(detected_by_age) * 100)
#zdf = pd.DataFrame(dead_by_age)
#zdf['infected'] = infected_by_age
#zdf['ifr'] = zdf.dead.divide(zdf.infected.replace(0, np.inf)) * 100
#print(zdf)
df.loc[today_date] = rec
by_age_group = POP_ATTRS
if step_callback is not None and (day % callback_day_interval == 0
or day == range(days) - 1):
ret = step_callback(df)
if not ret:
raise ExecutionInterrupted()
context.iterate()
if False:
import cProfile
import pstats
cProfile.runctx("context.iterate()", globals(), locals(),
"profile.prof")
s = pstats.Stats("profile.prof")
s.strip_dirs().sort_stats("cumtime").print_stats()
arr = ag_array.flatten()
adf = pd.DataFrame(arr,
index=pd.MultiIndex.from_product(
[date_index, POP_ATTRS, age_groups],
names=['date', 'attr', 'age_group']),
columns=['pop'])
adf = adf.unstack('attr').unstack('age_group')
adf.columns = adf.columns.droplevel()
return df, adf
def simulate_individuals(variables, step_callback=None):
pc = PerfCounter()
df = get_population_for_area().sum(axis=1)
ages = df.index.values
counts = df.values
avg_contacts_per_day = get_physical_contacts_for_country()
hc_cap = (variables['hospital_beds'], variables['icu_units'])
max_age = max(ages)
age_counts = np.array(np.zeros(max_age + 1, dtype=np.int32))
for age, count in zip(ages, counts):
age_counts[age] = count
people = create_population(age_counts)
avg_contacts = np.array(avg_contacts_per_day.values, dtype=np.float32)
assert avg_contacts.size == max_age + 1
pop = Population(age_counts, avg_contacts)
hc = HealthcareSystem(hc_cap[0], hc_cap[1])
sevvar = variables['p_severe']
sev_arr = np.ndarray((len(sevvar), 2), dtype=np.float32)
for idx, (age, sev) in enumerate(sevvar):
sev_arr[idx] = (age, sev / 100)
disease = Disease(
p_infection=variables['p_infection'] / 100,
p_asymptomatic=variables['p_asymptomatic'] / 100,
p_severe=sev_arr,
p_critical=variables['p_critical'] / 100,
p_hospital_death=variables['p_hospital_death'] / 100,
p_icu_death=variables['p_icu_death'] / 100,
p_hospital_death_no_beds=variables['p_hospital_death_no_beds'] / 100,
p_icu_death_no_beds=variables['p_icu_death_no_beds'] / 100,
)
context = Context(pop,
people,
hc,
disease,
start_date=variables['start_date'])
start_date = date.fromisoformat(variables['start_date'])
ivs = nb.typed.List()
for iv in variables['interventions']:
iv_id = iv[0]
iv_date = iv[1]
if len(iv) > 2:
iv_value = iv[2]
else:
iv_value = None
# Extremely awkward, but Numba poses some limitations.
ivs.append(make_iv(context, iv_id, iv_date, value=iv_value))
context.interventions = ivs
pc.display('after init')
days = variables['simulation_days']
df = pd.DataFrame(columns=POP_ATTRS + STATE_ATTRS,
index=pd.date_range(start_date, periods=days))
for day in range(days):
state = context.generate_state()
rec = {attr: sum(getattr(state, attr)) for attr in POP_ATTRS}
rec['hospital_beds'] = state.available_hospital_beds
rec['icu_units'] = state.available_icu_units
rec['r'] = state.r
rec['exposed_per_day'] = state.exposed_per_day
rec['tests_run_per_day'] = state.tests_run_per_day
rec['sim_time_ms'] = pc.measure()
d = start_date + timedelta(days=day)
df.loc[d] = rec
if step_callback is not None:
ret = step_callback(df)
if not ret:
raise ExecutionInterrupted()
context.iterate()
return df
def simulate_individuals(variables, step_callback=None):
pc = PerfCounter()
df = get_population_for_area().sum(axis=1)
ages = df.index.values
counts = df.values
avg_contacts_per_day = get_contacts_for_country()
hc_cap = (variables['hospital_beds'], variables['icu_units'])
max_age = max(ages)
age_counts = np.array(np.zeros(max_age + 1, dtype=np.int32))
for age, count in zip(ages, counts):
age_counts[age] = count
pop = model.Population(age_counts, list(avg_contacts_per_day.items()))
hc = model.HealthcareSystem(
beds=hc_cap[0],
icu_units=hc_cap[1],
p_detected_anyway=variables['p_detected_anyway'] / 100)
disease = create_disease(variables)
context = model.Context(pop,
hc,
disease,
start_date=variables['start_date'],
random_seed=variables['random_seed'])
start_date = date.fromisoformat(variables['start_date'])
for iv in variables['interventions']:
d = (date.fromisoformat(iv[1]) - start_date).days
if len(iv) > 2:
val = iv[2]
else:
val = 0
context.add_intervention(d, iv[0], val)
pc.measure()
days = variables['simulation_days']
df = pd.DataFrame(columns=POP_ATTRS + STATE_ATTRS + ['us_per_infected'],
index=pd.date_range(start_date, periods=days))
for day in range(days):
s = context.generate_state()
rec = {attr: sum(s[attr]) for attr in POP_ATTRS}
for state_attr in STATE_ATTRS:
rec[state_attr] = s[state_attr]
rec['us_per_infected'] = pc.measure(
) * 1000 / rec['infected'] if rec['infected'] else 0
"""
dead = context.get_population_stats('dead')
all_infected = context.get_population_stats('all_infected')
age_groups = pd.interval_range(0, 100, freq=10, closed='left')
s = pd.Series(dead)
dead_by_age = s.groupby(pd.cut(s.index, age_groups)).sum()
dead_by_age.name = 'dead'
s = pd.Series(all_infected)
infected_by_age = s.groupby(pd.cut(s.index, age_groups)).sum()
zdf = pd.DataFrame(dead_by_age)
zdf['infected'] = infected_by_age
zdf['ifr'] = zdf.dead.divide(zdf.infected.replace(0, np.inf)) * 100
print(zdf)
"""
d = start_date + timedelta(days=day)
df.loc[d] = rec
if step_callback is not None:
ret = step_callback(df)
if not ret:
raise ExecutionInterrupted()
context.iterate()
if False:
import cProfile
import pstats
cProfile.runctx("context.iterate()", globals(), locals(),
"profile.prof")
s = pstats.Stats("profile.prof")
s.strip_dirs().sort_stats("time").print_stats()
return df
def simulate_individuals(variables, step_callback=None):
pc = PerfCounter()
age_structure = get_population_for_area().sum(axis=1)
pop_params = dict(
age_structure=age_structure,
contacts_per_day=get_contacts_per_day(),
)
hc_params = dict(hospital_beds=variables['hospital_beds'], icu_units=variables['icu_units'])
disease_params = create_disease_params(variables)
context = model.Context(
population_params=pop_params,
healthcare_params=hc_params,
disease_params=disease_params,
start_date=variables['start_date'],
random_seed=variables['random_seed']
)
start_date = date.fromisoformat(variables['start_date'])
for iv in variables['interventions']:
d = (date.fromisoformat(iv[1]) - start_date).days
if len(iv) > 2:
val = iv[2]
else:
val = 0
context.add_intervention(d, iv[0], val)
pc.measure()
days = variables['simulation_days']
df = pd.DataFrame(
columns=POP_ATTRS + STATE_ATTRS + ['us_per_infected'],
index=pd.date_range(start_date, periods=days)
)
for day in range(days):
s = context.generate_state()
rec = {attr: sum(s[attr]) for attr in POP_ATTRS}
for state_attr in STATE_ATTRS:
rec[state_attr] = s[state_attr]
rec['us_per_infected'] = pc.measure() * 1000 / rec['infected'] if rec['infected'] else 0
if False:
dead = context.get_population_stats('dead')
all_infected = context.get_population_stats('all_infected')
age_groups = pd.interval_range(0, 100, freq=10, closed='left')
s = pd.Series(dead)
dead_by_age = s.groupby(pd.cut(s.index, age_groups)).sum()
dead_by_age.name = 'dead'
s = pd.Series(all_infected)
infected_by_age = s.groupby(pd.cut(s.index, age_groups)).sum()
print(infected_by_age)
zdf = pd.DataFrame(dead_by_age)
zdf['infected'] = infected_by_age
zdf['ifr'] = zdf.dead.divide(zdf.infected.replace(0, np.inf)) * 100
#print(zdf)
d = start_date + timedelta(days=day)
df.loc[d] = rec
if step_callback is not None:
ret = step_callback(df)
if not ret:
raise ExecutionInterrupted()
context.iterate()
if False:
import cProfile
import pstats
cProfile.runctx("context.iterate()", globals(), locals(), "profile.prof")
s = pstats.Stats("profile.prof")
s.strip_dirs().sort_stats("time").print_stats()
return df