# Separate the scenarios out and save them
final_msims = sc.objdict()
final_sims = sc.objdict()
keys = set([sim.label for sim in big_msim.sims])
for key in keys:
final_sims[key] = []
for sim in big_msim.sims:
if sim.label == key:
final_sims[key].append(sim)
for key in keys:
final_msims[key] = cv.MultiSim(sims=final_sims[key])
final_msims[key].reduce()
sc.saveobj(filename='uk_scens.obj', obj=final_msims)
else:
final_msims = sc.loadobj(filename='uk_scens.obj')
# Save the key figures
if do_plot:
# Make individual plots
plot_customizations = dict(
interval=90, # Number of days between tick marks
dateformat='%m/%Y', # Date format for ticks
fig_args={'figsize': (14, 8)}, # Size of the figure (x and y)
axis_args={'left': 0.15}, # Space on left side of plot
font_size=24, # Size of font to use
)
for mkey, msim in final_msims.items():
def test_legacy():
'''
Preserved for completeness, but too fragile to be used in automated unit testing
due to reliance on openpyxl (which is not a required Sciris dependency).
'''
# Define filenames
filedir = 'files' + os.sep
files = sc.prettyobj()
files.excel = filedir + 'test.xlsx'
files.binary = filedir + 'test.obj'
files.text = filedir + 'text.txt'
files.zip = filedir + 'test.zip'
tidyup = True
# Define the test data
nrows = 15
ncols = 3
testdata = pl.zeros((nrows + 1, ncols),
dtype=object) # Includes header row
testdata[0, :] = ['A', 'B', 'C'] # Create header
testdata[1:, :] = pl.rand(nrows, ncols) # Create data
# Test spreadsheet writing, and create the file for later
formats = {
'header': {
'bold': True,
'bg_color': '#3c7d3e',
'color': '#ffffff'
},
'plain': {},
'big': {
'bg_color': '#ffcccc'
}
}
formatdata = pl.zeros(
(nrows + 1, ncols),
dtype=object) # Format data needs to be the same size
formatdata[1:, :] = 'plain' # Format data
formatdata[1:, :][testdata[
1:, :] > 0.7] = 'big' # Find "big" numbers and format them differently
formatdata[0, :] = 'header' # Format header
sc.savespreadsheet(filename=files.excel,
data=testdata,
formats=formats,
formatdata=formatdata)
# Test loading
sc.heading('Loading spreadsheet')
data = sc.loadspreadsheet(files.excel)
print(data)
excel_path = filedir + 'exampledata.xlsx'
if os.path.exists(excel_path):
sc.heading('Reading cells')
wb = sc.Spreadsheet(
filename=excel_path
) # Load a sample databook to try pulling cells from
celltest = wb.readcells(method='xlrd',
sheetname='Baseline year population inputs',
cells=[[46, 2], [47,
2]]) # Grab cells using xlrd
celltest2 = wb.readcells(
method='openpyexcel',
wbargs={'data_only': True},
sheetname='Baseline year population inputs',
cells=[[46, 2], [47, 2]]
) # Grab cells using openpyexcel. You have to set wbargs={'data_only': True} to pull out cached values instead of formula strings
print('xlrd output: %s' % celltest)
print('openpyxl output: %s' % celltest2)
else:
print(f'{excel_path} not found, skipping...')
sc.heading('Loading a blobject')
blob = sc.Blobject(files.excel)
f = blob.tofile()
wb = openpyexcel.load_workbook(f)
ws = wb.active
ws['B7'] = 'Hi! '
wb.save(f)
blob.load(f)
blob.tofile(output=False)
data = sc.loadspreadsheet(fileobj=blob.bytes)
print(blob)
sc.pp(data)
# Test spreadsheet saving
sc.heading('Using a Spreadsheet')
S = sc.Spreadsheet(files.excel)
S.writecells(cells=['A6', 'B7', 'C8', 'D9'],
vals=['This', 'is', 'a', 'test']) # Method 1
S.writecells(cells=[pl.array([7, 1]) + i for i in range(4)],
vals=['And', 'so', 'is', 'this']) # Method 2
newdata = (pl.rand(3, 3) * 100).round()
S.writecells(startrow=14, startcol=1, vals=newdata,
verbose=True) # Method 3
S.save()
data = S.readcells(header=False)
print(S)
sc.pp(data)
sc.heading('Saveobj/loadobj')
sc.saveobj(files.binary, testdata)
obj = sc.loadobj(files.binary)
print(obj)
sc.heading('Savetext/loadtext')
sc.savetext(files.text, testdata)
obj = sc.loadtext(files.text)
print(obj)
sc.heading('Get files')
print('Files in current folder:')
sc.pp(sc.getfilelist())
sc.heading('Save zip')
sc.savezip(files.zip, [files.text, files.excel])
'''
Check that loading an object with a non-existent class works. The file
deadclass.obj was created with:
deadclass.py:
-------------------------------------------------
class DeadClass():
def __init__(self, x):
self.x = x
-------------------------------------------------
then:
-------------------------------------------------
import deadclass as dc
import sciris as sc
deadclass = dc.DeadClass(238473)
sc.saveobj('deadclass.obj', deadclass)
-------------------------------------------------
'''
dead_path = filedir + 'deadclass.obj'
if os.path.exists(dead_path):
sc.heading('Intentionally loading corrupted file')
obj = sc.loadobj(dead_path)
print('Loading corrupted object succeeded, x=%s' % obj.x)
else:
print(f'{dead_path} not found, skipping...')
# Tidy up
if tidyup:
sc.blank()
sc.heading('Tidying up')
for fn in [files.excel, files.binary, files.text, files.zip]:
try:
os.remove(fn)
print('Removed %s' % fn)
except:
pass
print('Done, all fileio tests succeeded')
return S
import sciris as sc
bod = sc.loadobj('gbd-data.dat')
countrydata = sc.loadspreadsheet('country-data.xlsx')
bod_countries = set(bod.keys())
data_countries = set(countrydata['name'].tolist())
print(f'Matching:\n{data_countries.intersection(bod_countries)}\n')
print(f'Extra:\n{data_countries-bod_countries}\n')
print(f'Missing:\n{bod_countries-data_countries}\n')
print('Done.')
import sciris as sc
interv_data = sc.loadspreadsheet('rapid_interventions.xlsx')
spend_data = sc.loadspreadsheet('rapid_spending.xlsx')
R = sc.loadobj('results/rapid_results.obj')
i_int_names = set(interv_data['Short name'].tolist())
s_int_names = set(spend_data['Short name'].tolist())
print(f'Matching:\n{i_int_names.intersection(s_int_names)}\n')
print(f'Extra:\n{s_int_names-i_int_names}\n')
print(f'Missing:\n{i_int_names-s_int_names}\n')
msim = cv.MultiSim(sims)
msim.run()
# msim.run(par_args={'n_cpus':24})
# Figure out the seeds that give a good fit
mismatches = np.array(
[sim.compute_fit().mismatch for sim in msim.sims])
threshold = np.quantile(mismatches, 0.01) # Take the best 1%
goodseeds = [
i for i in range(len(mismatches)) if mismatches[i] < threshold
]
sc.saveobj(f'{resfolder}/goodseeds.obj', goodseeds)
# Run calibration with best-fitting seeds and parameters
elif whattorun == 'finalisefit':
goodseeds = sc.loadobj(f'{resfolder}/goodseeds.obj')
s0 = make_sim(seed=1, beta=beta, end_day='2020-08-25', verbose=-1)
sims = []
for seed in goodseeds:
sim = s0.copy()
sim['rand_seed'] = seed
sim.set_seed()
sim.label = f"Sim {seed}"
sims.append(sim)
msim = cv.MultiSim(sims)
msim.run()
if save_sim:
msim.save(f'{resfolder}/uk_sim.obj')
if do_plot:
msim.reduce()
import os
import sciris as sc
import numpy as np
import covasim as cv
t = cv.daydiff('2020-01-21', '2020-09-01')
scenarios = ['low_comp', 'high_comp', 'low_comp_notschools', 'high_comp_notschools']
scenario = 0
trace = 0.47
for test, expected in [(0.0171, 110000), (0.12, 4000)]:
## run the simulation
cmd = "python UK_Test-Trace_phaseplots_26August.py --samples 12 --scenario %s --test %.02f --trace %.02f" % (scenario, test, trace)
os.system(cmd)
## load the results
outfile = "%s/test%.02f-trace%.02f.obj" % (scenarios[scenario], test, trace)
results = sc.loadobj(outfile)
## ensure that we get at least some infections
if not np.any(results["msim"]["new_infections"]):
raise ValueError("%s got no infections at all, how strange\ncommand: %s" % (scenarios[scenario], cmd))
## ensure that the expected value of infections after september is within 10% of expected
infections = max(results["msim"]["new_infections"][t:])
if infections < 0.9*expected or infections > 1.1*expected:
raise ValueError("%s: with test=%s and trace=%s expected %s peak infections after %s and got %s\ncommand: %s" % (scenarios[scenario], test, trace, expected, september, infections, cmd))
import pylab as pl
import sciris as sc
import hiptool as hp
sc.heading('Initializing...')
sc.tic()
dosave = True
missing_data = ['remove', 'assumption'][1] # Choose how to handle missing data
# Load input files
D = sc.odict() # Data
R = sc.odict() # Results
bod_data = sc.loadobj('gbd-data.dat')
country_data = sc.loadspreadsheet('country-data.xlsx')
spend_data = sc.loadspreadsheet('rapid_spending.xlsx')
baseline_factor = country_data.findrow(
'Zambia', asdict=True)['icer_multiplier'] # Zambia was used for this
# Get country list
countries = sc.dcp(spend_data.cols)
countries.remove('Short name')
# Create default
P = hp.Project()
P.loadburden(filename='rapid_BoD.xlsx')
P.loadinterventions(filename='rapid_interventions.xlsx')
ninterventions = P.intervsets[0].data.nrows
# Load data
sc.heading('Loading data...')
xgapr = 0.075
ygapb = 0.065
ygapm = 0.05
ygapt = 0.04
nrows = 2
ncols = 2
dc = 0.025
dx = (1 - ncols * xgapm - xgapl - xgapr - dc) / ncols
dy = (1 - (nrows - 1) * ygapm - ygapb - ygapt) / nrows
ax = {}
# Load all results
sweep_summaries = {}
for scen in scenarios:
filepath = f'{resfolder}/uk_tti_sweeps_{scen}.obj'
sweep_summaries[scen] = sc.loadobj(filepath)
# Translate them into a dict of dataframes
dfs = sc.odict()
cbar_lims = {}
for res in resnames.keys():
dfs[res] = sc.odict()
for scen in scenarios:
dfs[res][scen] = pd.DataFrame(sweep_summaries[scen][res])
if res == 'cum_inf':
dfs[res][scen] = (dfs[res][scen]) / 1e6
elif res == 'cum_death':
dfs[res][scen] = (dfs[res][scen]) / 1e3
else:
dfs[res][scen] /= 1e3
cbar_lims[res] = max(dfs[res]['masks15_notschools'].max())
import synthpops as sp
import sciris as sc
import os
n = 5000
f = os.path.join(sp.datadir, 'synthpop_{0}.pop'.format(n))
print(f)
pop = sc.loadobj(f)
print(pop)
print(type(pop))
print()
keys = pop.keys()
print(keys)
print(pop['popdict'])
sim = s0.copy()
sim['rand_seed'] = seed
sim.set_seed()
sim.label = f"Sim {seed}"
sims.append(sim)
msim = cv.MultiSim(sims)
msim.run()
fitsummary.append(
[sim.compute_fit().mismatch for sim in msim.sims])
sc.saveobj(f'{resfolder}/fitsummary.obj', fitsummary)
# Run calibration with best-fitting seeds and parameters
elif whattorun == 'finialisefit':
sims = []
fitsummary = sc.loadobj(f'{resfolder}/fitsummary.obj')
for bn, beta in enumerate(betas):
goodseeds = [i for i in range(n_runs) if fitsummary[bn][i] < 163]
sc.blank()
print('---------------\n')
print(f'Beta: {beta}, goodseeds: {len(goodseeds)}')
print('---------------\n')
if len(goodseeds) > 0:
s0 = make_sim(seed=1, beta=beta, end_day=data_end)
for seed in goodseeds:
sim = s0.copy()
sim['rand_seed'] = seed
sim.set_seed()
sim.label = f"Sim {seed}"
sims.append(sim)
ygapt = 0.06
nrows = 2
ncols = 3
dx = (1 - (ncols - 1) * xgapm - xgapl - xgapr) / ncols
dy = (1 - (nrows - 1) * ygapm - ygapb - ygapt) / nrows
nplots = nrows * ncols
ax = {}
# Import files
filepaths = [
f'{resfolder}/vietnam_sim_{policy}.obj'
for policy in ['remain', 'drop', 'dynamic']
]
sims = []
for fp in filepaths:
simsfile = sc.loadobj(fp)
sims.append(simsfile.sims)
sim = sims[0][0] # Extract a sim to refer to
colors = [[0.03137255, 0.37401, 0.63813918, 1.], '#c75649']
linecolor = [0, 0, 0]
importday = sim.day('2020-12-01')
# Add text
headings = [
" Constant high compliance ",
" Increased apathy ",
" Self-regulating behavior "
]
epsx = 0.003
epsy = 0.008
}}
t_keys = list(testings.keys())
n_testings = len(testings)
all_scens = sc.odict()
for tkey, testing in testings.items():
scen = sc.dcp(origscen)
for stype, spec in scen.items():
if spec is not None:
spec['testing'] = testing
scen['es']['verbose'] = scen['ms']['verbose'] = scen['hs'][
'verbose'] = debug
all_scens[tkey] = scen
# Create the sim
people = sc.loadobj(bypass_popfile)
base_sim = cs.create_sim(params,
pop_size=pop_size,
load_pop=False,
people=people,
verbose=0.1)
#%% Run the sims
sims = []
for key, scen in all_scens.items():
for seed in range(n_seeds):
sim = sc.dcp(base_sim)
sim.set_seed(seed=sim['rand_seed'] + seed)
sim.label = key
sim['interventions'] += [cvsch.schools_manager(scen)]
sims.append(sim)
import pylab as pl
import numpy as np
from matplotlib import ticker
import datetime as dt
import matplotlib.patches as patches
# Filepaths
resultsfolder = 'sweeps'
figsfolder = 'figs'
simsfilepath = 'nsw_calibration.obj'
today = '2020-09-30'
T = sc.tic()
# Import files
simsfile = sc.loadobj(simsfilepath)
# Define plotting functions
#%% Helper functions
def format_ax(ax, sim, key=None):
@ticker.FuncFormatter
def date_formatter(x, pos):
return (sim['start_day'] + dt.timedelta(days=x)).strftime('%b-%d')
ax.xaxis.set_major_formatter(date_formatter)
pl.xlim([0, 213])
# pl.xlim([0, sim['n_days']])
sc.boxoff()
return