def get_output_files(sim):
''' Create output files for download '''
datestamp = sc.getdate(dateformat='%Y-%b-%d_%H.%M.%S')
ss = sim.to_excel()
files = {}
files['xlsx'] = {
'filename':
f'covasim_results_{datestamp}.xlsx',
'content':
'data:application/vnd.openxmlformats-officedocument.spreadsheetml.sheet;base64,'
+ base64.b64encode(ss.blob).decode("utf-8"),
}
json_string = sim.to_json(verbose=False)
files['json'] = {
'filename':
f'covasim_results_{datestamp}.json',
'content':
'data:application/text;base64,' +
base64.b64encode(json_string.encode()).decode("utf-8"),
}
# Summary output
summary = {
'days': sim.npts - 1,
'cases': round(sim.results['cum_infections'][-1]),
'deaths': round(sim.results['cum_deaths'][-1]),
}
return files, summary
def set_metadata(self, filename):
''' Set the metadata for the simulation -- creation time and filename '''
self.created = sc.now()
if filename is None:
datestr = sc.getdate(obj=self.created,
dateformat='%Y-%b-%d_%H.%M.%S')
self.filename = f'covasim_{datestr}.sim'
return
def jsonify_project(project_id, verbose=False):
""" Return the project json, given the Project UID. """
proj = load_project(
project_id
) # Load the project record matching the UID of the project passed in.
json = {
'project': {
'id': str(proj.uid),
'name': proj.name,
'username': proj.webapp.username,
'hasData': len(proj.burdensets) > 0 and len(proj.intervsets) > 0,
'creationTime': sc.getdate(proj.created),
'updatedTime': sc.getdate(proj.modified),
}
}
if verbose: sc.pp(json)
return json
def savefig(filename=None, comments=None, **kwargs):
'''
Wrapper for Matplotlib's savefig() function which automatically stores Covasim
metadata in the figure. By default, saves
Args:
filename (str): name of the file to save to (default, timestamp)
comments (str): additional metadata to save to the figure
kwargs (dict): passed to savefig()
**Example**::
cv.Sim().run(do_plot=True)
filename = cv.savefig()
'''
# Handle inputs
dpi = kwargs.pop('dpi', 150)
metadata = kwargs.pop('metadata', {})
if filename is None:
now = sc.getdate(dateformat='%Y-%b-%d_%H.%M.%S')
filename = f'covasim_{now}.png'
metadata = {}
metadata['Covasim version'] = cvver.__version__
gitinfo = git_info()
for key, value in gitinfo['covasim'].items():
metadata[f'Covasim {key}'] = value
for key, value in gitinfo['called_by'].items():
metadata[f'Covasim caller {key}'] = value
metadata['Covasim current time'] = sc.getdate()
metadata['Covasim calling file'] = get_caller()
if comments:
metadata['Covasim comments'] = comments
# Handle different formats
lcfn = filename.lower() # Lowercase filename
if lcfn.endswith('pdf') or lcfn.endswith('svg'):
metadata = {
'Keywords': str(metadata)
} # PDF and SVG doesn't support storing a dict
# Save the figure
pl.savefig(filename, dpi=dpi, metadata=metadata, **kwargs)
return filename
def __init__(self,
sim=None,
metapars=None,
scenarios=None,
basepars=None,
filename=None):
# For this object, metapars are the foundation
default_pars = make_metapars() # Start with default pars
super().__init__(
default_pars) # Initialize and set the parameters as attributes
# Handle filename
self.created = sc.now()
if filename is None:
datestr = sc.getdate(obj=self.created,
dateformat='%Y-%b-%d_%H.%M.%S')
filename = f'covasim_scenarios_{datestr}.scens'
self.filename = filename
# Handle scenarios -- by default, create a baseline scenario
if scenarios is None:
scenarios = sc.dcp(default_scenario)
self.scenarios = scenarios
# Handle metapars
if metapars is None:
metapars = {}
self.metapars = metapars
self.update_pars(self.metapars)
# Create the simulation and handle basepars
if sim is None:
sim = cvsim.Sim()
self.base_sim = sim
if basepars is None:
basepars = {}
self.basepars = basepars
self.base_sim.update_pars(self.basepars)
self.base_sim.validate_pars()
self.base_sim.init_results()
# Copy quantities from the base sim to the main object
self.npts = self.base_sim.npts
self.tvec = self.base_sim.tvec
self.reskeys = self.base_sim.reskeys
# Create the results object; order is: results key, scenario, best/low/high
self.sims = sc.objdict()
self.allres = sc.objdict()
for reskey in self.reskeys:
self.allres[reskey] = sc.objdict()
for scenkey in scenarios.keys():
self.allres[reskey][scenkey] = sc.objdict()
for nblh in ['name', 'best', 'low', 'high']:
self.allres[reskey][scenkey][
nblh] = None # This will get populated below
return
def __repr__(self):
''' Print out useful information when called '''
output = sc.objrepr(self)
output += ' Project name: %s\n' % self.name
output += ' Country: %s\n' % self.country
output += '\n'
output += ' Burden sets: %i\n' % len(self.burdensets)
output += ' Intervention sets: %i\n' % len(self.intervsets)
output += ' Health packages: %i\n' % len(self.packagesets)
output += '\n'
output += ' HP version: %s\n' % self.version
output += ' Date created: %s\n' % sc.getdate(self.created)
output += ' Date modified: %s\n' % sc.getdate(self.modified)
output += ' Git branch: %s\n' % self.gitinfo['branch']
output += ' Git hash: %s\n' % self.gitinfo['hash']
output += ' UID: %s\n' % self.uid
output += '============================================================\n'
return output
def set_metadata(self, simfile, label):
''' Set the metadata for the simulation -- creation time and filename '''
self.created = sc.now()
self.version = cvv.__version__
self.git_info = cvm.git_info()
if simfile is None:
datestr = sc.getdate(obj=self.created, dateformat='%Y-%b-%d_%H.%M.%S')
self.simfile = f'covasim_{datestr}.sim'
if label is not None:
self.label = label
return
def savefig(filename=None, dpi=None, comments=None, **kwargs):
'''
Wrapper for Matplotlib's savefig() function which automatically stores Covasim
metadata in the figure. By default, saves
Args:
filename (str): name of the file to save to (default, timestamp)
dpi (int): resolution of image (default 150)
comments (str): additional metadata to save to the figure
kwargs (dict): passed to savefig()
**Example**::
cv.Sim().run(do_plot=True)
filename = cv.savefig()
'''
# Handle inputs
dpi = kwargs.get('dpi', 150)
metadata = kwargs.get('metadata', {})
if filename is None:
now = sc.getdate(dateformat='%Y-%b-%d_%H.%M.%S')
filename = f'covasim_{now}.png'
metadata = {}
metadata['Covasim version'] = cvver.__version__
gitinfo = git_info()
for key, value in gitinfo.items():
metadata[f'Covasim {key}'] = value
metadata['Covasim current time'] = sc.getdate()
metadata['Covasim calling file'] = get_caller()
if comments:
metadata['Covasim comments'] = comments
# Save the figure
pl.savefig(filename, dpi=dpi, metadata=metadata, **kwargs)
return filename
def download_projects(project_keys, username):
"""
Given a list of project UIDs, make a .zip file containing all of these
projects as .prj files, and return the full path to this file.
"""
basedir = get_path(
'', username) # Use the downloads directory to put the file in.
project_paths = []
for project_key in project_keys:
proj = load_project(project_key)
project_path = proj.save(folder=basedir)
project_paths.append(project_path)
zip_fname = 'Projects %s.zip' % sc.getdate(
) # Make the zip file name and the full server file path version of the same..
server_zip_fname = get_path(zip_fname, username)
sc.savezip(server_zip_fname, project_paths)
print(">> load_zip_of_prj_files %s" %
(server_zip_fname)) # Display the call information.
return server_zip_fname # Return the server file name.
def test_readdate():
sc.heading('Test string-to-date conversion')
string1 = '2020-Mar-21'
string2 = '2020-03-21'
string3 = 'Sat Mar 21 23:13:56 2020'
dateobj1 = sc.readdate(string1)
dateobj2 = sc.readdate(string2)
sc.readdate(string3)
assert dateobj1 == dateobj2
with pytest.raises(ValueError):
sc.readdate('Not a date')
# Automated tests
formats_to_try = sc.readdate(return_defaults=True)
for key, fmt in formats_to_try.items():
datestr = sc.getdate(dateformat=fmt)
dateobj = sc.readdate(datestr, dateformat=fmt)
print(f'{key:15s} {fmt:22s}: {dateobj}')
return dateobj1
def __init__(self, pars=None, datafile=None, filename=None):
default_pars = cvpars.make_pars() # Start with default pars
super().__init__(
default_pars) # Initialize and set the parameters as attributes
self.datafile = datafile # Store this
self.data = None
if datafile is not None: # If a data file is provided, load it
self.data = cvpars.load_data(datafile)
self.created = sc.now()
if filename is None:
datestr = sc.getdate(obj=self.created,
dateformat='%Y-%b-%d_%H.%M.%S')
filename = f'covasim_{datestr}.sim'
self.filename = filename
self.stopped = None # If the simulation has stopped
self.results_ready = False # Whether or not results are ready
self.people = {}
self.results = {}
self.calculated = {}
if pars is not None:
self.update_pars(pars)
return
def run_sim(sim_pars=None, epi_pars=None, intervention_pars=None, datafile=None, show_animation=False, n_days=90, verbose=True):
''' Create, run, and plot everything '''
err = ''
try:
# Fix up things that JavaScript mangles
orig_pars = cv.make_pars(set_prognoses=True, prog_by_age=False, use_layers=False)
defaults = get_defaults(merge=True)
web_pars = {}
web_pars['verbose'] = verbose # Control verbosity here
for key,entry in {**sim_pars, **epi_pars}.items():
print(key, entry)
best = defaults[key]['best']
minval = defaults[key]['min']
maxval = defaults[key]['max']
try:
web_pars[key] = np.clip(float(entry['best']), minval, maxval)
except Exception:
user_key = entry['name']
user_val = entry['best']
err1 = f'Could not convert parameter "{user_key}", value "{user_val}"; using default value instead\n'
print(err1)
err += err1
web_pars[key] = best
if die: raise
if key in sim_pars: sim_pars[key]['best'] = web_pars[key]
else: epi_pars[key]['best'] = web_pars[key]
# Convert durations
web_pars['dur'] = sc.dcp(orig_pars['dur']) # This is complicated, so just copy it
web_pars['dur']['exp2inf']['par1'] = web_pars.pop('web_exp2inf')
web_pars['dur']['inf2sym']['par1'] = web_pars.pop('web_inf2sym')
web_pars['dur']['crit2die']['par1'] = web_pars.pop('web_timetodie')
web_dur = web_pars.pop('web_dur')
for key in ['asym2rec', 'mild2rec', 'sev2rec', 'crit2rec']:
web_pars['dur'][key]['par1'] = web_dur
# Add n_days
web_pars['n_days'] = n_days
# Add the intervention
web_pars['interventions'] = []
switcher = {
'social_distance': map_social_distance,
'school_closures': map_school_closures,
'symptomatic_testing': map_symptomatic_testing,
'contact_tracing': map_contact_tracing
}
if intervention_pars is not None:
for key,scenario in intervention_pars.items():
func = switcher.get(key)
func(scenario, web_pars)
# Handle CFR -- ignore symptoms and set to 1
web_pars['prognoses'] = sc.dcp(orig_pars['prognoses'])
web_pars['rel_symp_prob'] = 1e4 # Arbitrarily large
web_pars['rel_severe_prob'] = 1e4
web_pars['rel_crit_prob'] = 1e4
web_pars['prognoses']['death_probs'][0] = web_pars.pop('web_cfr')
if web_pars['rand_seed'] == 0:
web_pars['rand_seed'] = None
web_pars['timelimit'] = max_time # Set the time limit
web_pars['pop_size'] = int(web_pars['pop_size']) # Set data type
web_pars['contacts'] = int(web_pars['contacts']) # Set data type
except Exception as E:
err2 = f'Parameter conversion failed! {str(E)}\n'
print(err2)
err += err2
if die: raise
# Create the sim and update the parameters
try:
sim = cv.Sim(pars=web_pars,datafile=datafile)
except Exception as E:
err3 = f'Sim creation failed! {str(E)}\n'
print(err3)
err += err3
if die: raise
if verbose:
print('Input parameters:')
print(web_pars)
# Core algorithm
try:
sim.run(do_plot=False)
except TimeoutError:
day = sim.t
err4 = f"The simulation stopped on day {day} because run time limit ({sim['timelimit']} seconds) was exceeded. Please reduce the population size and/or number of days simulated."
err += err4
if die: raise
except Exception as E:
err4 = f'Sim run failed! {str(E)}\n'
print(err4)
err += err4
if die: raise
# Core plotting
graphs = []
try:
to_plot = sc.dcp(cv.default_sim_plots)
for p,title,keylabels in to_plot.enumitems():
fig = go.Figure()
for key in keylabels:
label = sim.results[key].name
this_color = sim.results[key].color
y = sim.results[key][:]
fig.add_trace(go.Scatter(x=sim.results['t'][:], y=y, mode='lines', name=label, line_color=this_color))
if sim.data is not None and key in sim.data:
data_t = (sim.data.index-sim['start_day'])/np.timedelta64(1,'D')
print(sim.data.index, sim['start_day'], np.timedelta64(1,'D'), data_t)
ydata = sim.data[key]
fig.add_trace(go.Scatter(x=data_t, y=ydata, mode='markers', name=label + ' (data)', line_color=this_color))
if sim['interventions']:
interv_day = sim['interventions'][0].days[0]
if interv_day > 0 and interv_day < sim['n_days']:
fig.add_shape(dict(type="line", xref="x", yref="paper", x0=interv_day, x1=interv_day, y0=0, y1=1, name='Intervention', line=dict(width=0.5, dash='dash')))
fig.update_layout(annotations=[dict(x=interv_day, y=1.07, xref="x", yref="paper", text="Intervention start", showarrow=False)])
fig.update_layout(title={'text':title}, xaxis_title='Day', yaxis_title='Count', autosize=True)
output = {'json': fig.to_json(), 'id': str(sc.uuid())}
d = json.loads(output['json'])
d['config'] = {'responsive': True}
output['json'] = json.dumps(d)
graphs.append(output)
graphs.append(plot_people(sim))
if show_animation:
graphs.append(animate_people(sim))
except Exception as E:
err5 = f'Plotting failed! {str(E)}\n'
print(err5)
err += err5
if die: raise
# Create and send output files (base64 encoded content)
files = {}
summary = {}
try:
datestamp = sc.getdate(dateformat='%Y-%b-%d_%H.%M.%S')
ss = sim.to_excel()
files['xlsx'] = {
'filename': f'covasim_results_{datestamp}.xlsx',
'content': 'data:application/vnd.openxmlformats-officedocument.spreadsheetml.sheet;base64,' + base64.b64encode(ss.blob).decode("utf-8"),
}
json_string = sim.to_json(verbose=False)
files['json'] = {
'filename': f'covasim_results_{datestamp}.json',
'content': 'data:application/text;base64,' + base64.b64encode(json_string.encode()).decode("utf-8"),
}
# Summary output
summary = {
'days': sim.npts-1,
'cases': round(sim.results['cum_infections'][-1]),
'deaths': round(sim.results['cum_deaths'][-1]),
}
except Exception as E:
err6 = f'File saving failed! {str(E)}\n'
print(err6)
err += err6
if die: raise
output = {}
output['err'] = err
output['sim_pars'] = sim_pars
output['epi_pars'] = epi_pars
output['graphs'] = graphs
output['files'] = files
output['summary'] = summary
return output
def run_sim(sim_pars=None, epi_pars=None, show_animation=False, verbose=True):
''' Create, run, and plot everything '''
err = ''
try:
# Fix up things that JavaScript mangles
orig_pars = cv.make_pars()
defaults = get_defaults(merge=True)
web_pars = {}
web_pars['verbose'] = verbose # Control verbosity here
for key, entry in {**sim_pars, **epi_pars}.items():
print(key, entry)
best = defaults[key]['best']
minval = defaults[key]['min']
maxval = defaults[key]['max']
try:
web_pars[key] = np.clip(float(entry['best']), minval, maxval)
except Exception:
user_key = entry['name']
user_val = entry['best']
err1 = f'Could not convert parameter "{user_key}", value "{user_val}"; using default value instead\n'
print(err1)
err += err1
web_pars[key] = best
if key in sim_pars: sim_pars[key]['best'] = web_pars[key]
else: epi_pars[key]['best'] = web_pars[key]
# Convert durations
web_pars['dur'] = sc.dcp(
orig_pars['dur']) # This is complicated, so just copy it
web_pars['dur']['exp2inf']['par1'] = web_pars.pop('web_exp2inf')
web_pars['dur']['inf2sym']['par1'] = web_pars.pop('web_inf2sym')
web_pars['dur']['crit2die']['par1'] = web_pars.pop('web_timetodie')
web_dur = web_pars.pop('web_dur')
for key in ['asym2rec', 'mild2rec', 'sev2rec', 'crit2rec']:
web_pars['dur'][key]['par1'] = web_dur
# Add the intervention
web_pars['interventions'] = []
if web_pars['web_int_day'] is not None:
web_pars['interventions'] = cv.change_beta(
days=web_pars.pop('web_int_day'),
changes=(1 - web_pars.pop('web_int_eff')))
# Handle CFR -- ignore symptoms and set to 1
prog_pars = cv.get_default_prognoses(by_age=False)
web_pars['rel_symp_prob'] = 1.0 / prog_pars.symp_prob
web_pars['rel_severe_prob'] = 1.0 / prog_pars.severe_prob
web_pars['rel_crit_prob'] = 1.0 / prog_pars.crit_prob
web_pars['rel_death_prob'] = web_pars.pop(
'web_cfr') / prog_pars.death_prob
except Exception as E:
err2 = f'Parameter conversion failed! {str(E)}\n'
print(err2)
err += err2
# Create the sim and update the parameters
try:
sim = cv.Sim()
sim['prog_by_age'] = False # So the user can override this value
sim['timelimit'] = max_time # Set the time limit
if web_pars['seed'] == 0:
web_pars['seed'] = None # Reset
sim.update_pars(web_pars)
except Exception as E:
err3 = f'Sim creation failed! {str(E)}\n'
print(err3)
err += err3
if verbose:
print('Input parameters:')
print(web_pars)
# Core algorithm
try:
sim.run(do_plot=False)
except Exception as E:
err4 = f'Sim run failed! {str(E)}\n'
print(err4)
err += err4
if sim.stopped:
try: # Assume it stopped because of the time, but if not, don't worry
day = sim.stopped['t']
time_exceeded = f"The simulation stopped on day {day} because run time limit ({sim['timelimit']} seconds) was exceeded. Please reduce the population size and/or number of days simulated."
err += time_exceeded
except:
pass
# Core plotting
graphs = []
try:
to_plot = sc.dcp(cv.default_sim_plots)
for p, title, keylabels in to_plot.enumitems():
fig = go.Figure()
for key in keylabels:
label = sim.results[key].name
this_color = sim.results[key].color
y = sim.results[key][:]
fig.add_trace(
go.Scatter(x=sim.results['t'][:],
y=y,
mode='lines',
name=label,
line_color=this_color))
if sim['interventions']:
interv_day = sim['interventions'][0].days[0]
if interv_day > 0 and interv_day < sim['n_days']:
fig.add_shape(
dict(type="line",
xref="x",
yref="paper",
x0=interv_day,
x1=interv_day,
y0=0,
y1=1,
name='Intervention',
line=dict(width=0.5, dash='dash')))
fig.update_layout(annotations=[
dict(x=interv_day,
y=1.07,
xref="x",
yref="paper",
text="Intervention start",
showarrow=False)
])
fig.update_layout(title={'text': title},
xaxis_title='Day',
yaxis_title='Count',
autosize=True)
output = {'json': fig.to_json(), 'id': str(sc.uuid())}
d = json.loads(output['json'])
d['config'] = {'responsive': True}
output['json'] = json.dumps(d)
graphs.append(output)
graphs.append(plot_people(sim))
if show_animation:
graphs.append(animate_people(sim))
except Exception as E:
err5 = f'Plotting failed! {str(E)}\n'
print(err5)
err += err5
# Create and send output files (base64 encoded content)
files = {}
summary = {}
try:
datestamp = sc.getdate(dateformat='%Y-%b-%d_%H.%M.%S')
ss = sim.to_xlsx()
files['xlsx'] = {
'filename':
f'COVASim_results_{datestamp}.xlsx',
'content':
'data:application/vnd.openxmlformats-officedocument.spreadsheetml.sheet;base64,'
+ base64.b64encode(ss.blob).decode("utf-8"),
}
json_string = sim.to_json()
files['json'] = {
'filename':
f'COVASim_results_{datestamp}.txt',
'content':
'data:application/text;base64,' +
base64.b64encode(json_string.encode()).decode("utf-8"),
}
# Summary output
summary = {
'days': sim.npts - 1,
'cases': round(sim.results['cum_infections'][-1]),
'deaths': round(sim.results['cum_deaths'][-1]),
}
except Exception as E:
err6 = f'File saving failed! {str(E)}\n'
print(err6)
err += err6
output = {}
output['err'] = err
output['sim_pars'] = sim_pars
output['epi_pars'] = epi_pars
output['graphs'] = graphs
output['files'] = files
output['summary'] = summary
return output
def savefig(filename=None, comments=None, fig=None, **kwargs):
'''
Wrapper for Matplotlib's ``pl.savefig()`` function which automatically stores
Covasim metadata in the figure.
By default, saves (git) information from both the Covasim version and the calling
function. Additional comments can be added to the saved file as well. These can
be retrieved via ``cv.get_png_metadata()`` (or ``sc.loadmetadata``). Metadata can
also be stored for PDF, but cannot be automatically retrieved.
Args:
filename (str/list): name of the file to save to (default, timestamp); can also be a list of names
comments (str/dict): additional metadata to save to the figure
fig (fig/list): figure to save (by default, current one); can also be a list of figures
kwargs (dict): passed to ``fig.savefig()``
**Example**::
cv.Sim().run().plot()
cv.savefig()
'''
# Handle inputs
dpi = kwargs.pop('dpi', 150)
metadata = kwargs.pop('metadata', {})
if fig is None:
fig = pl.gcf()
figlist = sc.tolist(fig)
if filename is None: # pragma: no cover
now = sc.getdate(dateformat='%Y-%b-%d_%H.%M.%S')
filename = f'covasim_{now}.png'
filenamelist = sc.tolist(filename)
if len(figlist) != len(filenamelist):
errormsg = f'You have supplied {len(figlist)} figures and {len(filenamelist)} filenames: these must be the same length'
raise ValueError(errormsg)
metadata = {}
metadata['Covasim version'] = cvv.__version__
gitinfo = git_info()
for key,value in gitinfo['covasim'].items():
metadata[f'Covasim {key}'] = value
for key,value in gitinfo['called_by'].items():
metadata[f'Covasim caller {key}'] = value
metadata['Covasim current time'] = sc.getdate()
metadata['Covasim calling file'] = sc.getcaller()
if comments:
metadata['Covasim comments'] = comments
# Loop over the figures (usually just one)
for thisfig, thisfilename in zip(figlist, filenamelist):
# Handle different formats
lcfn = thisfilename.lower() # Lowercase filename
if lcfn.endswith('pdf') or lcfn.endswith('svg'):
metadata = {'Keywords':str(metadata)} # PDF and SVG doesn't support storing a dict
# Save the figure
thisfig.savefig(thisfilename, dpi=dpi, metadata=metadata, **kwargs)
return filename
