def test_all_tutorials():
# Get and run tests
filenames = sc.getfilelist(tex.examples_dir, pattern='t*.py', nopath=True)
for filename in filenames:
if filename[
1] in '0123456789': # Should have format e.g. t5_foo.py, not test_foo.py
sc.heading(f'Running {filename}...')
try:
tex.run_example(filename)
except (
pickle.PicklingError, NameError
): # Ignore these: issue with how the modules are loaded in the run_example function
pass
else:
print(f'[Skipping "{filename}" since does not match pattern]')
# Tidy up
testfiles = sc.getfilelist(tex.examples_dir, pattern='my-*.*')
sc.heading('Tidying...')
print(f'Deleting:')
for filename in testfiles:
print(f' {filename}')
print('in 3 seconds...')
sc.timedsleep(3)
for filename in testfiles:
os.remove(filename)
print(f' Deleted {filename}')
return
def get_version_pars(version, verbose=True):
'''
Function for loading parameters from the specified version.
Args:
version (str): the version to load parameters from
Returns:
Dictionary of parameters from that version
'''
regression_folder = sc.thisdir(__file__, 'regression')
pattern = 'pars_v*.json'
requested = pattern.replace('*', version)
filepaths = sc.getfilelist(regression_folder, pattern=pattern)
files = [os.path.basename(f) for f in filepaths]
if requested in files: # If there's an exact match
match = requested
else: # No match, find the nearest matching file
withmatch = files + [requested] # Add this version
withmatch.sort() # Sort the files
index = withmatch.index(requested)
if index > 0:
match = withmatch[
index -
1] # Get latest earlier version -- note that this assumes versions are in alphabetical order, which they currently are!
else:
filestr = '\n'.join(files)
errormsg = f'Could not find version {version} among options:\n{filestr}'
raise ValueError(errormsg)
# Load the parameters
pars = sc.loadjson(filename=match, folder=regression_folder)
if verbose:
if match == requested:
print(f'Loaded parameters from {match}')
else:
print(
f'No exact match for parameters "{version}" found; using "{match}" instead'
)
return pars
obj = sc.loadobj(files.binary)
print(obj)
if check('savetext', ['loadtext', 'savezip']):
sc.savetext(files.text, testdata)
if check('loadtext'):
obj = sc.loadtext(files.text)
print(obj)
if check('getfilelist'):
print('Files in current folder:')
sc.pp(sc.getfilelist())
if check('savezip'):
sc.savezip(files.zip, [files.text, files.excel])
if check('loadfailed'):
'''
Check that loading an object with a non-existent class works. The file
deadclass.obj was created with:
deadclass.py:
-------------------------------------------------
class DeadClass():
def __init__(self, x):
'''
Converts Excel files to binary for faster loading.
'''
import os
import numpy as np
import pandas as pd
import sciris as sc
folder = os.path.join(sc.thisdir(__file__), os.pardir, 'data', 'demographics',
'contact_matrices_152_countries')
ext1 = '_1.xlsx'
ext2 = '_2.xlsx'
files1 = sc.getfilelist(folder, pattern=f'*{ext1}')
files2 = sc.getfilelist(folder, pattern=f'*{ext2}')
basefilenames = [f[len(folder) + 1:-len(ext1)]
for f in files1] # 7 is the length of the extension
datadict = {}
for fn in basefilenames:
datadict[fn] = {}
for fn in basefilenames:
for i, ext in enumerate([ext1, ext2]):
thisfile = folder + os.sep + fn + ext
print(f'Working on {thisfile}...')
xls = pd.ExcelFile(thisfile)
for sheet_name in xls.sheet_names:
if i == 0:
header = 0
#!/usr/bin/env pytest
# Run all dev tests. Note: these are not guaranteed to work.
import sciris as sc
import runpy
import pytest
import pylab as pl
pl.switch_backend(
'agg'
) # To avoid graphs from appearing -- if you want them, run the scripts directly
scripts = sc.getfilelist(ext='py')
script_str = "\n".join(scripts)
print(f'Running tests on:\n{script_str}...')
@pytest.mark.parametrize('script', scripts)
def test_script_execution(script):
runpy.run_path(script)
# Simple script to load in population files and print some useful information about schools
import sciris as sc
files = sc.getfilelist('v20201016_225k/inputs')
n_files = len(files)
pops = []
for fn in files:
print(f'Working on {fn}...')
pops.append(sc.loadobj(fn))
#%%
school_ids = []
school_ids_type = []
for p, pop in enumerate(pops):
print(
f'Pop {p} has {sum(pop.student_flag)} students, {sum(pop.teacher_flag)} teachers, and {sum(pop.staff_flag)} staff'
)
students = {}
teachers = {}
staff = {}
for st, sids in pop.school_types.items():
if st in ['pk', 'uv']:
continue
students[st] = 0
teachers[st] = 0
staff[st] = 0
for sid in sids:
def test_legacy():
'''
Preserved for completeness, but too fragile to be used in automated unit testing
due to reliance on openpyxl (which is not a required Sciris dependency).
'''
# Define filenames
filedir = 'files' + os.sep
files = sc.prettyobj()
files.excel = filedir + 'test.xlsx'
files.binary = filedir + 'test.obj'
files.text = filedir + 'text.txt'
files.zip = filedir + 'test.zip'
tidyup = True
# Define the test data
nrows = 15
ncols = 3
testdata = pl.zeros((nrows + 1, ncols),
dtype=object) # Includes header row
testdata[0, :] = ['A', 'B', 'C'] # Create header
testdata[1:, :] = pl.rand(nrows, ncols) # Create data
# Test spreadsheet writing, and create the file for later
formats = {
'header': {
'bold': True,
'bg_color': '#3c7d3e',
'color': '#ffffff'
},
'plain': {},
'big': {
'bg_color': '#ffcccc'
}
}
formatdata = pl.zeros(
(nrows + 1, ncols),
dtype=object) # Format data needs to be the same size
formatdata[1:, :] = 'plain' # Format data
formatdata[1:, :][testdata[
1:, :] > 0.7] = 'big' # Find "big" numbers and format them differently
formatdata[0, :] = 'header' # Format header
sc.savespreadsheet(filename=files.excel,
data=testdata,
formats=formats,
formatdata=formatdata)
# Test loading
sc.heading('Loading spreadsheet')
data = sc.loadspreadsheet(files.excel)
print(data)
excel_path = filedir + 'exampledata.xlsx'
if os.path.exists(excel_path):
sc.heading('Reading cells')
wb = sc.Spreadsheet(
filename=excel_path
) # Load a sample databook to try pulling cells from
celltest = wb.readcells(method='xlrd',
sheetname='Baseline year population inputs',
cells=[[46, 2], [47,
2]]) # Grab cells using xlrd
celltest2 = wb.readcells(
method='openpyexcel',
wbargs={'data_only': True},
sheetname='Baseline year population inputs',
cells=[[46, 2], [47, 2]]
) # Grab cells using openpyexcel. You have to set wbargs={'data_only': True} to pull out cached values instead of formula strings
print('xlrd output: %s' % celltest)
print('openpyxl output: %s' % celltest2)
else:
print(f'{excel_path} not found, skipping...')
sc.heading('Loading a blobject')
blob = sc.Blobject(files.excel)
f = blob.tofile()
wb = openpyexcel.load_workbook(f)
ws = wb.active
ws['B7'] = 'Hi! '
wb.save(f)
blob.load(f)
blob.tofile(output=False)
data = sc.loadspreadsheet(fileobj=blob.bytes)
print(blob)
sc.pp(data)
# Test spreadsheet saving
sc.heading('Using a Spreadsheet')
S = sc.Spreadsheet(files.excel)
S.writecells(cells=['A6', 'B7', 'C8', 'D9'],
vals=['This', 'is', 'a', 'test']) # Method 1
S.writecells(cells=[pl.array([7, 1]) + i for i in range(4)],
vals=['And', 'so', 'is', 'this']) # Method 2
newdata = (pl.rand(3, 3) * 100).round()
S.writecells(startrow=14, startcol=1, vals=newdata,
verbose=True) # Method 3
S.save()
data = S.readcells(header=False)
print(S)
sc.pp(data)
sc.heading('Saveobj/loadobj')
sc.saveobj(files.binary, testdata)
obj = sc.loadobj(files.binary)
print(obj)
sc.heading('Savetext/loadtext')
sc.savetext(files.text, testdata)
obj = sc.loadtext(files.text)
print(obj)
sc.heading('Get files')
print('Files in current folder:')
sc.pp(sc.getfilelist())
sc.heading('Save zip')
sc.savezip(files.zip, [files.text, files.excel])
'''
Check that loading an object with a non-existent class works. The file
deadclass.obj was created with:
deadclass.py:
-------------------------------------------------
class DeadClass():
def __init__(self, x):
self.x = x
-------------------------------------------------
then:
-------------------------------------------------
import deadclass as dc
import sciris as sc
deadclass = dc.DeadClass(238473)
sc.saveobj('deadclass.obj', deadclass)
-------------------------------------------------
'''
dead_path = filedir + 'deadclass.obj'
if os.path.exists(dead_path):
sc.heading('Intentionally loading corrupted file')
obj = sc.loadobj(dead_path)
print('Loading corrupted object succeeded, x=%s' % obj.x)
else:
print(f'{dead_path} not found, skipping...')
# Tidy up
if tidyup:
sc.blank()
sc.heading('Tidying up')
for fn in [files.excel, files.binary, files.text, files.zip]:
try:
os.remove(fn)
print('Removed %s' % fn)
except:
pass
print('Done, all fileio tests succeeded')
return S
#%% Plotting
if __name__ == '__main__':
# Plotting/run options -- note, some plots take considerable time even with small databases
save_json = 1
plot_trend = 0
plot_best = 0
plot_stride = 0
plot_all = 0
best_thresh = 1.5 # Goodness-of-fit threshold; only used for plotting
die = False # Whether to raise an exception or keep going if an error is encountered
T = sc.tic()
dbnames = sc.getfilelist(pattern='*.db',
aspath=True) # Find all Optuna databases
# Define parameters
for dbname in dbnames:
dbname = str(dbname.stem) # Remove extension and convert to string
sg = 1 if 'sg1' in dbname else 0 # Determine whether or not SafeGraph was used from the filename
storage = f'sqlite:///{dbname}.db'
name = 'covasim_ttq'
sc.heading(f'Processing {dbname}')
#%% Load and analyze the data
try:
print('Loading data...')
assert os.path.exists(
f'{dbname}.db'
), 'File does not exist' # Otherwise it creates it
