def test_all_tutorials(): # Get and run tests filenames = sc.getfilelist(tex.examples_dir, pattern='t*.py', nopath=True) for filename in filenames: if filename[ 1] in '0123456789': # Should have format e.g. t5_foo.py, not test_foo.py sc.heading(f'Running {filename}...') try: tex.run_example(filename) except ( pickle.PicklingError, NameError ): # Ignore these: issue with how the modules are loaded in the run_example function pass else: print(f'[Skipping "{filename}" since does not match pattern]') # Tidy up testfiles = sc.getfilelist(tex.examples_dir, pattern='my-*.*') sc.heading('Tidying...') print(f'Deleting:') for filename in testfiles: print(f' {filename}') print('in 3 seconds...') sc.timedsleep(3) for filename in testfiles: os.remove(filename) print(f' Deleted {filename}') return
def get_version_pars(version, verbose=True): ''' Function for loading parameters from the specified version. Args: version (str): the version to load parameters from Returns: Dictionary of parameters from that version ''' regression_folder = sc.thisdir(__file__, 'regression') pattern = 'pars_v*.json' requested = pattern.replace('*', version) filepaths = sc.getfilelist(regression_folder, pattern=pattern) files = [os.path.basename(f) for f in filepaths] if requested in files: # If there's an exact match match = requested else: # No match, find the nearest matching file withmatch = files + [requested] # Add this version withmatch.sort() # Sort the files index = withmatch.index(requested) if index > 0: match = withmatch[ index - 1] # Get latest earlier version -- note that this assumes versions are in alphabetical order, which they currently are! else: filestr = '\n'.join(files) errormsg = f'Could not find version {version} among options:\n{filestr}' raise ValueError(errormsg) # Load the parameters pars = sc.loadjson(filename=match, folder=regression_folder) if verbose: if match == requested: print(f'Loaded parameters from {match}') else: print( f'No exact match for parameters "{version}" found; using "{match}" instead' ) return pars
obj = sc.loadobj(files.binary) print(obj) if check('savetext', ['loadtext', 'savezip']): sc.savetext(files.text, testdata) if check('loadtext'): obj = sc.loadtext(files.text) print(obj) if check('getfilelist'): print('Files in current folder:') sc.pp(sc.getfilelist()) if check('savezip'): sc.savezip(files.zip, [files.text, files.excel]) if check('loadfailed'): ''' Check that loading an object with a non-existent class works. The file deadclass.obj was created with: deadclass.py: ------------------------------------------------- class DeadClass(): def __init__(self, x):
''' Converts Excel files to binary for faster loading. ''' import os import numpy as np import pandas as pd import sciris as sc folder = os.path.join(sc.thisdir(__file__), os.pardir, 'data', 'demographics', 'contact_matrices_152_countries') ext1 = '_1.xlsx' ext2 = '_2.xlsx' files1 = sc.getfilelist(folder, pattern=f'*{ext1}') files2 = sc.getfilelist(folder, pattern=f'*{ext2}') basefilenames = [f[len(folder) + 1:-len(ext1)] for f in files1] # 7 is the length of the extension datadict = {} for fn in basefilenames: datadict[fn] = {} for fn in basefilenames: for i, ext in enumerate([ext1, ext2]): thisfile = folder + os.sep + fn + ext print(f'Working on {thisfile}...') xls = pd.ExcelFile(thisfile) for sheet_name in xls.sheet_names: if i == 0: header = 0
#!/usr/bin/env pytest # Run all dev tests. Note: these are not guaranteed to work. import sciris as sc import runpy import pytest import pylab as pl pl.switch_backend( 'agg' ) # To avoid graphs from appearing -- if you want them, run the scripts directly scripts = sc.getfilelist(ext='py') script_str = "\n".join(scripts) print(f'Running tests on:\n{script_str}...') @pytest.mark.parametrize('script', scripts) def test_script_execution(script): runpy.run_path(script)
# Simple script to load in population files and print some useful information about schools import sciris as sc files = sc.getfilelist('v20201016_225k/inputs') n_files = len(files) pops = [] for fn in files: print(f'Working on {fn}...') pops.append(sc.loadobj(fn)) #%% school_ids = [] school_ids_type = [] for p, pop in enumerate(pops): print( f'Pop {p} has {sum(pop.student_flag)} students, {sum(pop.teacher_flag)} teachers, and {sum(pop.staff_flag)} staff' ) students = {} teachers = {} staff = {} for st, sids in pop.school_types.items(): if st in ['pk', 'uv']: continue students[st] = 0 teachers[st] = 0 staff[st] = 0 for sid in sids:
def test_legacy(): ''' Preserved for completeness, but too fragile to be used in automated unit testing due to reliance on openpyxl (which is not a required Sciris dependency). ''' # Define filenames filedir = 'files' + os.sep files = sc.prettyobj() files.excel = filedir + 'test.xlsx' files.binary = filedir + 'test.obj' files.text = filedir + 'text.txt' files.zip = filedir + 'test.zip' tidyup = True # Define the test data nrows = 15 ncols = 3 testdata = pl.zeros((nrows + 1, ncols), dtype=object) # Includes header row testdata[0, :] = ['A', 'B', 'C'] # Create header testdata[1:, :] = pl.rand(nrows, ncols) # Create data # Test spreadsheet writing, and create the file for later formats = { 'header': { 'bold': True, 'bg_color': '#3c7d3e', 'color': '#ffffff' }, 'plain': {}, 'big': { 'bg_color': '#ffcccc' } } formatdata = pl.zeros( (nrows + 1, ncols), dtype=object) # Format data needs to be the same size formatdata[1:, :] = 'plain' # Format data formatdata[1:, :][testdata[ 1:, :] > 0.7] = 'big' # Find "big" numbers and format them differently formatdata[0, :] = 'header' # Format header sc.savespreadsheet(filename=files.excel, data=testdata, formats=formats, formatdata=formatdata) # Test loading sc.heading('Loading spreadsheet') data = sc.loadspreadsheet(files.excel) print(data) excel_path = filedir + 'exampledata.xlsx' if os.path.exists(excel_path): sc.heading('Reading cells') wb = sc.Spreadsheet( filename=excel_path ) # Load a sample databook to try pulling cells from celltest = wb.readcells(method='xlrd', sheetname='Baseline year population inputs', cells=[[46, 2], [47, 2]]) # Grab cells using xlrd celltest2 = wb.readcells( method='openpyexcel', wbargs={'data_only': True}, sheetname='Baseline year population inputs', cells=[[46, 2], [47, 2]] ) # Grab cells using openpyexcel. You have to set wbargs={'data_only': True} to pull out cached values instead of formula strings print('xlrd output: %s' % celltest) print('openpyxl output: %s' % celltest2) else: print(f'{excel_path} not found, skipping...') sc.heading('Loading a blobject') blob = sc.Blobject(files.excel) f = blob.tofile() wb = openpyexcel.load_workbook(f) ws = wb.active ws['B7'] = 'Hi! ' wb.save(f) blob.load(f) blob.tofile(output=False) data = sc.loadspreadsheet(fileobj=blob.bytes) print(blob) sc.pp(data) # Test spreadsheet saving sc.heading('Using a Spreadsheet') S = sc.Spreadsheet(files.excel) S.writecells(cells=['A6', 'B7', 'C8', 'D9'], vals=['This', 'is', 'a', 'test']) # Method 1 S.writecells(cells=[pl.array([7, 1]) + i for i in range(4)], vals=['And', 'so', 'is', 'this']) # Method 2 newdata = (pl.rand(3, 3) * 100).round() S.writecells(startrow=14, startcol=1, vals=newdata, verbose=True) # Method 3 S.save() data = S.readcells(header=False) print(S) sc.pp(data) sc.heading('Saveobj/loadobj') sc.saveobj(files.binary, testdata) obj = sc.loadobj(files.binary) print(obj) sc.heading('Savetext/loadtext') sc.savetext(files.text, testdata) obj = sc.loadtext(files.text) print(obj) sc.heading('Get files') print('Files in current folder:') sc.pp(sc.getfilelist()) sc.heading('Save zip') sc.savezip(files.zip, [files.text, files.excel]) ''' Check that loading an object with a non-existent class works. The file deadclass.obj was created with: deadclass.py: ------------------------------------------------- class DeadClass(): def __init__(self, x): self.x = x ------------------------------------------------- then: ------------------------------------------------- import deadclass as dc import sciris as sc deadclass = dc.DeadClass(238473) sc.saveobj('deadclass.obj', deadclass) ------------------------------------------------- ''' dead_path = filedir + 'deadclass.obj' if os.path.exists(dead_path): sc.heading('Intentionally loading corrupted file') obj = sc.loadobj(dead_path) print('Loading corrupted object succeeded, x=%s' % obj.x) else: print(f'{dead_path} not found, skipping...') # Tidy up if tidyup: sc.blank() sc.heading('Tidying up') for fn in [files.excel, files.binary, files.text, files.zip]: try: os.remove(fn) print('Removed %s' % fn) except: pass print('Done, all fileio tests succeeded') return S
#%% Plotting if __name__ == '__main__': # Plotting/run options -- note, some plots take considerable time even with small databases save_json = 1 plot_trend = 0 plot_best = 0 plot_stride = 0 plot_all = 0 best_thresh = 1.5 # Goodness-of-fit threshold; only used for plotting die = False # Whether to raise an exception or keep going if an error is encountered T = sc.tic() dbnames = sc.getfilelist(pattern='*.db', aspath=True) # Find all Optuna databases # Define parameters for dbname in dbnames: dbname = str(dbname.stem) # Remove extension and convert to string sg = 1 if 'sg1' in dbname else 0 # Determine whether or not SafeGraph was used from the filename storage = f'sqlite:///{dbname}.db' name = 'covasim_ttq' sc.heading(f'Processing {dbname}') #%% Load and analyze the data try: print('Loading data...') assert os.path.exists( f'{dbname}.db' ), 'File does not exist' # Otherwise it creates it