def try_loading_cached_sim(sim, cachefile, statusfile, do_load):
''' Load the sim from file, or at least try '''
# Assign these to variables to prevent accidental changes!
success = 'success'
failed = 'failed'
if os.path.isfile(statusfile):
status = sc.loadtext(statusfile)
else:
status = 'not yet run'
sim_loaded = False
if do_load and os.path.isfile(cachefile) and failed not in status:
try:
sim_orig = sim.copy()
sim = cv.Sim.load(cachefile)
sim.meta = sim_orig.meta
sim['interventions'] = sim_orig['interventions']
for interv in sim['interventions']:
interv.initialize(sim)
if status != success: # Update the status to note success
sc.savetext(statusfile, success)
sim_loaded = True
except Exception as E:
errormsg = f'WARNING, {failed} to load cached sim from {cachefile}! Reason: {str(E)}'
print(errormsg)
sc.savetext(statusfile, errormsg)
if sim_loaded: loadstr = 'loaded from cache :)'
elif not do_load: loadstr = 'loading from cache disabled'
else: loadstr = 'could not load from cache'
return sim, sim_loaded, loadstr
def run_sim(index,
scenpars=None,
label=None,
runinfo=None,
do_shrink=True,
from_cache=True,
verbose=True):
''' Load, modify, and run the simulation '''
if scenpars is None:
print('WARNING, loading default parameters!')
scenpars = sc.dcp(d_pars)
if label is None:
label = f'full_sim_trial{index}_nolabel'
# Run sim or load up until scenarios start
sim_loaded = 0
filename = f'./simcache/projection_trial{index}.sim'
if from_cache and os.path.exists(filename):
tries = 0
while not sim_loaded and tries < 3:
try:
print(f'run_sim(): Loading sim from cache ({filename})...')
sim = cv.load(filename)
assert isinstance(
sim, cv.Sim
) # Sometimes unpickling fails, but can reload locally
tn = sim.get_interventions('tn')
tn.subtarget = test_num_subtarg
sim_loaded = 1
except Exception as E:
print(f'Loading failed on try {tries}! {E}')
string = '\n\n'.join([
f'Index {index}', filename, f'Try {tries}',
str(E),
sc.traceback()
])
fn = f'simcache_exception_index{index}_try{tries}.err'
sc.savetext(fn, string)
tries += 1
sc.timedsleep(0.5)
if not sim_loaded:
print(f'run_sim(): Creating new sim (and saving to {filename})...')
sim = create_sim(index, verbose=verbose)
sim.run(until=sim.sceninfo.calib_end)
sim.save(filename, keep_people=True)
# Modify the sim with these scenario values
sim = modify_sim(sim, scenpars=scenpars, label=label, runinfo=runinfo)
# Rerun till the end and optionally shrink
sim.run(restore_pars=False) # So we can see the changes made
if do_shrink:
sim.shrink()
return sim
sc.toc(T)
def save_cached_sim(sim, cachefile, statusfile, do_save):
''' Save the sim to disk '''
if do_save:
print(f'Saving cache file to {cachefile}')
sim.save(cachefile, keep_people=True)
sc.savetext(statusfile, 'saved but not yet loaded')
return
def worker():
''' Run a single worker '''
try:
study = op.load_study(storage=storage, study_name=name)
output = study.optimize(objective, n_trials=n_trials)
except Exception as E:
print(f'An exception was encountered! {E}')
string = '\n\n'.join([storage, str(E), sc.traceback()])
fn = f'optuna_exception_{get_fn()}.err'
sc.savetext(fn, string)
output = str(E)
return output
def run_scenario(index, v=None, which=None, doshrink=True, verbose=True):
if verbose:
print(f'Now running {index}:')
cv.set_seed(index + start_seed)
if which == 'a':
if v is None:
sweeps = construct1dsweeps()
v = sweeps[index]
f = get_f_a(v)
scenpars = scenconvert_a(f)
elif which == 'b':
f = get_f_b(v, index=index)
scenpars = scenconvert_b(f)
if resample:
eind = (index + entry_offset) % n_top_entries
else:
eind = index + entry_offset
labelvals = [f'{v:0.2f}' for v in f.values()]
labelstr = f'i={index} e={eind} f=' + ','.join(labelvals)
runinfo = sc.objdict()
runinfo.f = f
runinfo.scenpars = scenpars
runinfo.ind = index
runinfo.eind = eind
sim = cs.run_sim(index=eind,
scenpars=scenpars,
label=labelstr,
runinfo=runinfo,
verbose=verbose)
# Try to save progress, but don't worry about it
try:
sc.savetext(f'progress/{index}.ind', str(f))
except Exception as E:
print(f'Could not save file {index}: {str(E)}')
return sim
data = S.readcells(header=False)
print(S)
sc.pp(data)
if check('saveobj', ['loadobj']):
sc.saveobj(files.binary, testdata)
if check('loadobj'):
obj = sc.loadobj(files.binary)
print(obj)
if check('savetext', ['loadtext', 'savezip']):
sc.savetext(files.text, testdata)
if check('loadtext'):
obj = sc.loadtext(files.text)
print(obj)
if check('getfilelist'):
print('Files in current folder:')
sc.pp(sc.getfilelist())
if check('savezip'):
sc.savezip(files.zip, [files.text, files.excel])
def test_legacy():
'''
Preserved for completeness, but too fragile to be used in automated unit testing
due to reliance on openpyxl (which is not a required Sciris dependency).
'''
# Define filenames
filedir = 'files' + os.sep
files = sc.prettyobj()
files.excel = filedir + 'test.xlsx'
files.binary = filedir + 'test.obj'
files.text = filedir + 'text.txt'
files.zip = filedir + 'test.zip'
tidyup = True
# Define the test data
nrows = 15
ncols = 3
testdata = pl.zeros((nrows + 1, ncols),
dtype=object) # Includes header row
testdata[0, :] = ['A', 'B', 'C'] # Create header
testdata[1:, :] = pl.rand(nrows, ncols) # Create data
# Test spreadsheet writing, and create the file for later
formats = {
'header': {
'bold': True,
'bg_color': '#3c7d3e',
'color': '#ffffff'
},
'plain': {},
'big': {
'bg_color': '#ffcccc'
}
}
formatdata = pl.zeros(
(nrows + 1, ncols),
dtype=object) # Format data needs to be the same size
formatdata[1:, :] = 'plain' # Format data
formatdata[1:, :][testdata[
1:, :] > 0.7] = 'big' # Find "big" numbers and format them differently
formatdata[0, :] = 'header' # Format header
sc.savespreadsheet(filename=files.excel,
data=testdata,
formats=formats,
formatdata=formatdata)
# Test loading
sc.heading('Loading spreadsheet')
data = sc.loadspreadsheet(files.excel)
print(data)
excel_path = filedir + 'exampledata.xlsx'
if os.path.exists(excel_path):
sc.heading('Reading cells')
wb = sc.Spreadsheet(
filename=excel_path
) # Load a sample databook to try pulling cells from
celltest = wb.readcells(method='xlrd',
sheetname='Baseline year population inputs',
cells=[[46, 2], [47,
2]]) # Grab cells using xlrd
celltest2 = wb.readcells(
method='openpyexcel',
wbargs={'data_only': True},
sheetname='Baseline year population inputs',
cells=[[46, 2], [47, 2]]
) # Grab cells using openpyexcel. You have to set wbargs={'data_only': True} to pull out cached values instead of formula strings
print('xlrd output: %s' % celltest)
print('openpyxl output: %s' % celltest2)
else:
print(f'{excel_path} not found, skipping...')
sc.heading('Loading a blobject')
blob = sc.Blobject(files.excel)
f = blob.tofile()
wb = openpyexcel.load_workbook(f)
ws = wb.active
ws['B7'] = 'Hi! '
wb.save(f)
blob.load(f)
blob.tofile(output=False)
data = sc.loadspreadsheet(fileobj=blob.bytes)
print(blob)
sc.pp(data)
# Test spreadsheet saving
sc.heading('Using a Spreadsheet')
S = sc.Spreadsheet(files.excel)
S.writecells(cells=['A6', 'B7', 'C8', 'D9'],
vals=['This', 'is', 'a', 'test']) # Method 1
S.writecells(cells=[pl.array([7, 1]) + i for i in range(4)],
vals=['And', 'so', 'is', 'this']) # Method 2
newdata = (pl.rand(3, 3) * 100).round()
S.writecells(startrow=14, startcol=1, vals=newdata,
verbose=True) # Method 3
S.save()
data = S.readcells(header=False)
print(S)
sc.pp(data)
sc.heading('Saveobj/loadobj')
sc.saveobj(files.binary, testdata)
obj = sc.loadobj(files.binary)
print(obj)
sc.heading('Savetext/loadtext')
sc.savetext(files.text, testdata)
obj = sc.loadtext(files.text)
print(obj)
sc.heading('Get files')
print('Files in current folder:')
sc.pp(sc.getfilelist())
sc.heading('Save zip')
sc.savezip(files.zip, [files.text, files.excel])
'''
Check that loading an object with a non-existent class works. The file
deadclass.obj was created with:
deadclass.py:
-------------------------------------------------
class DeadClass():
def __init__(self, x):
self.x = x
-------------------------------------------------
then:
-------------------------------------------------
import deadclass as dc
import sciris as sc
deadclass = dc.DeadClass(238473)
sc.saveobj('deadclass.obj', deadclass)
-------------------------------------------------
'''
dead_path = filedir + 'deadclass.obj'
if os.path.exists(dead_path):
sc.heading('Intentionally loading corrupted file')
obj = sc.loadobj(dead_path)
print('Loading corrupted object succeeded, x=%s' % obj.x)
else:
print(f'{dead_path} not found, skipping...')
# Tidy up
if tidyup:
sc.blank()
sc.heading('Tidying up')
for fn in [files.excel, files.binary, files.text, files.zip]:
try:
os.remove(fn)
print('Removed %s' % fn)
except:
pass
print('Done, all fileio tests succeeded')
return S