def roi_df(args, model_name, roi):
if args.fixed_t:
args.roi = roi # Temporary
csv = Path(args.data_path) / ("covidtimeseries_%s.csv" % args.roi)
csv = csv.resolve()
assert csv.exists(), "No such csv file: %s" % csv
stan_data, t0 = ncs.get_stan_data(csv, args)
global_start = datetime.strptime('01/22/20', '%m/%d/%y')
frame_start = datetime.strptime(t0, '%m/%d/%y')
day_offset = (frame_start - global_start).days
else:
day_offset = 0
model_path = ncs.get_model_path(args.models_path, model_name)
extension = ['csv', 'pkl'][args.fit_format]
rois = ncs.list_rois(args.fits_path, model_name, extension)
if args.rois:
rois = list(set(rois).intersection(args.rois))
fit_path = ncs.get_fit_path(args.fits_path, model_name, roi)
if args.fit_format == 1:
fit = ncs.load_fit(fit_path, model_path)
stats = ncs.get_waic_and_loo(fit)
samples = fit.to_dataframe()
elif args.fit_format == 0:
samples = ncs.extract_samples(args.fits_path, args.models_path,
model_name, roi, args.fit_format)
stats = ncs.get_waic(samples)
df = ncs.make_table(roi,
samples,
args.params,
stats,
quantiles=args.quantiles,
day_offset=day_offset)
return model_name, roi, df
def roi_df(args, model_name, roi):
model_path = ncs.get_model_path(args.models_path, model_name)
extension = ['csv', 'pkl'][args.fit_format]
rois = ncs.list_rois(args.fits_path, model_name, extension)
if args.rois:
rois = list(set(rois).intersection(args.rois))
fit_path = ncs.get_fit_path(args.fits_path, model_name, roi)
if args.fit_format == 1:
fit = ncs.load_fit(fit_path, model_path)
stats = ncs.get_waic_and_loo(fit)
samples = fit.to_dataframe()
elif args.fit_format == 0:
samples = ncs.extract_samples(args.fits_path, args.models_path,
model_name, roi, args.fit_format)
stats = ncs.get_waic(samples)
df = ncs.make_table(roi,
samples,
args.params,
stats,
quantiles=args.quantiles)
return model_name, roi, df
import numpy as np
from datetime import datetime
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
import seaborn as sns
import statsmodels.stats.proportion as smp
import pylab
from scipy.optimize import curve_fit
import niddk_covid_sicr as ncs
import requests
import bz2
import pickle
import _pickle as cPickle
fitloc = 'C:\\Users\\osman\\Documents\\fitUS06-27-20'
rois = ncs.list_rois(fitloc, 'SICRMQC2R2DX2', '.pkl')
df_roi_t0 = pd.read_csv('n_data.csv')
for current_roi in rois:
df = ncs.extract_samples(fitloc, '../models/', 'SICRMQC2R2DX2',
current_roi, 1)
df_withprojonly = df[[
'chain', 'draw', 'warmup', 'f1', 'f2', 'sigmar', 'sigmad', 'sigmar1',
'sigmad1', 'sigmau', 'mbase', 'mlocation', 'trelax', 'extra_std',
'extra_std_R', 'extra_std_D', 'q', 'cbase', 'clocation', 'n_pop',
'phi[1]', 'phi[2]', 'phi[3]', 'u_init[1]', 'u_init[2]', 'u_init[3]',
'u_init[4]', 'u_init[5]', 'u_init[6]', 'u_init[7]', 'u_init[8]',
'sigmac', 'beta', 'sigma', 'R0'
]]
df_withprojonly.to_csv(current_roi + '.csv')
if not args.max_jobs:
args.max_jobs = cpu_count()
# If no model_names are provided, use all of them
if not args.model_names:
args.model_names = ncs.list_models(args.models_path)
assert len(args.model_names),\
("No such model files matching: *.stan' at %s" % (args.models_path))
# Get all model_names, roi combinations
if not args.average_only:
combos = []
for model_name in args.model_names:
model_path = ncs.get_model_path(args.models_path, model_name)
extension = ['csv', 'pkl'][args.fit_format]
rois = ncs.list_rois(args.fits_path, model_name, extension)
if args.rois:
rois = list(set(rois).intersection(args.rois))
combos += [(model_name, roi) for roi in rois]
# Organize into (model_name, roi) tuples
combos = list(zip(*combos))
assert len(combos), "No combinations of models and ROIs found"
print("There are %d combinations of models and ROIs" % len(combos))
def roi_df(args, model_name, roi):
if args.fixed_t:
args.roi = roi # Temporary
csv = Path(args.data_path) / ("covidtimeseries_%s.csv" % args.roi)
csv = csv.resolve()
assert csv.exists(), "No such csv file: %s" % csv
if not args.totwk:
import pandas as pd
from pathlib import Path
import niddk_covid_sicr as ncs
# Parse all the command-line arguments
parser = argparse.ArgumentParser(description='')
parser.add_argument('-dp',
'--data-path',
default='./data',
help='Path to directory containing the data files')
parser.add_argument('-ld',
'--last-date',
help=('Last date to use in the data; dates past this '
'will be ignored'))
args = parser.parse_args()
# Get all model_names, roi combinations
rois = ncs.list_rois(args.data_path, 'covidtimeseries', '.csv')
df = pd.DataFrame(index=rois, columns=['n_data_pts'], dtype=int)
for roi in rois:
csv = Path(args.data_path) / ("covidtimeseries_%s.csv" % roi)
csv = csv.resolve()
assert csv.exists(), "No such csv file: %s" % csv
stan_data, t0 = ncs.get_stan_data(csv, args)
n_data = ncs.get_n_data(stan_data)
df.loc[roi, 'n_data_pts'] = int(n_data)
df.loc[roi, 't0'] = t0
df.index.name = 'roi'
df['n_data_pts'] = df['n_data_pts'].astype(int)
df['t0'] = df['t0'].astype('datetime64')
df.to_csv('n_data.csv')
parser.add_argument('-f',
'--fit-format',
type=int,
default=1,
help='Version of fit format')
parser.add_argument('-i',
'--init',
help=('Fit file to use for initial conditions '
'(uses last sample)'))
args = parser.parse_args()
if args.n_threads == 0:
args.n_threads = args.n_chains
prefix = ncs.get_data_prefix()
if not args.rois:
args.rois = ncs.list_rois(args.data_path, prefix, '.csv')
assert len(args.rois),\
("No such data files matching: %s*.csv' at %s"
% (prefix, args.data_path))
if not args.model_names:
args.model_names = ncs.list_models(args.models_path)
assert len(args.model_names),\
("No such model files matching: *.stan' at %s" % (args.models_path))
model_paths = [
Path(args.models_path) / ('%s.stan' % model_name)
for model_name in args.model_names
]
for model_path in model_paths:
assert model_path.is_file(), "No such .stan file: %s" % model_path
models_path = Path(args.models_path)
results_path = Path(args.results_path)
results_path.mkdir(parents=True, exist_ok=True)
assert any([x.name.endswith('.csv') for x in data_path.iterdir()]),\
"No .csv files found in data_path %s" % (data_path.resolve())
assert any([x.name.endswith('.stan') for x in models_path.iterdir()]),\
"No .stan files found in models_path %s" % (models_path.resolve())
assert any([x.name.endswith('.pkl') or x.name.endswith('.csv')
for x in fits_path.iterdir()]),\
"No .pkl or .csv files found in fits_path %s" % (fits_path.resolve())
ending = get_ending(args.fit_format)
if not args.rois:
data_rois = list_rois(data_path, get_data_prefix(), '.csv')
fit_rois = list_rois(fits_path, args.model_name, ending)
args.rois = list(set(data_rois).intersection(fit_rois))
args.n_threads = min(args.n_threads, len(args.rois))
print("Running visualization notebook for %d rois on model '%s'" %
(len(args.rois), args.model_name))
# Make sure all ROI pickle files exist
for roi in args.rois:
file = fits_path / ('%s_%s%s' % (args.model_name, roi, ending))
assert file.is_file(), "No such %s file: %s" % (ending, file.resolve())
# Function to be execute on each ROI
if not args.max_jobs:
args.max_jobs = cpu_count()
# If no model_names are provided, use all of them
if not args.model_names:
args.model_names = ncs.list_models(args.models_path)
assert len(args.model_names),\
("No such model files matching: *.stan' at %s" % (args.models_path))
# Get all model_names, roi combinations
if not args.average_only:
combos = []
for model_name in args.model_names:
model_path = ncs.get_model_path(args.models_path, model_name)
extension = ['csv', 'pkl'][args.fit_format]
rois = ncs.list_rois(args.fits_path, model_name, extension)
if args.rois:
rois = list(set(rois).intersection(args.rois))
combos += [(model_name, roi) for roi in rois]
# Organize into (model_name, roi) tuples
combos = list(zip(*combos))
assert len(combos), "No combinations of models and ROIs found"
print("There are %d combinations of models and ROIs" % len(combos))
def roi_df(args, model_name, roi):
if args.fixed_t:
args.roi = roi # Temporary
csv = Path(args.data_path) / ("covidtimeseries_%s.csv" % args.roi)
csv = csv.resolve()
assert csv.exists(), "No such csv file: %s" % csv
if not args.totwk:
