def plot(f, x_start, x_end, x_interval=None, epsilon=0.15):
if x_interval is None:
x_interval = x_end - x_start
x = py.arange(x_start, x_end + 0.1 * x_interval, x_interval)
# y = x^2
py.plot(x, f(x), 'ko', label='$y=x^2-10$')
# y = 0
py.plot(x, py.zeros_like(x), 'ro', label='$y=0$')
# +/- epsilon
py.plot(x, epsilon * py.ones_like(x), 'r-.', label=r'$+\epsilon$')
py.plot(x, -epsilon * py.ones_like(x), 'r--', label=r'$-\epsilon$')
# x 축 이름표
# x axis label
py.xlabel('x')
# y 축 이름표
# y axis label
py.ylabel('y')
# 범례 표시
# Show legend
py.legend()
# 모눈 표시
# Indicate grid
py.grid()
return x
def alphas_hist(df, outdir=FIGS_DIR, categories=None):
subset_cols = ['neuron name', 'neuron type']
if categories != None:
for category in categories:
if category != 'neuron type':
subset_cols.append(category)
df2 = df.drop_duplicates(subset_cols)
alphas = None
weights = None
labels = None
if categories == None:
alphas = list(df2['alpha'])
print "all neurons mean alpha", pylab.mean(alphas), "+/-", pylab.std(
alphas, ddof=1)
weights = pylab.ones_like(alphas) / len(alphas)
else:
alphas = []
weights = []
labels = []
for name, group in df2.groupby(categories):
cat_alphas = group['alpha']
print name + " neurons mean alpha", pylab.mean(cat_alphas)
cat_weights = pylab.ones_like(cat_alphas) / len(cat_alphas)
alphas.append(cat_alphas)
weights.append(cat_weights)
labels.append(name)
pylab.figure()
sns.set()
if labels == None:
pylab.hist(alphas, range=(0, 1), weights=weights)
else:
pylab.hist(alphas, range=(0, 1), weights=weights, label=labels)
pylab.legend()
curr_ax = pylab.gca()
curr_ax.set_ylim((0, 1))
pylab.xlabel('alpha', size=20)
pylab.ylabel('proportion', size=20)
pylab.xticks(fontsize=20)
pylab.yticks(fontsize=20)
pylab.tight_layout()
name = 'alphas_hist'
if categories != None:
cat_str = '_'.join(categories)
cat_str = cat_str.replace(' ', '_')
name += '_' + cat_str
outname = '%s/%s.pdf' % (outdir, name)
outname = outname.replace(' ', '_')
pylab.savefig('%s/%s.pdf' % (outdir, name), format='pdf')
pylab.close()
def panel():
global Ras, Rms, Ie, Ies
global tstart
global pan, t0, t1
#Pulse start time
tstart = 0.1
#Ras = [1.,500.,20.] #Ohm*cm
#Rms = [200.,10000.,200.] #Ohm*cm^2
Ras = [5.0, 500.0, 50.0] #Ohm*cm
Rms = [200.0, 10000.0, 1000.0] #Ohm*cm^2
Ie = 4. #nA
Ies = pl.c_[pl.ones_like(ns.t)]
Ies[:pl.ceil(tstart / ns.dt)] = 0.
t0 = 0.101
t1 = 0.6
pan = simulationpanel.SimulationPanel()
pan.Move((640, 600))
pan.setdict(globals())
pan.addcommand('sim()')
pan.addcommand('plottao()')
pan.addvar('Ras')
pan.addvar('Rms')
pan.addvar('Ie')
pan.addvar('t0')
pan.addvar('t1')
def fit_age_standardizing_model(model):
"""Age-standardizing model"""
## Spline model to represent age-specific rate
model.vars += dismod3.age_pattern.spline(
name="std", ages=model.ages, knots=knots, smoothing=pl.inf, interpolation_method="linear"
)
model.vars += dismod3.age_group.age_standardize_approx(
name="std",
ages=model.ages,
age_weights=pl.ones_like(model.ages),
mu_age=model.vars["mu_age"],
age_start=model.input_data["age_start"],
age_end=model.input_data["age_end"],
)
model.vars += {"delta": mc.Uniform("delta_std", 0.0, 1000.0, value=10.0)}
## Negative binomial rate model
model.vars += dismod3.rate_model.neg_binom(
name="std",
pi=model.vars["mu_interval"],
delta=model.vars["delta"],
p=model.input_data["value"],
n=model.input_data["effective_sample_size"],
)
fit_model(model)
def plot_funnel(pi_true, delta_str):
delta = float(delta_str)
n = pl.exp(mc.rnormal(10, 2**-2, size=10000))
p = pi_true*pl.ones_like(n)
# old way:
#delta = delta * p * n
nb = rate_model.neg_binom_model('funnel', pi_true, delta, p, n)
r = nb['p_pred'].value
pl.vlines([pi_true], .1*n.min(), 10*n.max(),
linewidth=5, linestyle='--', color='black', zorder=10)
pl.plot(r, n, 'o', color=colors[0], ms=10,
mew=0, alpha=.25)
pl.semilogy(schiz['r'], schiz['n'], 's', mew=1, mec='white', ms=15,
color=colors[1],
label='Observed Values')
pl.xlabel('Rate (Per 1000 PY)', size=32)
pl.ylabel('Study Size (PY)', size=32)
pl.axis([-.0001, .0101, 50., 15000000])
pl.title(r'$\delta = %s$'%delta_str, size=48)
pl.xticks([0, .005, .01], [0, 5, 10], size=30)
pl.yticks(size=30)
def plot_funnel(pi_true, delta_str):
delta = float(delta_str)
n = pl.exp(mc.rnormal(10, 2**-2, size=10000))
p = pi_true * pl.ones_like(n)
# old way:
#delta = delta * p * n
nb = rate_model.neg_binom_model('funnel', pi_true, delta, p, n)
r = nb['p_pred'].value
pl.vlines([pi_true],
.1 * n.min(),
10 * n.max(),
linewidth=5,
linestyle='--',
color='black',
zorder=10)
pl.plot(r, n, 'o', color=colors[0], ms=10, mew=0, alpha=.25)
pl.semilogy(schiz['r'],
schiz['n'],
's',
mew=1,
mec='white',
ms=15,
color=colors[1],
label='Observed Values')
pl.xlabel('Rate (Per 1000 PY)', size=32)
pl.ylabel('Study Size (PY)', size=32)
pl.axis([-.0001, .0101, 50., 15000000])
pl.title(r'$\delta = %s$' % delta_str, size=48)
pl.xticks([0, .005, .01], [0, 5, 10], size=30)
pl.yticks(size=30)
def fit_age_standardizing_model(model):
"""Age-standardizing model"""
## Spline model to represent age-specific rate
model.vars += dismod3.age_pattern.spline(name='std', ages=model.ages,
knots=knots,
smoothing=pl.inf,
interpolation_method='linear')
model.vars += dismod3.age_group.age_standardize_approx(name='std', ages=model.ages,
age_weights=pl.ones_like(model.ages),
mu_age=model.vars['mu_age'],
age_start=model.input_data['age_start'], age_end=model.input_data['age_end'])
delta = mc.Uniform('delta_mid', 0., 1000., value=10.)
model.vars += {'delta': delta}
## Negative binomial rate model
model.vars += dismod3.rate_model.neg_binom(name='std',
pi=model.vars['mu_interval'],
delta=model.vars['delta'],
p=model.input_data['value'],
n=model.input_data['effective_sample_size'])
fit_model(model)
def test_age_standardizing_approx():
# simulate data
n = 50
sigma_true = .025 * pl.ones(n)
a = pl.arange(0, 100, 1)
pi_age_true = .0001 * (a * (100. - a) + 100.)
ages = pl.arange(101)
d = data_simulation.simulated_age_intervals('p', n, a, pi_age_true,
sigma_true)
# create model and priors
vars = {}
vars.update(
age_pattern.age_pattern('test',
ages,
knots=pl.arange(0, 101, 5),
smoothing=.01))
vars.update(
age_integrating_model.age_standardize_approx(
'test', pl.ones_like(vars['mu_age'].value), vars['mu_age'],
d['age_start'], d['age_end'], ages))
vars['pi'] = vars['mu_interval']
vars.update(
rate_model.normal_model('test',
pi=vars['pi'],
sigma=0,
p=d['value'],
s=sigma_true))
# fit model
m = mc.MCMC(vars)
m.sample(3)
def panel():
global Ras,Rms,Ie,Ies
global tstart
global pan,t0,t1
#Pulse start time
tstart = 0.1
#Ras = [1.,500.,20.] #Ohm*cm
#Rms = [200.,10000.,200.] #Ohm*cm^2
Ras = [5.0, 500.0, 50.0] #Ohm*cm
Rms = [200.0, 10000.0, 1000.0] #Ohm*cm^2
Ie = 4. #nA
Ies = pl.c_[pl.ones_like(ns.t)]
Ies[:pl.ceil(tstart/ns.dt)] = 0.
t0 = 0.101
t1 = 0.6
pan = simulationpanel.SimulationPanel()
pan.Move((640,600))
pan.setdict(globals())
pan.addcommand('sim()')
pan.addcommand('plottao()')
pan.addvar('Ras')
pan.addvar('Rms')
pan.addvar('Ie')
pan.addvar('t0')
pan.addvar('t1')
def SWE_Cgrid(beta, g, H, dx, yu, yv, dt, nt, u, v, h, h0):
"solves the SWE using forward-backward time-stepping and the C-grid"
"doubly periodic domain stating from u, v and h"
[nx, ny] = py.shape(u)
dy = yu[1] - yu[0]
H = H * py.ones_like(h)
# loop through all times
for it in xrange(nt):
# update old values
uOld = u.copy()
vOld = v.copy()
hOld = h.copy()
# alternate between calculating u or v first
for iu in xrange(2):
if iu % 2 == 0:
# loop over x and y directions (using i and j)
u = uOld + dt * (beta * yu * vatu(v) - g * ddxC(h + h0, dx))
else:
v = vOld + dt * (-beta * yv * uatv(u) - g * ddyC(h + h0, dy))
# then calculate h using updated values of u and v
h = hOld - dt * divC(hatu(H - h0) * u, hatv(H - h0) * v, dx, dy)
return [u, v, h]
def test_covariate_model_sim_no_hierarchy():
# simulate normal data
model = data.ModelData()
model.hierarchy, model.output_template = data_simulation.small_output()
X = mc.rnormal(0., 1.**2, size=(128, 3))
beta_true = [-.1, .1, .2]
Y_true = pl.dot(X, beta_true)
pi_true = pl.exp(Y_true)
sigma_true = .01 * pl.ones_like(pi_true)
p = mc.rnormal(pi_true, 1. / sigma_true**2.)
model.input_data = pandas.DataFrame(
dict(value=p, x_0=X[:, 0], x_1=X[:, 1], x_2=X[:, 2]))
model.input_data['area'] = 'all'
model.input_data['sex'] = 'total'
model.input_data['year_start'] = 2000
model.input_data['year_end'] = 2000
# create model and priors
vars = {}
vars.update(
covariate_model.mean_covariate_model('test', 1, model.input_data, {},
model, 'all', 'total', 'all'))
vars.update(rate_model.normal_model('test', vars['pi'], 0., p, sigma_true))
# fit model
m = mc.MCMC(vars)
m.sample(2)
def test_covariate_model_sim_no_hierarchy():
# simulate normal data
model = data.ModelData()
model.hierarchy, model.output_template = data_simulation.small_output()
X = mc.rnormal(0., 1.**2, size=(128,3))
beta_true = [-.1, .1, .2]
Y_true = pl.dot(X, beta_true)
pi_true = pl.exp(Y_true)
sigma_true = .01*pl.ones_like(pi_true)
p = mc.rnormal(pi_true, 1./sigma_true**2.)
model.input_data = pandas.DataFrame(dict(value=p, x_0=X[:,0], x_1=X[:,1], x_2=X[:,2]))
model.input_data['area'] = 'all'
model.input_data['sex'] = 'total'
model.input_data['year_start'] = 2000
model.input_data['year_end'] = 2000
# create model and priors
vars = {}
vars.update(covariate_model.mean_covariate_model('test', 1, model.input_data, {}, model, 'all', 'total', 'all'))
vars.update(rate_model.normal_model('test', vars['pi'], 0., p, sigma_true))
# fit model
m = mc.MCMC(vars)
m.sample(2)
def fit_age_standardizing_model(model):
"""Age-standardizing model"""
## Spline model to represent age-specific rate
model.vars += dismod3.age_pattern.spline(name='std',
ages=model.ages,
knots=knots,
smoothing=pl.inf,
interpolation_method='linear')
model.vars += dismod3.age_group.age_standardize_approx(
name='std',
ages=model.ages,
age_weights=pl.ones_like(model.ages),
mu_age=model.vars['mu_age'],
age_start=model.input_data['age_start'],
age_end=model.input_data['age_end'])
model.vars += {'delta': mc.Uniform('delta_std', 0., 1000., value=10.)}
## Negative binomial rate model
model.vars += dismod3.rate_model.neg_binom(
name='std',
pi=model.vars['mu_interval'],
delta=model.vars['delta'],
p=model.input_data['value'],
n=model.input_data['effective_sample_size'])
fit_model(model)
def mortality(self, key):
""" Calculate the all-cause mortality rate for the
region and sex of disease_model, and return it
in an array corresponding to age_mesh
Parameters
----------
key : str, optional
of the form 'all-cause_mortality+gbd_region+year+sex'
data: list, optional
the data list to extract all-cause mortality from
"""
data = self.filter_data(key.replace('world', 'all')) # TODO: generate world mortality rate and use it when key is world, instead of using replace-world-all hack
if len(data) == 0:
return NEARLY_ZERO * pl.ones_like(self.get_estimate_age_mesh())
else:
ages = []
vals = []
for d in data:
a0 = d.get('age_start', MISSING)
y = self.value_per_1(d)
if MISSING in [a0, y]:
continue
ages.append(a0)
vals.append(y)
# add one additional point so that interpolation is all within range of data
ages.append(dismod3.settings.MAX_AGE)
vals.append(y)
m_all = dismod3.utils.interpolate(ages, vals, self.get_estimate_age_mesh())
return m_all
def alphas_hist(df):
df2 = df.drop_duplicates(subset='name')
alphas = list(df2['alpha'])
weights = pylab.ones_like(alphas) / len(alphas)
pylab.hist(alphas, range=(0, 1), weights=weights)
curr_ax = pylab.gca()
curr_ax.set_ylim((0, 1))
pylab.savefig('%s/alphas_hist.pdf' % OUTDIR, format='pdf')
def make_count_table(dm, wb):
"""Make a table representation of disease counts
Parameters
----------
dm : DiseaseJson object
wb : the xlwt Workbook to add the count table to
"""
ws = wb.add_sheet('Counts')
params = 'incidence prevalence excess-mortality incidence_x_duration'.split()
cols = 'region year sex age param mean lower upper'.split()
for col, col_head in enumerate(cols):
ws.write(0, col, col_head)
row = 1
for region in dismod3.settings.gbd_regions:
for year in dismod3.settings.gbd_years:
for sex in dismod3.settings.gbd_sexes:
for param in params:
mean_array = dm.get_mcmc('mean', dismod3.utils.gbd_key_for(param, region, year, sex))
lower_array = dm.get_mcmc('lower_ui', dismod3.utils.gbd_key_for(param, region, year, sex))
upper_array = dm.get_mcmc('upper_ui', dismod3.utils.gbd_key_for(param, region, year, sex))
if len(mean_array) == 0:
continue
pop = population_by_region_year_sex(region, year, sex)
if param == 'excess-mortality':
# use estimate of prevalent population
pop *= dm.get_mcmc('mean', dismod3.utils.gbd_key_for('prevalence', region, year, sex))
elif param == 'incidence_x_duration':
pop = pl.ones_like(pop) # population already included in iX stoch
for a in range(len(dismod3.settings.gbd_ages)-1):
age = dismod3.settings.gbd_ages[a]
ages = range(age, dismod3.settings.gbd_ages[a+1])
# special case for birth prevalence
if param == 'prevalence':
# increase end of age range for age==0
if age == 0:
ages = [0,1]
pop_weights = pop[ages]
mean = dismod3.utils.rate_for_range(mean_array, ages, pop_weights)
lower = dismod3.utils.rate_for_range(lower_array, ages, pop_weights)
upper = dismod3.utils.rate_for_range(upper_array, ages, pop_weights)
for col, val in enumerate(cols):
ws.write(row, col, vars()[val])
row += 1
def chebspace1(a, b, n, returnWeights=False):
"""Chebyshev nodes for given degree n"""
apb = 0.5 * (a + b)
bma = 0.5 * (b - a)
cs = apb - bma * cos(arange(1, 2*n, 2) * pi / (2*n))
if returnWeights:
weights = ones_like(cs)
weights = sin(arange(1, 2 * n, 2) * pi / (2 * n))
weights[1::2] = -1 * weights[1::2]
return cs, weights
else:
return cs
def chebspace1(a, b, n, returnWeights=False):
"""Chebyshev nodes for given degree n"""
apb = 0.5 * (a + b)
bma = 0.5 * (b - a)
cs = apb - bma * cos(arange(1, 2 * n, 2) * pi / (2 * n))
if returnWeights:
weights = ones_like(cs)
weights = sin(arange(1, 2 * n, 2) * pi / (2 * n))
weights[1::2] = -1 * weights[1::2]
return cs, weights
else:
return cs
def plot_funnel(pi_true, delta_str):
n = pl.exp(mc.rnormal(10, 2**-2, size=10000))
delta = float(delta_str) * pl.ones_like(n)
p = pi_true * pl.ones_like(n)
# old way:
#delta = delta * p * n
nb = rate_model.neg_binom('funnel', p, delta, p, n)
r = nb['p_pred'].value
pl.vlines([pi_true],
.1 * n.min(),
10 * n.max(),
linewidth=2,
linestyle='-',
color='w',
zorder=9)
pl.vlines([pi_true],
.1 * n.min(),
10 * n.max(),
linewidth=1,
linestyle='--',
color='black',
zorder=10)
pl.plot(r, n, 'ko', mew=0, alpha=.25)
pl.semilogy(schiz['r'],
schiz['n'],
'ks',
mew=1,
mec='white',
ms=4,
label='Observed values')
pl.xlabel('Rate (per PY)')
pl.ylabel('Study size (PY)')
pl.xticks([0, .005, .01])
pl.axis([-.0001, .0101, 50., 15000000])
pl.title(r'$\delta = %s$' % delta_str)
def truncation_hist(df, outdir=FIGS_DIR):
df2 = df[df['neuron type'].isin(['axon', 'truncated axon'])]
df2 = df2.drop_duplicates(['neuron name', 'neuron type'])
type_alphas = defaultdict(list)
for neuron_name, group in df2.groupby('neuron name'):
if len(group['neuron type']) < 2:
continue
for neuron_type, group2 in group.groupby('neuron type'):
type_alphas[neuron_type] += list(group2['alpha'])
alphas = []
weights = []
labels = []
for neuron_type in type_alphas:
alpha = type_alphas[neuron_type]
alphas.append(alpha)
weights.append(pylab.ones_like(alpha) / float(len(alpha)))
labels.append(neuron_type)
pylab.figure()
sns.set()
pylab.hist(alphas, range=(0, 1), weights=weights, label=labels)
leg = pylab.legend(frameon=True)
pylab.setp(leg.get_texts(), fontsize=20)
leg_frame = leg.get_frame()
leg_frame.set_linewidth(5)
leg_frame.set_edgecolor('k')
curr_ax = pylab.gca()
curr_ax.set_ylim((0, 1))
pylab.xlabel('alpha', size=30)
pylab.ylabel('proportion', size=30)
pylab.tight_layout()
name = 'truncation_hist'
outname = '%s/%s.pdf' % (outdir, name)
outname = outname.replace(' ', '_')
pylab.savefig('%s/%s.pdf' % (outdir, name), format='pdf')
pylab.close()
axons = pylab.array(type_alphas['axon'])
truncated_axons = pylab.array(type_alphas['truncated axon'])
differences = axons - truncated_axons
print '-------------------------'
print "Truncation test"
print min(differences), pylab.median(differences), max(differences)
print pylab.mean(differences), "+/-", pylab.std(differences, ddof=1)
print wilcoxon(axons, truncated_axons)
print ttest_rel(axons, truncated_axons)
def angles_stats():
df = pd.read_csv('%s/%s' % (OUTDIR, FNAME), skipinitialspace=True)
df['degrees'] = degrees(df['angle'])
deg = pylab.array(df['degrees'])
weight = pylab.ones_like(deg) / float(len(deg))
pylab.figure()
pylab.hist(deg, weights=weight)
pylab.xlabel('degree')
pylab.ylabel('proportion')
pylab.savefig('branch_angles/branch_angles.pdf', format='pdf')
pylab.close()
def plot_funnel(pi_true, delta_str):
n = pl.exp(mc.rnormal(10, 2 ** -2, size=10000))
delta = float(delta_str) * pl.ones_like(n)
p = pi_true * pl.ones_like(n)
# old way:
# delta = delta * p * n
nb = rate_model.neg_binom("funnel", p, delta, p, n)
r = nb["p_pred"].value
pl.vlines([pi_true], 0.1 * n.min(), 10 * n.max(), linewidth=2, linestyle="-", color="w", zorder=9)
pl.vlines([pi_true], 0.1 * n.min(), 10 * n.max(), linewidth=1, linestyle="--", color="black", zorder=10)
pl.plot(r, n, "ko", mew=0, alpha=0.25)
pl.semilogy(schiz["r"], schiz["n"], "ks", mew=1, mec="white", ms=4, label="Observed values")
pl.xlabel("Rate (per PY)")
pl.ylabel("Study size (PY)")
pl.xticks([0, 0.005, 0.01])
pl.axis([-0.0001, 0.0101, 50.0, 15000000])
pl.title(r"$\delta = %s$" % delta_str)
def neuron_types_hist(df):
df2 = df.drop_duplicates(subset='name')
pylab.figure()
for neuron_type, group in df2.groupby('neuron_type'):
alphas = list(group['alpha'])
weights = pylab.ones_like(alphas) / len(alphas)
pylab.hist(alphas, alpha=0.5, label=neuron_type,\
range=(0, 1), weights=weights)
pylab.legend()
curr_ax = pylab.gca()
curr_ax.set_ylim((0, 1))
pylab.savefig('%s/neuron_types_hist.pdf' % OUTDIR, format='pdf')
pylab.close()
def runtimes_stats():
df = pd.read_csv('test_runtimes.csv', skipinitialspace=True)
print "total trials"
print len(df['algorithm']) / len(df['algorithm'].unique())
ratios = []
labels = []
weights = []
hist_algorithms = ['prim', 'khuller']
algorithm_labels = {'prim': 'Karger', 'khuller': 'Khuller'}
sns.set()
pylab.figure()
for algorithm, group in df.groupby('algorithm'):
print algorithm
comparisons = group['comparisons'].sum()
dominated = group['dominated'].sum()
print float(dominated) / float(
comparisons), "(", dominated, "/", comparisons, ")"
print binom_test(dominated, comparisons)
group = group.groupby('points', as_index=False).agg(pylab.mean)
pylab.plot(group['points'], group['runtime'], label=algorithm)
ratio = group['cost ratio']
ratio = ratio[~pylab.isnan(ratio)]
ratio = ratio - 1
print "cost comparisons", len(ratio)
print "cost ratio", pylab.mean(ratio), "+/-", pylab.std(ratio, ddof=1)
if algorithm in hist_algorithms:
ratios.append(ratio)
labels.append(algorithm_labels[algorithm])
weight = pylab.ones_like(ratio) / float(len(ratio))
weights.append(weight)
pylab.legend(loc=2)
pylab.xlabel('number of points')
pylab.ylabel('rumtime (minutes)')
pylab.savefig('test_runtimes/runtimes.pdf', format='pdf')
pylab.close()
pylab.figure()
pylab.hist(ratios, label=labels, weights=weights)
pylab.xlabel('percent better/worse than Steiner', size=20)
pylab.ylabel('proportion', size=20)
pylab.legend()
ax = pylab.gca()
pylab.setp(ax.get_legend().get_texts(), fontsize=20) # for legend text
pylab.tight_layout()
pylab.savefig('test_runtimes/cost_ratios_hist.pdf', format='pdf')
pylab.close()
def plot_funnel(pi_true, sigma_str):
n = pl.exp(mc.rnormal(10, 2**-2, size=10000))
sigma = float(sigma_str)*pl.ones_like(n)
p = pi_true*pl.ones_like(n)
oln = rate_model.offset_log_normal('funnel', p, sigma, p, pl.sqrt(p*(1-p)/n))
r = oln['p_pred'].value
pl.vlines([pi_true], .1*n.min(), 10*n.max(),
linewidth=2, linestyle='-', color='w', zorder=9)
pl.vlines([pi_true], .1*n.min(), 10*n.max(),
linewidth=1, linestyle='--', color='black', zorder=10)
pl.plot(r, n, 'ko',
mew=0, alpha=.25)
pl.semilogy(schiz['r'], schiz['n'], 'ks', mew=1, mec='white', ms=4,
label='Observed values')
pl.xlabel('Rate (per PY)')
pl.ylabel('Study size (PY)')
pl.xticks([0, .005, .01])
pl.axis([-.0001, .0101, 50., 15000000])
pl.title(r'$\sigma = %s$'%sigma_str)
def graph_stats(df):
G = make_graph(df)
print G.number_of_nodes(), "nodes"
print G.number_of_edges(), "edges"
degrees = []
for u in G.nodes_iter():
degrees.append(G.degree(u))
bins = range(max(degrees) + 1)
weights = pylab.ones_like(degrees) / float(len(degrees))
pylab.figure()
pylab.hist(degrees, bins=bins, weights=weights)
pylab.savefig('repeatability/figs/degree_dist.pdf', format='pdf')
print "transitivity", nx.transitivity(G)
print "average clustering", nx.average_clustering(G)
shortest_path_lengths = nx.shortest_path_length(G, weight=None)
visited = set()
path_lens = []
for source in shortest_path_lengths:
for target in shortest_path_lengths[source]:
if source == target:
continue
if tuple(sorted((source, target))) in visited:
continue
path_lens.append(shortest_path_lengths[source][target])
visited.add(tuple(sorted((source, target))))
print "mean shortest path len", pylab.mean(path_lens)
print "diameter", max(path_lens)
weights = pylab.ones_like(path_lens) / float(len(path_lens))
pylab.figure()
pylab.hist(path_lens, weights=weights)
#pylab.xlabel('shortest path length (mm)')
pylab.savefig('repeatability/figs/path_lengths.pdf', format='pdf')
pylab.close()
def chebspace(a, b, n, returnWeights=False):
"""Chebyshev nodes for given degree n"""
apb = 0.5 * (a + b)
bma = 0.5 * (b - a)
cs = apb - bma * cos(arange(n) * pi / (n - 1))
# ensure that endpoints are exact
cs[0] = a
cs[-1] = b
if returnWeights:
weights = ones_like(cs)
weights[::2] = -1
weights[0] /= 2
weights[-1] /= 2
return cs, weights
else:
return cs
def chebspace(a, b, n, returnWeights=False):
"""Chebyshev nodes for given degree n"""
apb = 0.5 * (a + b)
bma = 0.5 * (b - a)
cs = apb - bma * cos(arange(n) * pi / (n-1))
# ensure that endpoints are exact
cs[0] = a
cs[-1] = b
if returnWeights:
weights = ones_like(cs)
weights[::2] = -1
weights[0] /= 2
weights[-1] /= 2
return cs, weights
else:
return cs
def difficulty_hist(df):
difficulties = []
weights = []
labels = []
pylab.figure()
for name, group in df.groupby('Different plant'):
difficulty = group['Junction difficulty']
print name
print pylab.mean(difficulty), "pm", pylab.std(difficulty, ddof=1)
weight = pylab.ones_like(difficulty) / float(len(difficulty))
difficulties.append(difficulty)
weights.append(weight)
labels.append(name + ' plant')
pylab.hist(difficulties, bins=[1, 2, 3, 4, 5], label=labels, weights=weights)
pylab.xlabel('edge difficulty')
pylab.ylabel('frequency')
pylab.legend()
pylab.savefig('repeatability/figs/repeatability.pdf', format='pdf')
pylab.close()
def test_age_pattern_model_sim():
# simulate normal data
a = pl.arange(0, 100, 5)
pi_true = .0001 * (a * (100. - a) + 100.)
sigma_true = .025*pl.ones_like(pi_true)
p = pl.maximum(0., mc.rnormal(pi_true, 1./sigma_true**2.))
# create model and priors
vars = {}
vars.update(age_pattern.age_pattern('test', ages=pl.arange(101), knots=pl.arange(0,101,5), smoothing=.1))
vars['pi'] = mc.Lambda('pi', lambda mu=vars['mu_age'], a=a: mu[a])
vars.update(rate_model.normal_model('test', vars['pi'], 0., p, sigma_true))
# fit model
m = mc.MCMC(vars)
m.sample(2)
def test_covariate_model_sim_w_hierarchy():
n = 50
# setup hierarchy
hierarchy, output_template = data_simulation.small_output()
# simulate normal data
area_list = pl.array(['all', 'USA', 'CAN'])
area = area_list[mc.rcategorical([.3, .3, .4], n)]
sex_list = pl.array(['male', 'female', 'total'])
sex = sex_list[mc.rcategorical([.3, .3, .4], n)]
year = pl.array(mc.runiform(1990, 2010, n), dtype=int)
alpha_true = dict(all=0., USA=.1, CAN=-.2)
pi_true = pl.exp([alpha_true[a] for a in area])
sigma_true = .05 * pl.ones_like(pi_true)
p = mc.rnormal(pi_true, 1. / sigma_true**2.)
model = data.ModelData()
model.input_data = pandas.DataFrame(
dict(value=p, area=area, sex=sex, year_start=year, year_end=year))
model.hierarchy, model.output_template = hierarchy, output_template
# create model and priors
vars = {}
vars.update(
covariate_model.mean_covariate_model('test', 1, model.input_data, {},
model, 'all', 'total', 'all'))
vars.update(rate_model.normal_model('test', vars['pi'], 0., p, sigma_true))
# fit model
m = mc.MCMC(vars)
m.sample(2)
assert 'sex' not in vars['U']
assert 'x_sex' in vars['X']
assert len(vars['beta']) == 1
def generate_data(N, delta_true, pi_true, heterogeneity, bias, sigma_prior):
a = pl.arange(0, 101, 1)
pi_age_true = pi_true(a)
model = data_simulation.simple_model(N)
model.parameters['p']['parameter_age_mesh'] = range(0, 101, 10)
model.parameters['p']['smoothness'] = dict(amount='Moderately')
model.parameters['p']['heterogeneity'] = heterogeneity
age_start = pl.array(mc.runiform(0, 100, size=N), dtype=int)
age_end = pl.array(mc.runiform(age_start, 100, size=N), dtype=int)
age_weights = pl.ones_like(a)
sum_pi_wt = pl.cumsum(pi_age_true*age_weights)
sum_wt = pl.cumsum(age_weights)
p = (sum_pi_wt[age_end] - sum_pi_wt[age_start]) / (sum_wt[age_end] - sum_wt[age_start])
# correct cases where age_start == age_end
i = age_start == age_end
if pl.any(i):
p[i] = pi_age_true[age_start[i]]
n = mc.runiform(10000, 100000, size=N)
model.input_data['age_start'] = age_start
model.input_data['age_end'] = age_end
model.input_data['effective_sample_size'] = n
model.input_data['true'] = p
model.input_data['value'] = mc.rnegative_binomial(n*p, delta_true*n*p) / n * pl.exp(bias)
emp_priors = {}
emp_priors['p', 'mu'] = pi_age_true
emp_priors['p', 'sigma'] = sigma_prior*pi_age_true
model.emp_priors = emp_priors
model.a = a
model.pi_age_true = pi_age_true
model.delta_true = delta_true
return model
def test_covariate_model_sim_w_hierarchy():
n = 50
# setup hierarchy
hierarchy, output_template = data_simulation.small_output()
# simulate normal data
area_list = pl.array(['all', 'USA', 'CAN'])
area = area_list[mc.rcategorical([.3, .3, .4], n)]
sex_list = pl.array(['male', 'female', 'total'])
sex = sex_list[mc.rcategorical([.3, .3, .4], n)]
year = pl.array(mc.runiform(1990, 2010, n), dtype=int)
alpha_true = dict(all=0., USA=.1, CAN=-.2)
pi_true = pl.exp([alpha_true[a] for a in area])
sigma_true = .05*pl.ones_like(pi_true)
p = mc.rnormal(pi_true, 1./sigma_true**2.)
model = data.ModelData()
model.input_data = pandas.DataFrame(dict(value=p, area=area, sex=sex, year_start=year, year_end=year))
model.hierarchy, model.output_template = hierarchy, output_template
# create model and priors
vars = {}
vars.update(covariate_model.mean_covariate_model('test', 1, model.input_data, {}, model,
'all', 'total', 'all'))
vars.update(rate_model.normal_model('test', vars['pi'], 0., p, sigma_true))
# fit model
m = mc.MCMC(vars)
m.sample(2)
assert 'sex' not in vars['U']
assert 'x_sex' in vars['X']
assert len(vars['beta']) == 1
def test_age_pattern_model_sim():
# simulate normal data
a = pl.arange(0, 100, 5)
pi_true = .0001 * (a * (100. - a) + 100.)
sigma_true = .025 * pl.ones_like(pi_true)
p = pl.maximum(0., mc.rnormal(pi_true, 1. / sigma_true**2.))
# create model and priors
vars = {}
vars.update(
age_pattern.age_pattern('test',
ages=pl.arange(101),
knots=pl.arange(0, 101, 5),
smoothing=.1))
vars['pi'] = mc.Lambda('pi', lambda mu=vars['mu_age'], a=a: mu[a])
vars.update(rate_model.normal_model('test', vars['pi'], 0., p, sigma_true))
# fit model
m = mc.MCMC(vars)
m.sample(2)
def sim(vx0,y0,L01,a0,k1,dE,m,ygrd,t,fs,steps):
# function for using fmin
# def optfunc_dE(par):
# return abs(-.5*k1*c**2+.5*(k1+par)*(c-(c*par/(k1+par)))**2-dE)
# function for using fsolve
def optfunc_dE(par, spring_par):
k1 = spring_par['k']
c = spring_par['c']
return abs(-.5*k1*c**2+.5*(k1+par)*(c-(c*par/(k1+par)))**2-dE)
g = -9.81 # gravitational force
betaTO_sim = zeros(steps)
xF=0
print 'x'
x_sim = [0]
y_sim = [0]
vx_sim = [0]
vy_sim = [0]
t_sim = [0]
k2 = 0
L02 = 0
skip_all = False
st_lo_idx = -1
# leg orientation
xL = L01 * cos(a0 * pi/180.)
yL = L01 * sin(a0 * pi/180.)
# first ground contact
t_gc = sqrt(2.*(L01*sin(a0*pi/180.)-y0)/g)
if isnan(t_gc) == True:
skip_all = True
if skip_all == False:
x_gc = vx0*t_gc
y_gc = .5*g*t_gc**2 + y0
# system parameter for first step
s = zeros(4)
s[0] = 0
s[1] = y0
s[2] = vx0
s[3] = 0
# definitions for calculation
end_sim = False
res = zeros([1,4])
res_app = []
t_v = [0]
skip_stance = False
# start loop
while end_sim == False:
st_lo_idx += 1
if st_lo_idx != 0: # if not first step
t_gc = (-res[-1,3]/g) + \
sqrt( ((res[-1,3]**2)/(g**2)) - \
((2*res[-1,1])/g-(2*L01*sin(a0*pi/180)/g)) )
if isnan(t_gc) == False and isinf(t_gc) == False:
y_gc = 0.5*g*t_gc**2 + res[-1,3]*t_gc + res[-1,1]
t_vec = linspace(0,t_gc,ceil(t_gc*fs)+1)
#print len(t_vec)
s[0] = res[-1,2]*t_gc + res[-1,0]
s[1] = y_gc
s[2] = res[-1,2]
s[3] = g*t_gc +res[-1,3]
else:
t_gc = 1
t_vec = linspace(0,t_gc,ceil(t_gc*fs)+1)
end_sim = True
skip_stance = True
#break
vx = zeros_like(t_vec)
vy = zeros_like(t_vec)
x = zeros_like(t_vec)
y = zeros_like(t_vec)
vx[0] = res[-1,2]
vy[0] = res[-1,3]
x[0] = res[-1,0]
y[0] = res[-1,1]
else: # if first step
t_vec = linspace(0,t_gc,ceil(t_gc*fs)+1)
#print t_vec
vx = zeros_like(t_vec)
vy = zeros_like(t_vec)
x = zeros_like(t_vec)
y = zeros_like(t_vec)
#print vx
vx[0] = s[2]
vy[0] = s[3]
x[0] = s[0]
y[0] = s[1]
### flight phase
x[1:] = vx[0]*t_vec[1:] + x[0]
y[1:] = .5*g*(t_vec[1:]**2)+vy[0]*t_vec[1:] + y[0]
vy[1:] = g*t_vec[1:] + vy[0]
vx[1:] = vx[0]*ones_like(t_vec[1:])
# end at apex
if st_lo_idx == steps or skip_stance == True:
tapex = vy[0]/-g
#print 'y0=',y0,'vy0=',vy[0],' tapex=',tapex
tvec_apex = linspace(0,tapex,ceil(tapex*fs)+1)
vyapex = g*tvec_apex+vy[0]
#print 'vyapex=',vyapex
yapex = .5*g*(tvec_apex**2) + vyapex[0]*tvec_apex + y[0]
xapex = vx[0] * tvec_apex + x[0]
vxapex = vx[0] * ones_like(tvec_apex)
#print 'lvy:',len(vyapex),' lvx:',len(vxapex)
x_sim = append(x_sim,xapex[1:])
y_sim = append(y_sim,yapex[1:])
vx_sim = append(vx_sim,vxapex[1:])
vy_sim = append(vy_sim,vyapex[1:])
t_sim = append(t_sim,tvec_apex[1:]+t_sim[-1])
skip_stance = True
"""
for kk in arange(2,len(y)):
if y[kk-2]<y[kk-1] and y[kk-1]>y[kk]:
end_sim = True
x_sim = append(x_sim,x[1:kk])
y_sim = append(y_sim,y[1:kk])
vx_sim = append(vx_sim,vx[1:kk])
vy_sim = append(vy_sim,vy[1:kk])
t_sim = append(t_sim,t_vec[1:kk]+t_sim[-1])
"""
### stance phase
if skip_stance == False:
# system parameter
if st_lo_idx == 0: # if first step
s[0] = x_gc # x
s[1] = y_gc # y
s[2] = vx0 # vx
s[3] = g*t_gc + 0 # vy
xF = s[0]+xL
yF = ygrd
parameter = {'k':k1,'L0':L01,'m':m,'g':g,'xF':xF,'yF':yF}
calc_length = 0.1*fs
find_pos = False
round_prec = 11 # precision of rounding
#print 'round_prec = ', round_prec
step_size = 1./fs
res = zeros([1,4])
t_v = [0]
res_app = False
skip = False
L0 = L01
FinMin = False
# start integration of stance phase
while find_pos == False:
t_vint = (step_size) * arange(calc_length)
res_int = odeint(s_dot,s,t_vint, args=(parameter,), rtol=1e-11)
# stop condition vx or y < 0
if res_int[-1,1] < 0 or res_int[-1,2] < 0:
find_pos = True
end_sim = True
skip = True
# if ymin: calculation of new k and L for energy changes
for idx in arange(1,len(res_int[:,1])-1):
if res_int[idx,1]<res_int[idx-1,1] and res_int[idx,1]<res_int[idx+1,1]:
res_int=delete(res_int,s_[idx+1::],0)
t_vint = delete(t_vint,s_[idx+1::],0)
c= L0-(sqrt(res_int[-1,1]**2 + (xF-res_int[-1,0])**2))
#dk = fmin(optfunc_dE,0,full_output=1,disp=0,xtol=1e-12)
spring_par = {'k':k1,'c':c}
dk = fsolve(optfunc_dE,x0=[0],args=(spring_par,))
#print '###### dk ####',dk
# if not find a minimum stop loop --> no step
#if dk[-1] <> 0:
# find_pos = True
# end_sim = True
# skip = True
# raise ValueError, 'no energy management possible'
# break
k2 = k1+dk[0]
L02 = L01 - c*dk[0]/(k1+dk[0])
#print 'delta L = ',L0 - L02, ' delta k = ', dk[0]
L0 = L02
parameter = {'k':k2,'L0':L02,'m':m,'g':g,'xF':xF,'yF':yF}
FinMin = True
break
# if not ymin: start one before to avoid ignoring while change
if FinMin == False:
res_int = res_int[:-1]
t_vint = t_vint[:-1]
# virtual leg after integration
L_virt = sqrt(res_int[-1,1]**2 + (xF-res_int[-1,0])**2)
if L_virt < L0 and skip == False:
s[0] = res_int[-1,0]
s[1] = res_int[-1,1]
s[2] = res_int[-1,2]
s[3] = res_int[-1,3]
elif L_virt >= L0 and skip == False:
for jj in arange(1,len(res_int)):
L_virt_1 = sqrt(res_int[jj,1]**2 + abs(xF-res_int[jj,0])**2)
L_virt_2 = sqrt(res_int[jj-1,1]**2 + abs(xF-res_int[jj-1,0])**2)
if round(L_virt_2,round_prec) == round(L0,round_prec):
find_pos = True
res_int = res_int[0:jj,:]
res_app = True
t_vint = t_vint[0:jj]
if res_int[-1,3] <=0: end_sim = True
break
elif L_virt_2 < L0 and L_virt_1 > L0:
s[0] = res_int[jj-1,0]
s[1] = res_int[jj-1,1]
s[2] = res_int[jj-1,2]
s[3] = res_int[jj-1,3]
res_int = res_int[0:jj,:]
res_app = True
t_vint = t_vint[0:jj]
step_size = step_size / 10.
break
# append arrays
if res_app == False:
res = res_int
t_v = t_vint
res_app = True
else:
res = append(res,res_int[1::],0)
t_v = append(t_v,t_vint[1::]+t_v[-1])
# take of angle
#print st_lo_idx
betaTO_sim[st_lo_idx] = arctan(res[-1,1]/(res[-1,0]-xF))*180./pi
# append arrays
x_sim = append(x_sim,append(x,res[:,0]))
y_sim = append(y_sim,append(y,res[:,1]))
vx_sim = append(vx_sim,append(vx,res[:,2]))
vy_sim = append(vy_sim,append(vy,res[:,3]))
t_sim = append(t_sim,append(t_vec,t_v+t_gc)+t_sim[-1])
else: #skip_stance == True because of ending at apex
end_sim = True
# cut array
x_sim = x_sim[1::]
y_sim = y_sim[1::]
vx_sim = vx_sim[1::]
vy_sim = vy_sim[1::]
t_sim = t_sim[1::]
#print 'vor return'
return x_sim,y_sim,vx_sim,vy_sim,t_sim,st_lo_idx,betaTO_sim,xF,k2,L02
def fit_world(id, fast_fit=False, zero_re=True, alt_prior=False, global_heterogeneity='Slightly'):
""" Fit consistent for all data in world
Parameters
----------
id : int
The model id number for the job to fit
Example
-------
>>> import fit_world
>>> dm = fit_world.dismod3.load_disease_model(1234)
>>> fit_world.fit_world(dm)
"""
dir = dismod3.settings.JOB_WORKING_DIR % id
## load the model from disk or from web
import simplejson as json
import data
reload(data)
try:
model = data.ModelData.load(dir)
print 'loaded data from new format from %s' % dir
dm = dismod3.load_disease_model(id)
except (IOError, AssertionError):
dm = dismod3.load_disease_model(id)
model = data.ModelData.from_gbd_jsons(json.loads(dm.to_json()))
try:
model.save(dir)
print 'loaded data from json, saved in new format for next time in %s' % dir
except IOError:
print 'loaded data from json, failed to save in new format'
## next block fills in missing covariates with zero
for col in model.input_data.columns:
if col.startswith('x_'):
model.input_data[col] = model.input_data[col].fillna(0.)
# also fill all covariates missing in output template with zeros
model.output_template = model.output_template.fillna(0)
# set all heterogeneity priors to Slightly for the global fit
for t in model.parameters:
if 'heterogeneity' in model.parameters[t]:
model.parameters[t]['heterogeneity'] = global_heterogeneity
### For testing:
## speed up computation by reducing number of knots
## for t in 'irf':
## model.parameters[t]['parameter_age_mesh'] = [0, 100]
model.vars += dismod3.ism.consistent(model,
reference_area='all',
reference_sex='total',
reference_year='all',
priors={},
zero_re=zero_re)
## fit model to data
if fast_fit:
dm.map, dm.mcmc = dismod3.fit.fit_consistent(model, 105, 0, 1, 100)
else:
dm.map, dm.mcmc = dismod3.fit.fit_consistent(model, iter=50000, burn=10000, thin=40, tune_interval=1000, verbose=True)
dm.model = model
# borrow strength to inform sigma_alpha between rate types post-hoc
types_with_re = ['rr', 'f', 'i', 'm', 'smr', 'p', 'r', 'pf', 'm_with', 'X']
## first calculate sigma_alpha_bar from posterior draws from each alpha
alpha_vals = []
for type in types_with_re:
if 'alpha' in model.vars[type]:
for alpha_i in model.vars[type]['alpha']:
alpha_vals += [a for a in alpha_i.trace() if a != 0] # remove zeros because areas with no siblings are included for convenience but are pinned to zero
## then blend sigma_alpha_i and sigma_alpha_bar for each sigma_alpha_i
if len(alpha_vals) > 0:
sigma_alpha_bar = pl.std(alpha_vals)
for type in types_with_re:
if 'sigma_alpha' in model.vars[type]:
for sigma_alpha_i in model.vars[type]['sigma_alpha']:
cur_val = sigma_alpha_i.trace()
sigma_alpha_i.trace._trace[0] = (cur_val + sigma_alpha_bar) * pl.ones_like(sigma_alpha_i.trace._trace[0])
for t in 'p i r f rr pf m_with'.split():
param_type = dict(i='incidence', r='remission', f='excess-mortality', p='prevalence', rr='relative-risk', pf='prevalence_x_excess-mortality', m_with='mortality')[t]
#graphics.plot_one_type(model, model.vars[t], {}, t)
for a in [dismod3.utils.clean(a) for a in dismod3.settings.gbd_regions]:
print 'generating empirical prior for %s' % a
for s in dismod3.settings.gbd_sexes:
for y in dismod3.settings.gbd_years:
key = dismod3.utils.gbd_key_for(param_type, a, y, s)
if t in model.parameters and 'level_bounds' in model.parameters[t]:
lower=model.parameters[t]['level_bounds']['lower']
upper=model.parameters[t]['level_bounds']['upper']
else:
lower=0
upper=pl.inf
emp_priors = covariate_model.predict_for(model,
model.parameters.get(t, {}),
'all', 'total', 'all',
a, dismod3.utils.clean(s), int(y),
alt_prior,
model.vars[t], lower, upper)
dm.set_mcmc('emp_prior_mean', key, emp_priors.mean(0))
if 'eta' in model.vars[t]:
N,A = emp_priors.shape # N samples, for A age groups
delta_trace = pl.transpose([pl.exp(model.vars[t]['eta'].trace()) for _ in range(A)]) # shape delta matrix to match prediction matrix
emp_prior_std = pl.sqrt(emp_priors.var(0) + (emp_priors**2 / delta_trace).mean(0))
else:
emp_prior_std = emp_priors.std(0)
dm.set_mcmc('emp_prior_std', key, emp_prior_std)
from fit_emp_prior import store_effect_coefficients
store_effect_coefficients(dm, model.vars[t], param_type)
if 'p_pred' in model.vars[t]:
graphics.plot_one_ppc(model, t)
pl.savefig(dir + '/prior-%s-ppc.png'%param_type)
if 'p_pred' in model.vars[t] or 'lb' in model.vars[t]:
graphics.plot_one_effects(model, t)
pl.savefig(dir + '/prior-%s-effects.png'%param_type)
for t in 'i r f p rr pf X m_with smr'.split():
fname = dir + '/empirical_priors/data-%s.csv'%t
print 'saving tables for', t, 'to', fname
if 'data' in model.vars[t] and 'p_pred' in model.vars[t]:
stats = model.vars[t]['p_pred'].stats(batches=5)
model.vars[t]['data']['mu_pred'] = stats['mean']
model.vars[t]['data']['sigma_pred'] = stats['standard deviation']
stats = model.vars[t]['pi'].stats(batches=5)
model.vars[t]['data']['mc_error'] = stats['mc error']
model.vars[t]['data']['residual'] = model.vars[t]['data']['value'] - model.vars[t]['data']['mu_pred']
model.vars[t]['data']['abs_residual'] = pl.absolute(model.vars[t]['data']['residual'])
#if 'delta' in model.vars[t]:
# model.vars[t]['data']['logp'] = [mc.negative_binomial_like(n*p_obs, n*p_pred, n*p_pred*d) for n, p_obs, p_pred, d \
# in zip(model.vars[t]['data']['effective_sample_size'],
# model.vars[t]['data']['value'],
# model.vars[t]['data']['mu_pred'],
# pl.atleast_1d(model.vars[t]['delta'].stats()['mean']))]
model.vars[t]['data'].to_csv(fname)
graphics.plot_fit(model)
pl.savefig(dir + '/prior.png')
graphics.plot_acorr(model)
pl.savefig(dir + '/prior-convergence.png')
graphics.plot_trace(model)
pl.savefig(dir + '/prior-trace.png')
# save results (do this last, because it removes things from the disease model that plotting function, etc, might need
try:
dm.save('dm-%d-prior-%s.json' % (dm.id, 'all'))
except IOError, e:
print e
pl.title(r'$\sigma = %s$'%sigma_str)
pl.figure(figsize=(11, 8.5), dpi=120)
pl.subplots_adjust(wspace=.4)
pl.subplot(2,2,1)
plot_funnel(.004, '0.5')
pl.subplot(2,2,2)
plot_funnel(.004, '0.1')
pl.subplot(2,1,2)
r = pl.array(schiz['r'])
n = pl.array(schiz['n'])
pi = mc.Uniform('pi', 0, 1, value=.001)
sigma = mc.Uniform('sigma', 0, 100, value=.0001)
oln = rate_model.offset_log_normal('funnel', pi*pl.ones_like(n), sigma*pl.ones_like(n), r, pl.sqrt(r*(1-r)/n))
mcmc = mc.MCMC([pi, sigma, oln])
mcmc.sample(20000, 10000, 10, verbose=False, progress_bar=False)
sorted_indices = r.argsort().argsort()
jitter = mc.rnormal(0, .1**-2, len(oln['p_pred'].trace()))
for i,s_i in enumerate(sorted_indices):
pl.plot(s_i+jitter, oln['p_pred'].trace()[:, i], 'ko', mew=0, alpha=.25, zorder=-99)
pl.errorbar(sorted_indices, r, yerr=1.96*pl.sqrt(r*(1-r)/n), fmt='ws', mew=1, mec='white', ms=5, elinewidth=3, capsize=0)
pl.errorbar(sorted_indices, r, yerr=1.96*pl.sqrt(r*(1-r)/n), fmt='ks', mew=1, mec='white', ms=5)
pl.xticks([])
pl.ylabel('Rate (per PY)')
pl.axis([-.5, 15.5,-.0001,.0121])
def greedy_eval_stats(model='uniform'):
df = pd.read_csv('greedy_eval_%s.csv' % model, skipinitialspace=True)
df['success'] = df['cost ratio'] == 0
df['error'] = df['cost ratio'] < 0
mcost_ratios = []
scost_ratios = []
cost_ratios = []
time_ratios = []
algorithm_labels = {'brute force' : 'Brute force', 'steiner' : 'Greedy', 'prim' : 'Karger', 'khuller' : 'Khuller'}
labels = []
for algorithm, group in df.groupby('algorithm'):
print algorithm
alpha = group['alpha']
print len(alpha) / len(alpha.unique()), "point sets", len(alpha), "trials"
print "success rate", pylab.mean(group['success'])
print "error rate", pylab.mean(group['error'])
print "dominated", pylab.mean(group['dominated'])
mcost_ratio = group['mcost ratio']
scost_ratio = group['scost ratio']
cost_ratio = group['cost ratio']
time_ratio = group['time ratio']
print "mcost ratio", pylab.nanmean(mcost_ratio), "+/-", pylab.nanstd(mcost_ratio, ddof=1)
print "scost ratio", pylab.nanmean(scost_ratio), "+/-", pylab.nanstd(scost_ratio, ddof=1)
print "cost ratio", pylab.nanmean(cost_ratio), "+/-", pylab.nanstd(cost_ratio, ddof=1)
print "time ratio", pylab.nanmean(time_ratio), "+/-", pylab.nanstd(time_ratio, ddof=1)
print "max mcost ratio", pylab.nanmax(mcost_ratio)
print "max scost ratio", pylab.nanmax(scost_ratio)
print "max cost ratio", pylab.nanmax(cost_ratio)
print "max time ratio", pylab.nanmax(time_ratio)
mcost_ratios.append(mcost_ratio)
scost_ratios.append(scost_ratio)
cost_ratios.append(cost_ratio)
time_ratios.append(time_ratio)
labels.append(algorithm_labels[algorithm])
for points, points_group in group.groupby('points'):
print points, pylab.mean(points_group['time'])
sns.set()
axis_text = {'mcost' : 'wiring cost', 'scost' : 'conduction delay', 'cost' : 'Pareto cost', 'time' : 'runtime'}
for ratios, fname in zip([mcost_ratios, scost_ratios, cost_ratios, time_ratios], ['mcost', 'scost', 'cost', 'time']):
pylab.figure()
x = []
weights = []
for ratio, label in zip(ratios, labels):
ratio = pylab.array(ratio)
ratio = ratio[~pylab.isnan(ratio)]
x.append(ratio)
wt = pylab.ones_like(ratio) / float(len(ratio))
weights.append(wt)
pylab.hist(x, weights=weights, label=labels)
pylab.legend()
ax = pylab.gca()
pylab.setp(ax.get_legend().get_texts(), fontsize=20) # for legend text
ax_text = axis_text[fname]
pylab.xlabel('%s (percent higher/lower than %s)' % (ax_text, BASELINE), size=20)
pylab.ylabel('proportion', size=20)
pylab.tight_layout()
pylab.savefig('greedy_eval/%s_ratios_hist.pdf' % fname, format='pdf')
pylab.close()
def SLIP_step(IC, SLIP_params, sim_params=[]):
"""
simulates the SLIP
IC: initial state vector, containing y0, vx0
(x0 is assumed to be 0; vy0 = 0 (apex);
also ground level = 0)
SLIP_params:
k
L0
m
alpha
dE: energy change in "midstance" by changing k and L0
g: gravity (negative! should be ~ -9.81 for SI)
"""
alpha = SLIP_params['alpha']
k = SLIP_params['k']
L0 = SLIP_params['L0']
dE = SLIP_params['dE']
g = SLIP_params['g']
m = SLIP_params['m']
y0 = IC[0]
vx0 = IC[1]
if g >= 0:
raise ValueError, "gravity points into wrong direction!"
# concatenate state vector of four elements:
# (1) time to touchdown
# (2) time to vy = 0
# (3) time to takeoff
# (4) time to apex
# (1) and (4) are analytically described
y_land = L0 * sin(alpha)
if y0 < y_land:
raise ValueError, "invalid starting condition"
# before starting, define the model:
def SLIP_ode(y, t, params):
"""
defines the ODE of the SLIP, under stance condition
state:
[x
y
vx
vy]
params:
{'L0' : leg rest length
'x0' : leg touchdown position
'k' : spring stiffness
'm' : mass}
"""
dy0 = y[2]
dy1 = y[3]
L = sqrt((y[0] - params['xF'])**2 + y[1]**2)
F = params['k'] * (params['L0'] - L)
Fx = F * (y[0] - params['xF']) / L
Fy = F * y[1] / L
dy2 = Fx / m
dy3 = Fy / m + params['g']
return hstack([dy0, dy1, dy2, dy3])
def sim_until(IC, params, stop_fcn, tmax=2.):
"""
simulated the SLIP_ode until stop_fcn has a zero-crossing
includes a refinement of the time at this instant
stop_fcn must be a function of the system state, e.g.
stop_fcn(IC) must exist
this function is especially adapted to the SLIP state,
so it uses dot(x1) = x3, dot(x2) = x4
tmax: maximal simulation time [s]
"""
init_sign = sign(stop_fcn(IC))
#1st: evaluate a certain fraction
tvec_0 = .0001 * arange(50)
sim_results = []
sim_tvecs = []
newIC = IC
sim_results.append(
odeint(SLIP_ode, newIC, tvec_0, args=(params, ), rtol=1e-12))
sim_tvecs.append(tvec_0)
check_vec = [init_sign * stop_fcn(x) for x in sim_results[-1]]
t_tot = 0.
while min(check_vec) > 0:
newIC = sim_results[-1][-1, :]
sim_results.append(
odeint(SLIP_ode, newIC, tvec_0, args=(params, ), rtol=1e-12))
sim_tvecs.append(tvec_0)
check_vec = [init_sign * stop_fcn(x) for x in sim_results[-1]]
t_tot += tvec_0[-1]
# time exceeded or ground hit
if t_tot > tmax or min(sim_results[-1][:, 1] < 0):
raise SimFailError, "simulation failed"
# now: zero-crossing detected
# -> refine!
minidx = find(array(check_vec) < 0)[0]
if minidx == 0:
# this should not happen because the first value in
# check_vec should be BEFORE the zero_crossing by
# construction
raise ValueError, "ERROR: this should not happen!"
# refine simulation by factor 50, but only for two
# adjacent original time frames
newIC = sim_results[-1][minidx - 1, :]
sim_results[-1] = sim_results[-1][:minidx, :]
sim_tvecs[-1] = sim_tvecs[-1][:minidx]
# avoid that last position can be the zero-crossing
n_refine = 10000
tvec_0 = linspace(tvec_0[0], tvec_0[1] + 2. / n_refine, n_refine + 2)
sim_results.append(
odeint(SLIP_ode, newIC, tvec_0, args=(params, ), rtol=1e-12))
sim_tvecs.append(tvec_0)
# linearly interpolate to zero
check_vec = [init_sign * stop_fcn(x) for x in sim_results[-1]]
minidx = find(array(check_vec) < 0)[0]
if minidx == 0:
# this should not happen because the first value in
# check_vec should be BEFORE the zero_crossing by
# construction
raise ValueError, "ERROR: this should not happen! (2)"
# compute location of zero-crossing
y0 = sim_results[-1][minidx - 1, :]
y1 = sim_results[-1][minidx, :]
fcn0 = stop_fcn(y0)
fcn1 = stop_fcn(y1)
t0 = tvec_0[minidx - 1]
t1 = tvec_0[minidx]
t_zero = t0 - (t1 - t0) * fcn0 / (fcn1 - fcn0)
# cut last simulation result and replace last values
# by interpolated values
sim_results[-1] = sim_results[-1][:minidx + 1, :]
sim_tvecs[-1] = sim_tvecs[-1][:minidx + 1]
for coord in arange(sim_results[-1].shape[1]):
sim_results[-1][-1, coord] = interp(
t_zero, [t0, t1],
[sim_results[-1][-2, coord], sim_results[-1][-1, coord]])
sim_tvecs[-1][-1] = t_zero
#newIC = sim_results[-1][minidx-1,:]
#sim_results[-1] = sim_results[-1][:minidx,:]
#sim_tvecs[-1] = sim_tvecs[-1][:minidx]
#tvec_0 = linspace(tvec_0[0],tvec_0[1],100)
#sim_results.append ( odeint(SLIP_ode, newIC, tvec_0,
# args=(params,),rtol=1e-9))
#sim_tvecs.append(tvec_0)
# concatenate lists
sim_data = vstack(
[x[:-1, :] for x in sim_results[:-1] if x.shape[0] > 1] + [
sim_results[-1],
])
sim_time = [
sim_tvecs[0],
]
for idx in arange(1, len(sim_tvecs)):
sim_time.append(sim_tvecs[idx] + sim_time[-1][-1])
sim_time = hstack([x[:-1] for x in sim_time[:-1]] + [
sim_time[-1],
])
return sim_data, sim_time
# Section 1: time to touchdown
# TODO: make sampling frequency regular
t_flight1 = sqrt(-2. * (y0 - y_land) / g)
#t_flight = sqrt()
tvec_flight1 = .01 * arange(t_flight1 * 100.)
vy_flight1 = tvec_flight1 * g
y_flight1 = y0 + .5 * g * (tvec_flight1**2)
x_flight1 = vx0 * tvec_flight1
vx_flight1 = vx0 * ones_like(tvec_flight1)
# Section 2: time to vy = 0
# approach: calculate forward -> estimate interval of
# zero position of vy -> refine simulation in that interval
# until a point with vy sufficiently close to zero is in the
# resulting vector
params = {
'L0': L0,
'xF': t_flight1 * vx0 + L0 * cos(alpha),
'k': k,
'm': m,
'g': g
}
IC = array([t_flight1 * vx0, y_land, vx0, t_flight1 * g])
# initial guess: L0*cos(alpha)/vx0
#t_sim1 = L0*cos(alpha)/vx0
# TODO: implement sim_fail check!
sim_fail = False
try:
sim_phase2, t_phase2 = sim_until(IC, params, lambda x: x[3])
t_phase2 += t_flight1
except SimFailError:
print 'simulation aborted (phase 2)\n'
sim_fail = True
# Phase 3:
if not sim_fail:
L = sqrt(sim_phase2[-1, 1]**2 + (sim_phase2[-1, 0] - params['xF'])**2)
dk, dL = dk_dL(L0, k, L, dE)
params2 = deepcopy(params)
params2['k'] += dk
params2['L0'] += dL
IC = sim_phase2[-1, :]
compression = (lambda x: sqrt(
(x[0] - params2['xF'])**2 + x[1]**2) - params2['L0'])
#print ('L:', L, 'dk', dk, 'dL', dL, 'dE', dE, '\ncompression:', compression(IC),
# 'IC', IC)
try:
sim_phase3, t_phase3 = sim_until(IC, params2, compression)
sim_phase3 = sim_phase3[1:, :]
t_phase3 = t_phase3[1:] + t_phase2[-1]
except SimFailError:
print 'simulation aborted (phase 3)\n'
sim_fail = True
# Phase 4:
if not sim_fail:
# time to apex
# TODO: make sampling frequency regular
vy_liftoff = sim_phase3[-1, 3]
t_flight2 = -1. * vy_liftoff / g
#t_flight = sqrt()
tvec_flight2 = arange(t_flight2, 0, -.001)[::-1]
vy_flight2 = tvec_flight2 * g + vy_liftoff
y_flight2 = (sim_phase3[-1, 1] + vy_liftoff * tvec_flight2 + .5 * g *
(tvec_flight2**2))
x_flight2 = sim_phase3[-1, 0] + sim_phase3[-1, 2] * tvec_flight2
vx_flight2 = sim_phase3[-1, 2] * ones_like(tvec_flight2)
tvec_flight2 += t_phase3[-1]
# todo: return data until error
if sim_fail:
return {
't': None,
'x': None,
'y': None,
'vx': None,
'vy': None,
'sim_fail': sim_fail,
'dk': None,
'dL': None
}
# finally: concatenate phases
x_final = hstack(
[x_flight1, sim_phase2[:, 0], sim_phase3[:, 0], x_flight2])
y_final = hstack(
[y_flight1, sim_phase2[:, 1], sim_phase3[:, 1], y_flight2])
vx_final = hstack(
[vx_flight1, sim_phase2[:, 2], sim_phase3[:, 2], vx_flight2])
vy_final = hstack(
[vy_flight1, sim_phase2[:, 3], sim_phase3[:, 3], vy_flight2])
tvec_final = hstack([tvec_flight1, t_phase2, t_phase3, tvec_flight2])
return {
't': tvec_final,
'x': x_final,
'y': y_final,
'vx': vx_final,
'vy': vy_final,
'sim_fail': sim_fail,
'dk': dk,
'dL': dL,
#'sim_res':sim_res,
#'sim_phase2': sim_phase2_cut,
#'t_phase2': t_phase2_cut
}
def validate_age_integrating_model_sim(N=500,
delta_true=.15,
pi_true=quadratic):
## generate simulated data
a = pl.arange(0, 101, 1)
pi_age_true = pi_true(a)
model = data_simulation.simple_model(N)
#model.parameters['p']['parameter_age_mesh'] = range(0, 101, 10)
#model.parameters['p']['smoothness'] = dict(amount='Very')
age_start = pl.array(mc.runiform(0, 100, size=N), dtype=int)
age_end = pl.array(mc.runiform(age_start, 100, size=N), dtype=int)
age_weights = pl.ones_like(a)
sum_pi_wt = pl.cumsum(pi_age_true * age_weights)
sum_wt = pl.cumsum(age_weights)
p = (sum_pi_wt[age_end] - sum_pi_wt[age_start]) / (sum_wt[age_end] -
sum_wt[age_start])
# correct cases where age_start == age_end
i = age_start == age_end
if pl.any(i):
p[i] = pi_age_true[age_start[i]]
n = mc.runiform(100, 10000, size=N)
model.input_data['age_start'] = age_start
model.input_data['age_end'] = age_end
model.input_data['effective_sample_size'] = n
model.input_data['true'] = p
model.input_data['value'] = mc.rnegative_binomial(n * p,
delta_true * n * p) / n
## Then fit the model and compare the estimates to the truth
model.vars = {}
model.vars['p'] = data_model.data_model('p', model, 'p', 'all', 'total',
'all', None, None, None)
model.map, model.mcmc = fit_model.fit_data_model(model.vars['p'],
iter=10000,
burn=5000,
thin=25,
tune_interval=100)
graphics.plot_one_ppc(model.vars['p'], 'p')
graphics.plot_convergence_diag(model.vars)
graphics.plot_one_type(model, model.vars['p'], {}, 'p')
pl.plot(a, pi_age_true, 'r:', label='Truth')
pl.legend(fancybox=True, shadow=True, loc='upper left')
pl.show()
model.input_data['mu_pred'] = model.vars['p']['p_pred'].stats()['mean']
model.input_data['sigma_pred'] = model.vars['p']['p_pred'].stats(
)['standard deviation']
data_simulation.add_quality_metrics(model.input_data)
model.delta = pandas.DataFrame(dict(true=[delta_true]))
model.delta['mu_pred'] = pl.exp(model.vars['p']['eta'].trace()).mean()
model.delta['sigma_pred'] = pl.exp(model.vars['p']['eta'].trace()).std()
data_simulation.add_quality_metrics(model.delta)
print 'delta'
print model.delta
print '\ndata prediction bias: %.5f, MARE: %.3f, coverage: %.2f' % (
model.input_data['abs_err'].mean(),
pl.median(pl.absolute(model.input_data['rel_err'].dropna())),
model.input_data['covered?'].mean())
model.mu = pandas.DataFrame(
dict(true=pi_age_true,
mu_pred=model.vars['p']['mu_age'].stats()['mean'],
sigma_pred=model.vars['p']['mu_age'].stats()
['standard deviation']))
data_simulation.add_quality_metrics(model.mu)
model.results = dict(param=[], bias=[], mare=[], mae=[], pc=[])
data_simulation.add_to_results(model, 'delta')
data_simulation.add_to_results(model, 'mu')
data_simulation.add_to_results(model, 'input_data')
model.results = pandas.DataFrame(model.results,
columns='param bias mae mare pc'.split())
print model.results
return model
'incidence': 'i',
'remission': 'r',
'excess-mortality': 'f'
}[rate_type]
for i, k_i in enumerate(model.vars[t]['knots']):
model.vars[t]['gamma'][i].value = pl.log(pl.maximum(1.e-9, value[i]))
def set_birth_prev(value):
model.vars['logit_C0'].value = mc.logit(pl.maximum(1.e-9, value))
# <codecell>
### @export 'mental'
set_rate('remission', pl.ones_like(ages) * .25)
set_rate('incidence',
pl.where((ages > 14) & (ages < 65), (65 - ages) * .001, 0.))
set_birth_prev(0)
set_rate('excess-mortality', 1e-4 * pl.ones_like(ages))
book_graphics.plot_age_patterns(model, yticks=[0, .2, .4], panel='a')
pl.savefig('forward-sim-mental.pdf')
# <codecell>
### @export 'congenital'
set_rate('remission', pl.zeros_like(ages))
set_rate('incidence', pl.zeros_like(ages))
set_birth_prev(.2)
set_rate('excess-mortality', .5 * (ages / 100.)**2)
def SLIP_step(IC,SLIP_params,sim_params = []):
"""
simulates the SLIP
IC: initial state vector, containing y0, vx0
(x0 is assumed to be 0; vy0 = 0 (apex);
also ground level = 0)
SLIP_params:
k
L0
m
alpha
dE: energy change in "midstance" by changing k and L0
g: gravity (negative! should be ~ -9.81 for SI)
"""
alpha = SLIP_params['alpha']
k = SLIP_params['k']
L0 = SLIP_params['L0']
dE = SLIP_params['dE']
g = SLIP_params['g']
m = SLIP_params['m']
y0 = IC[0]
vx0 = IC[1]
if g >= 0:
raise ValueError, "gravity points into wrong direction!"
# concatenate state vector of four elements:
# (1) time to touchdown
# (2) time to vy = 0
# (3) time to takeoff
# (4) time to apex
# (1) and (4) are analytically described
y_land = L0*sin(alpha)
if y0 < y_land:
raise ValueError, "invalid starting condition"
# before starting, define the model:
def SLIP_ode(y,t,params):
"""
defines the ODE of the SLIP, under stance condition
state:
[x
y
vx
vy]
params:
{'L0' : leg rest length
'x0' : leg touchdown position
'k' : spring stiffness
'm' : mass}
"""
dy0 = y[2]
dy1 = y[3]
L = sqrt((y[0]-params['xF'])**2 + y[1]**2)
F = params['k']*(params['L0']-L)
Fx = F*(y[0]-params['xF'])/L
Fy = F*y[1]/L
dy2 = Fx/m
dy3 = Fy/m + params['g']
return hstack([dy0,dy1,dy2,dy3])
def sim_until(IC, params, stop_fcn, tmax = 2.):
"""
simulated the SLIP_ode until stop_fcn has a zero-crossing
includes a refinement of the time at this instant
stop_fcn must be a function of the system state, e.g.
stop_fcn(IC) must exist
this function is especially adapted to the SLIP state,
so it uses dot(x1) = x3, dot(x2) = x4
tmax: maximal simulation time [s]
"""
init_sign = sign(stop_fcn(IC))
#1st: evaluate a certain fraction
tvec_0 = .0001*arange(50)
sim_results = []
sim_tvecs = []
newIC = IC
sim_results.append (odeint(SLIP_ode,newIC,tvec_0,
args=(params,),rtol=1e-12))
sim_tvecs.append(tvec_0)
check_vec = [init_sign*stop_fcn(x) for x in sim_results[-1]]
t_tot = 0.
while min(check_vec) > 0:
newIC = sim_results[-1][-1,:]
sim_results.append ( odeint(SLIP_ode, newIC, tvec_0,
args=(params,),rtol=1e-12))
sim_tvecs.append(tvec_0)
check_vec = [init_sign*stop_fcn(x) for x in sim_results[-1]]
t_tot += tvec_0[-1]
# time exceeded or ground hit
if t_tot > tmax or min(sim_results[-1][:,1] < 0):
raise SimFailError, "simulation failed"
# now: zero-crossing detected
# -> refine!
minidx = find(array(check_vec) < 0)[0]
if minidx == 0:
# this should not happen because the first value in
# check_vec should be BEFORE the zero_crossing by
# construction
raise ValueError, "ERROR: this should not happen!"
# refine simulation by factor 50, but only for two
# adjacent original time frames
newIC = sim_results[-1][minidx-1,:]
sim_results[-1] = sim_results[-1][:minidx,:]
sim_tvecs[-1] = sim_tvecs[-1][:minidx]
# avoid that last position can be the zero-crossing
n_refine = 10000
tvec_0 = linspace(tvec_0[0], tvec_0[1] + 2./n_refine, n_refine+2)
sim_results.append ( odeint(SLIP_ode, newIC, tvec_0,
args=(params,),rtol=1e-12))
sim_tvecs.append(tvec_0)
# linearly interpolate to zero
check_vec = [init_sign*stop_fcn(x) for x in sim_results[-1]]
minidx = find(array(check_vec) < 0)[0]
if minidx == 0:
# this should not happen because the first value in
# check_vec should be BEFORE the zero_crossing by
# construction
raise ValueError, "ERROR: this should not happen! (2)"
# compute location of zero-crossing
y0 = sim_results[-1][minidx-1,:]
y1 = sim_results[-1][minidx,:]
fcn0 = stop_fcn(y0)
fcn1 = stop_fcn(y1)
t0 = tvec_0[minidx-1]
t1 = tvec_0[minidx]
t_zero = t0 - (t1-t0)*fcn0/(fcn1 - fcn0)
# cut last simulation result and replace last values
# by interpolated values
sim_results[-1] = sim_results[-1][:minidx+1,:]
sim_tvecs[-1] = sim_tvecs[-1][:minidx+1]
for coord in arange(sim_results[-1].shape[1]):
sim_results[-1][-1,coord] = interp(
t_zero, [t0,t1],
[sim_results[-1][-2,coord], sim_results[-1][-1,coord]] )
sim_tvecs[-1][-1] = t_zero
#newIC = sim_results[-1][minidx-1,:]
#sim_results[-1] = sim_results[-1][:minidx,:]
#sim_tvecs[-1] = sim_tvecs[-1][:minidx]
#tvec_0 = linspace(tvec_0[0],tvec_0[1],100)
#sim_results.append ( odeint(SLIP_ode, newIC, tvec_0,
# args=(params,),rtol=1e-9))
#sim_tvecs.append(tvec_0)
# concatenate lists
sim_data = vstack( [x[:-1,:] for x in sim_results[:-1] if x.shape[0] > 1]
+ [sim_results[-1],])
sim_time = [sim_tvecs[0],]
for idx in arange(1,len(sim_tvecs)):
sim_time.append(sim_tvecs[idx] + sim_time[-1][-1])
sim_time = hstack([x[:-1] for x in sim_time[:-1]] + [sim_time[-1],])
return sim_data, sim_time
# Section 1: time to touchdown
# TODO: make sampling frequency regular
t_flight1 = sqrt(-2.*(y0 - y_land)/g)
#t_flight = sqrt()
tvec_flight1 = .01*arange(t_flight1*100.)
vy_flight1 = tvec_flight1*g
y_flight1 = y0 + .5*g*(tvec_flight1**2)
x_flight1 = vx0*tvec_flight1
vx_flight1 = vx0*ones_like(tvec_flight1)
# Section 2: time to vy = 0
# approach: calculate forward -> estimate interval of
# zero position of vy -> refine simulation in that interval
# until a point with vy sufficiently close to zero is in the
# resulting vector
params = {'L0' : L0,
'xF' : t_flight1*vx0 + L0*cos(alpha),
'k' : k,
'm' : m,
'g' : g}
IC = array([t_flight1*vx0, y_land, vx0, t_flight1*g])
# initial guess: L0*cos(alpha)/vx0
#t_sim1 = L0*cos(alpha)/vx0
# TODO: implement sim_fail check!
sim_fail = False
try:
sim_phase2, t_phase2 = sim_until(IC,params,lambda x: x[3])
t_phase2 += t_flight1
except SimFailError:
print 'simulation aborted (phase 2)\n'
sim_fail = True
# Phase 3:
if not sim_fail:
L = sqrt(sim_phase2[-1,1]**2 + (sim_phase2[-1,0]-params['xF'])**2 )
dk, dL = dk_dL(L0,k,L,dE)
params2 = deepcopy(params)
params2['k'] += dk
params2['L0'] += dL
IC = sim_phase2[-1,:]
compression = (lambda x: sqrt(
(x[0]-params2['xF'])**2 + x[1]**2)
- params2['L0'] )
#print ('L:', L, 'dk', dk, 'dL', dL, 'dE', dE, '\ncompression:', compression(IC),
# 'IC', IC)
try:
sim_phase3, t_phase3 = sim_until(IC, params2,compression)
sim_phase3 = sim_phase3[1:,:]
t_phase3 = t_phase3[1:] + t_phase2[-1]
except SimFailError:
print 'simulation aborted (phase 3)\n'
sim_fail = True
# Phase 4:
if not sim_fail:
# time to apex
# TODO: make sampling frequency regular
vy_liftoff = sim_phase3[-1,3]
t_flight2 = -1.*vy_liftoff/g
#t_flight = sqrt()
tvec_flight2 = arange(t_flight2,0,-.001)[::-1]
vy_flight2 = tvec_flight2*g + vy_liftoff
y_flight2 = (sim_phase3[-1,1] + vy_liftoff*tvec_flight2
+ .5*g*(tvec_flight2**2) )
x_flight2 = sim_phase3[-1,0] + sim_phase3[-1,2]*tvec_flight2
vx_flight2 = sim_phase3[-1,2]*ones_like(tvec_flight2)
tvec_flight2 += t_phase3[-1]
# todo: return data until error
if sim_fail:
return { 't': None,
'x': None,
'y': None,
'vx': None,
'vy': None,
'sim_fail': sim_fail,
'dk': None,
'dL': None
}
# finally: concatenate phases
x_final = hstack([x_flight1, sim_phase2[:,0], sim_phase3[:,0], x_flight2 ])
y_final = hstack([y_flight1, sim_phase2[:,1], sim_phase3[:,1], y_flight2 ])
vx_final= hstack([vx_flight1, sim_phase2[:,2], sim_phase3[:,2], vx_flight2])
vy_final= hstack([vy_flight1, sim_phase2[:,3], sim_phase3[:,3], vy_flight2])
tvec_final = hstack([tvec_flight1, t_phase2, t_phase3, tvec_flight2 ])
return {'t': tvec_final,
'x': x_final,
'y': y_final,
'vx': vx_final,
'vy': vy_final,
'sim_fail': sim_fail,
'dk': dk,
'dL': dL,
#'sim_res':sim_res,
#'sim_phase2': sim_phase2_cut,
#'t_phase2': t_phase2_cut
}
#segf1 = Segment(0.0, 1.0, lambda x: exp(-1/x))
#segf2 = Segment(0.0, 0.5, lambda x:-1/log(x))
#figure()
#print estimateDegreeOfZero(lambda x: x**0.5, 0)
#n=7.0
#estimateDegreeOfZero(lambda x:(n+1)/(n) * x ** (1/n), 0)
#estimateDegreeOfZero(lambda x:pi/2 * sqrt(1 - (x-1) ** 2), 0)
#print estimateDegreeOfZero(lambda x: exp(-1/x), 0)
# estimateDegreeOfZero(lambda x: 1/(x+x**4), Inf)
# estimateDegreeOfZero(lambda x: exp(-x), Inf)
#print findinv(lambda x: 1/(1+exp(-x)), a=-1e300, b=1e300, c=0.5, rtol =1e-16, maxiter = 10000)
from numpy import ones_like, zeros_like
def _pole_test(f, x, pos = True, deriv = False):
return str(testPole(f, x, pos, deriv = deriv)) + " " + str(estimateDegreeOfPole(f, x, pos)) + " " + str(estimateDegreeOfPole(f, x, pos, deriv = True))
print("0,", _pole_test(lambda x: ones_like(x), 0))
print("0',", _pole_test(lambda x: ones_like(x), 0, deriv = True))
print("1,", _pole_test(lambda x: zeros_like(x), 0))
print("x,", _pole_test(lambda x: x, 0))
print("x',", _pole_test(lambda x: x, 0, deriv = True))
print("x**1.5,", _pole_test(lambda x: x**1.5, 0))
print("x**0.5,", _pole_test(lambda x: x**0.5, 0))
print("-log(x),", _pole_test(lambda x: -log(x), 0))
print("-log(sqrt(x)),", _pole_test(lambda x: -log(sqrt(x)), 0))
print("-log(-x),", _pole_test(lambda x: -log(-x), 0, pos = False))
print("1+x**0.5,", _pole_test(lambda x: 1+x**0.5, 0))
print("(1+x**0.5)',", _pole_test(lambda x: 1+x**0.5, 0, deriv = True))
print("x*log(x),", _pole_test(lambda x: x*log(x), 0))
print(_pole_test(lambda x: 1+(1*x+7)*x**-2.5, 0))
print(testPole(lambda x: 1.0/abs(2*x-1), 0.5, pos= False))
print(testPole(lambda x: 9.0*abs(2*x-1), 0.5, pos= True))
print "diff(sin(2*x), x, 3) =", diff(sin(2 * x), x, 3) print '''2.10.3.3''' print "series(cos(x), x) =", series(cos(x), x) print "series(1/cos(x), x) =", series(1 / cos(x), x) # Series 설명을 위하여 그래프를 그림 # 그래프 그리기 관련 기능 등을 담고 있는 pylab 모듈을 불러 들임 import pylab x_deg = pylab.arange(-90, 90 + 1) x_rad = pylab.deg2rad(x_deg) y_cos = pylab.cos(x_rad) y_series_1 = 1 * pylab.ones_like(x_rad) y_series_2 = 1 - x_rad ** 2 / 2 y_series_3 = 1 - x_rad ** 2 / 2 + x_rad ** 4 / 24 pylab.plot(x_deg, y_cos, label='cos') pylab.plot(x_deg, y_series_1, label='series 1') pylab.plot(x_deg, y_series_2, label='series 2') pylab.plot(x_deg, y_series_3, label='series 3') pylab.grid() pylab.legend(loc=0) # pylab.show() print '''2.10.3.5''' print "integrate(6*x**5, x) =", integrate(6 * x ** 5, x) print "integrate(sin(x), x) =", integrate(sin(x), x)
def age_specific_rate(model,
data_type,
reference_area='all',
reference_sex='total',
reference_year='all',
mu_age=None,
mu_age_parent=None,
sigma_age_parent=None,
rate_type='neg_binom',
lower_bound=None,
interpolation_method='linear',
include_covariates=True,
zero_re=False):
# TODO: expose (and document) interface for alternative rate_type as well as other options,
# record reference values in the model
""" Generate PyMC objects for model of epidemological age-interval data
:Parameters:
- `model` : data.ModelData
- `data_type` : str, one of 'i', 'r', 'f', 'p', or 'pf'
- `reference_area, reference_sex, reference_year` : the node of the model to fit consistently
- `mu_age` : pymc.Node, will be used as the age pattern, set to None if not needed
- `mu_age_parent` : pymc.Node, will be used as the age pattern of the parent of the root area, set to None if not needed
- `sigma_age_parent` : pymc.Node, will be used as the standard deviation of the age pattern, set to None if not needed
- `rate_type` : str, optional. One of 'beta_binom', 'binom', 'log_normal_model', 'neg_binom', 'neg_binom_lower_bound_model', 'neg_binom_model', 'normal_model', 'offest_log_normal', or 'poisson'
- `lower_bound` :
- `interpolation_method` : str, optional, one of 'linear', 'nearest', 'zero', 'slinear', 'quadratic, or 'cubic'
- `include_covariates` : boolean
- `zero_re` : boolean, change one stoch from each set of siblings in area hierarchy to a 'sum to zero' deterministic
:Results:
- Returns dict of PyMC objects, including 'pi', the covariate adjusted predicted values for each row of data
"""
name = data_type
import data
result = data.ModelVars()
if (mu_age_parent != None and pl.any(pl.isnan(mu_age_parent))) \
or (sigma_age_parent != None and pl.any(pl.isnan(sigma_age_parent))):
mu_age_parent = None
sigma_age_parent = None
print 'WARNING: nan found in parent mu/sigma. Ignoring'
ages = pl.array(model.parameters['ages'])
data = model.get_data(data_type)
if lower_bound:
lb_data = model.get_data(lower_bound)
parameters = model.parameters.get(data_type, {})
area_hierarchy = model.hierarchy
vars = dismod3.data.ModelVars()
vars += dict(data=data)
if 'parameter_age_mesh' in parameters:
knots = pl.array(parameters['parameter_age_mesh'])
else:
knots = pl.arange(ages[0], ages[-1] + 1, 5)
smoothing_dict = {
'No Prior': pl.inf,
'Slightly': .5,
'Moderately': .05,
'Very': .005
}
if 'smoothness' in parameters:
smoothing = smoothing_dict[parameters['smoothness']['amount']]
else:
smoothing = 0.
if mu_age == None:
vars.update(
age_pattern.age_pattern(name,
ages=ages,
knots=knots,
smoothing=smoothing,
interpolation_method=interpolation_method))
else:
vars.update(dict(mu_age=mu_age, ages=ages))
vars.update(
expert_prior_model.level_constraints(name, parameters, vars['mu_age'],
ages))
vars.update(
expert_prior_model.derivative_constraints(name, parameters,
vars['mu_age'], ages))
if mu_age_parent != None:
# setup a hierarchical prior on the simliarity between the
# consistent estimate here and (inconsistent) estimate for its
# parent in the areas hierarchy
#weight_dict = {'Unusable': 10., 'Slightly': 10., 'Moderately': 1., 'Very': .1}
#weight = weight_dict[parameters['heterogeneity']]
vars.update(
similarity_prior_model.similar('parent_similarity_%s' % name,
vars['mu_age'], mu_age_parent,
sigma_age_parent, 0.))
# also use this as the initial value for the age pattern, if it is not already specified
if mu_age == None:
if isinstance(mu_age_parent, mc.Node): # TODO: test this code
initial_mu = mu_age_parent.value
else:
initial_mu = mu_age_parent
for i, k_i in enumerate(knots):
vars['gamma'][i].value = (pl.log(
initial_mu[k_i - ages[0]])).clip(-12, 6)
age_weights = pl.ones_like(
vars['mu_age'].value
) # TODO: use age pattern appropriate to the rate type
if len(data) > 0:
vars.update(
age_integrating_model.age_standardize_approx(
name, age_weights, vars['mu_age'], data['age_start'],
data['age_end'], ages))
# uncomment the following to effectively remove alleffects
#if 'random_effects' in parameters:
# for i in range(5):
# effect = 'sigma_alpha_%s_%d' % (name, i)
# parameters['random_effects'][effect] = dict(dist='TruncatedNormal', mu=.0001, sigma=.00001, lower=.00009, upper=.00011)
#if 'fixed_effects' in parameters:
# for effect in ['x_sex', 'x_LDI_id_Updated_7July2011']:
# parameters['fixed_effects'][effect] = dict(dist='normal', mu=.0001, sigma=.00001)
if include_covariates:
vars.update(
covariate_model.mean_covariate_model(name,
vars['mu_interval'],
data,
parameters,
model,
reference_area,
reference_sex,
reference_year,
zero_re=zero_re))
else:
vars.update({'pi': vars['mu_interval']})
## ensure that all data has uncertainty quantified appropriately
# first replace all missing se from ci
missing_se = pl.isnan(
data['standard_error']) | (data['standard_error'] < 0)
data['standard_error'][missing_se] = (data['upper_ci'][missing_se] -
data['lower_ci'][missing_se]) / (
2 * 1.96)
# then replace all missing ess with se
missing_ess = pl.isnan(data['effective_sample_size'])
data['effective_sample_size'][missing_ess] = data['value'][
missing_ess] * (1 - data['value'][missing_ess]
) / data['standard_error'][missing_ess]**2
if rate_type == 'neg_binom':
# warn and drop data that doesn't have effective sample size quantified, or is is non-positive
missing_ess = pl.isnan(data['effective_sample_size']) | (
data['effective_sample_size'] < 0)
if sum(missing_ess) > 0:
print 'WARNING: %d rows of %s data has invalid quantification of uncertainty.' % (
sum(missing_ess), name)
data['effective_sample_size'][missing_ess] = 0.0
# warn and change data where ess is unreasonably huge
large_ess = data['effective_sample_size'] >= 1.e10
if sum(large_ess) > 0:
print 'WARNING: %d rows of %s data have effective sample size exceeding 10 billion.' % (
sum(large_ess), name)
data['effective_sample_size'][large_ess] = 1.e10
if 'heterogeneity' in parameters:
lower_dict = {'Slightly': 9., 'Moderately': 3., 'Very': 1.}
lower = lower_dict[parameters['heterogeneity']]
else:
lower = 1.
# special case, treat pf data as poisson
if data_type == 'pf':
lower = 1.e12
vars.update(
covariate_model.dispersion_covariate_model(
name, data, lower, lower * 9.))
vars.update(
rate_model.neg_binom_model(name, vars['pi'], vars['delta'],
data['value'],
data['effective_sample_size']))
elif rate_type == 'log_normal':
# warn and drop data that doesn't have effective sample size quantified
missing = pl.isnan(
data['standard_error']) | (data['standard_error'] < 0)
if sum(missing) > 0:
print 'WARNING: %d rows of %s data has no quantification of uncertainty.' % (
sum(missing), name)
data['standard_error'][missing] = 1.e6
# TODO: allow options for alternative priors for sigma
vars['sigma'] = mc.Uniform('sigma_%s' % name,
lower=.0001,
upper=1.,
value=.01)
#vars['sigma'] = mc.Exponential('sigma_%s'%name, beta=100., value=.01)
vars.update(
rate_model.log_normal_model(name, vars['pi'], vars['sigma'],
data['value'],
data['standard_error']))
elif rate_type == 'normal':
# warn and drop data that doesn't have standard error quantified
missing = pl.isnan(
data['standard_error']) | (data['standard_error'] < 0)
if sum(missing) > 0:
print 'WARNING: %d rows of %s data has no quantification of uncertainty.' % (
sum(missing), name)
data['standard_error'][missing] = 1.e6
vars['sigma'] = mc.Uniform('sigma_%s' % name,
lower=.0001,
upper=.1,
value=.01)
vars.update(
rate_model.normal_model(name, vars['pi'], vars['sigma'],
data['value'], data['standard_error']))
elif rate_type == 'binom':
missing_ess = pl.isnan(data['effective_sample_size']) | (
data['effective_sample_size'] < 0)
if sum(missing_ess) > 0:
print 'WARNING: %d rows of %s data has invalid quantification of uncertainty.' % (
sum(missing_ess), name)
data['effective_sample_size'][missing_ess] = 0.0
vars += rate_model.binom(name, vars['pi'], data['value'],
data['effective_sample_size'])
elif rate_type == 'beta_binom':
vars += rate_model.beta_binom(name, vars['pi'], data['value'],
data['effective_sample_size'])
elif rate_type == 'poisson':
missing_ess = pl.isnan(data['effective_sample_size']) | (
data['effective_sample_size'] < 0)
if sum(missing_ess) > 0:
print 'WARNING: %d rows of %s data has invalid quantification of uncertainty.' % (
sum(missing_ess), name)
data['effective_sample_size'][missing_ess] = 0.0
vars += rate_model.poisson(name, vars['pi'], data['value'],
data['effective_sample_size'])
elif rate_type == 'offset_log_normal':
vars['sigma'] = mc.Uniform('sigma_%s' % name,
lower=.0001,
upper=10.,
value=.01)
vars += rate_model.offset_log_normal(name, vars['pi'],
vars['sigma'], data['value'],
data['standard_error'])
else:
raise Exception, 'rate_model "%s" not implemented' % rate_type
else:
if include_covariates:
vars.update(
covariate_model.mean_covariate_model(name, [],
data,
parameters,
model,
reference_area,
reference_sex,
reference_year,
zero_re=zero_re))
if include_covariates:
vars.update(
expert_prior_model.covariate_level_constraints(
name, model, vars, ages))
if lower_bound and len(lb_data) > 0:
vars['lb'] = age_integrating_model.age_standardize_approx(
'lb_%s' % name, age_weights, vars['mu_age'], lb_data['age_start'],
lb_data['age_end'], ages)
if include_covariates:
vars['lb'].update(
covariate_model.mean_covariate_model('lb_%s' % name,
vars['lb']['mu_interval'],
lb_data,
parameters,
model,
reference_area,
reference_sex,
reference_year,
zero_re=zero_re))
else:
vars['lb'].update({'pi': vars['lb']['mu_interval']})
vars['lb'].update(
covariate_model.dispersion_covariate_model(
'lb_%s' % name, lb_data, 1e12, 1e13) # treat like poisson
)
## ensure that all data has uncertainty quantified appropriately
# first replace all missing se from ci
missing_se = pl.isnan(
lb_data['standard_error']) | (lb_data['standard_error'] <= 0)
lb_data['standard_error'][missing_se] = (
lb_data['upper_ci'][missing_se] -
lb_data['lower_ci'][missing_se]) / (2 * 1.96)
# then replace all missing ess with se
missing_ess = pl.isnan(lb_data['effective_sample_size'])
lb_data['effective_sample_size'][missing_ess] = lb_data['value'][
missing_ess] * (1 - lb_data['value'][missing_ess]
) / lb_data['standard_error'][missing_ess]**2
# warn and drop lb_data that doesn't have effective sample size quantified
missing_ess = pl.isnan(lb_data['effective_sample_size']) | (
lb_data['effective_sample_size'] <= 0)
if sum(missing_ess) > 0:
print 'WARNING: %d rows of %s lower bound data has no quantification of uncertainty.' % (
sum(missing_ess), name)
lb_data['effective_sample_size'][missing_ess] = 1.0
vars['lb'].update(
rate_model.neg_binom_lower_bound_model(
'lb_%s' % name, vars['lb']['pi'], vars['lb']['delta'],
lb_data['value'], lb_data['effective_sample_size']))
result[data_type] = vars
return result
dm.params['region_effect_prevalence'] = dict(std=level)
dismod3.neg_binom_model.fit_emp_prior(dm, 'prevalence', map_only=True)
models[-1].append(dm)
models.append([])
for level in [.1, 1, 10.]:
dm = initialize_model()
dm.params['beta_effect_prevalence'] = dict(mean=[0.], std=[level])
dismod3.neg_binom_model.fit_emp_prior(dm, 'prevalence', map_only=True)
models[-1].append(dm)
models.append([])
for level in [.2, 2., 20.]:
dm = initialize_model()
dm.params['gamma_effect_prevalence'] = dict(mean=list(pl.zeros_like(dm.get_estimate_age_mesh())),
std=list(level*pl.ones_like(dm.get_estimate_age_mesh())))
dismod3.neg_binom_model.fit_emp_prior(dm, 'prevalence', map_only=True)
models[-1].append(dm)
# this should change uncertainty, although it turns out not to change levels
models.append([])
for level in [.025, .25, 2.5]:
dm = initialize_model()
dm.params['delta_effect_prevalence'] = dict(mean=3.,
std=level)
dismod3.neg_binom_model.fit_emp_prior(dm, 'prevalence', map_only=True)
models[-1].append(dm)
models.append([])
## setup lognormal prior on intercept
gamma = mc.Normal('gamma', 0.,
2.**-2,
value=pl.log(data[:,1].mean()))
mu = mc.Lambda('mu', lambda gamma=gamma: pl.exp(gamma))
## setup Gaussian Process prior on age pattern
@mc.deterministic
def M(mu=mu):
return mc.gp.Mean(lambda x: mu*pl.ones(len(x)))
C = mc.gp.FullRankCovariance(mc.gp.matern.euclidean,
amp=data[:,1].max(),
scale=scale[smoothness],
diff_degree=2)
sm = mc.gp.GPSubmodel('sm', M, C, in_mesh,
init_vals=mu.value*pl.ones_like(in_mesh))
@mc.deterministic
def f(f=sm.f_eval):
return dismod3.utils.interpolate(in_mesh, f, out_mesh, kind=kind)
## condition on rate being positive
@mc.potential
def positive(f=f):
if pl.any(f < 0.):
return -pl.inf
else:
return 0.
## likelihood of observed data, using normal model for simplicity
@mc.deterministic
def Delta(x, tolerance):
"""Approximates delta function for plotting: Equals one when x is within tolerance of zero."""
y = py.ones_like(x)
y[x > tolerance] = 0
y[x <= -tolerance] = 0
return y
# +-----+
#
# In equations this represents
# $$S' = b-m,$$
# $$b = h_b S,$$
# $$m = h_m S.$$
#
# And then if $h_b$ and $h_m$ are constants, the closed form solution is
# $$S = S_0 e^{(h_b-h_m)t}.$$
# <codecell>
t = 1. * pl.arange(101)
S_0 = 1.
h_b = .02 * pl.ones_like(t)
h_m = .01 * pl.ones_like(t)
# <codecell>
def one_compartment_ode(S, t, h_b, h_m):
# piecewise-constant functions of time implementend as array
t = int(pl.clip(t, 0, len(h_b) - 1))
return (h_b[t] - h_m[t]) * S
# <markdowncell>
# SciPy has two wrappers for the odepack solver, <code>scipy.integrate.odeint</code> (function-based) and <code>scipy.integrate.ode</code> (object-based).
#
def validate_ai_re(N=500, delta_true=.15, sigma_true=[.1,.1,.1,.1,.1], pi_true=quadratic, smoothness='Moderately', heterogeneity='Slightly'):
## generate simulated data
a = pl.arange(0, 101, 1)
pi_age_true = pi_true(a)
import dismod3
import simplejson as json
model = data.ModelData.from_gbd_jsons(json.loads(dismod3.disease_json.DiseaseJson().to_json()))
gbd_hierarchy = model.hierarchy
model = data_simulation.simple_model(N)
model.hierarchy = gbd_hierarchy
model.parameters['p']['parameter_age_mesh'] = range(0, 101, 10)
model.parameters['p']['smoothness'] = dict(amount=smoothness)
model.parameters['p']['heterogeneity'] = heterogeneity
age_start = pl.array(mc.runiform(0, 100, size=N), dtype=int)
age_end = pl.array(mc.runiform(age_start, 100, size=N), dtype=int)
age_weights = pl.ones_like(a)
sum_pi_wt = pl.cumsum(pi_age_true*age_weights)
sum_wt = pl.cumsum(age_weights*1.)
p = (sum_pi_wt[age_end] - sum_pi_wt[age_start]) / (sum_wt[age_end] - sum_wt[age_start])
# correct cases where age_start == age_end
i = age_start == age_end
if pl.any(i):
p[i] = pi_age_true[age_start[i]]
model.input_data['age_start'] = age_start
model.input_data['age_end'] = age_end
model.input_data['effective_sample_size'] = mc.runiform(100, 10000, size=N)
from validate_covariates import alpha_true_sim
area_list = pl.array(['all', 'super-region_3', 'north_africa_middle_east', 'EGY', 'KWT', 'IRN', 'IRQ', 'JOR', 'SYR'])
alpha = alpha_true_sim(model, area_list, sigma_true)
print alpha
model.input_data['true'] = pl.nan
model.input_data['area'] = area_list[mc.rcategorical(pl.ones(len(area_list)) / float(len(area_list)), N)]
for i, a in model.input_data['area'].iteritems():
model.input_data['true'][i] = p[i] * pl.exp(pl.sum([alpha[n] for n in nx.shortest_path(model.hierarchy, 'all', a) if n in alpha]))
p = model.input_data['true']
n = model.input_data['effective_sample_size']
model.input_data['value'] = mc.rnegative_binomial(n*p, delta_true*n*p) / n
## Then fit the model and compare the estimates to the truth
model.vars = {}
model.vars['p'] = data_model.data_model('p', model, 'p', 'north_africa_middle_east', 'total', 'all', None, None, None)
#model.map, model.mcmc = fit_model.fit_data_model(model.vars['p'], iter=1005, burn=500, thin=5, tune_interval=100)
model.map, model.mcmc = fit_model.fit_data_model(model.vars['p'], iter=10000, burn=5000, thin=25, tune_interval=100)
graphics.plot_one_ppc(model.vars['p'], 'p')
graphics.plot_convergence_diag(model.vars)
graphics.plot_one_type(model, model.vars['p'], {}, 'p')
pl.plot(range(101), pi_age_true, 'r:', label='Truth')
pl.legend(fancybox=True, shadow=True, loc='upper left')
pl.show()
model.input_data['mu_pred'] = model.vars['p']['p_pred'].stats()['mean']
model.input_data['sigma_pred'] = model.vars['p']['p_pred'].stats()['standard deviation']
data_simulation.add_quality_metrics(model.input_data)
model.delta = pandas.DataFrame(dict(true=[delta_true]))
model.delta['mu_pred'] = pl.exp(model.vars['p']['eta'].trace()).mean()
model.delta['sigma_pred'] = pl.exp(model.vars['p']['eta'].trace()).std()
data_simulation.add_quality_metrics(model.delta)
model.alpha = pandas.DataFrame(index=[n for n in nx.traversal.dfs_preorder_nodes(model.hierarchy)])
model.alpha['true'] = pandas.Series(dict(alpha))
model.alpha['mu_pred'] = pandas.Series([n.stats()['mean'] for n in model.vars['p']['alpha']], index=model.vars['p']['U'].columns)
model.alpha['sigma_pred'] = pandas.Series([n.stats()['standard deviation'] for n in model.vars['p']['alpha']], index=model.vars['p']['U'].columns)
model.alpha = model.alpha.dropna()
data_simulation.add_quality_metrics(model.alpha)
model.sigma = pandas.DataFrame(dict(true=sigma_true))
model.sigma['mu_pred'] = [n.stats()['mean'] for n in model.vars['p']['sigma_alpha']]
model.sigma['sigma_pred']=[n.stats()['standard deviation'] for n in model.vars['p']['sigma_alpha']]
data_simulation.add_quality_metrics(model.sigma)
print 'delta'
print model.delta
print '\ndata prediction bias: %.5f, MARE: %.3f, coverage: %.2f' % (model.input_data['abs_err'].mean(),
pl.median(pl.absolute(model.input_data['rel_err'].dropna())),
model.input_data['covered?'].mean())
model.mu = pandas.DataFrame(dict(true=pi_age_true,
mu_pred=model.vars['p']['mu_age'].stats()['mean'],
sigma_pred=model.vars['p']['mu_age'].stats()['standard deviation']))
data_simulation.add_quality_metrics(model.mu)
data_simulation.initialize_results(model)
data_simulation.add_to_results(model, 'delta')
data_simulation.add_to_results(model, 'mu')
data_simulation.add_to_results(model, 'input_data')
data_simulation.add_to_results(model, 'alpha')
data_simulation.add_to_results(model, 'sigma')
data_simulation.finalize_results(model)
print model.results
return model
print "diff(tan(x), x) = ", diff(tan(x), x)
print "limit((tan(x+y)-tan(x))/y,y,0) =", limit((tan(x + y) - tan(x)) / y, y,
0)
print "diff(sin(2*x), x, 1) =", diff(sin(2 * x), x, 1)
print "diff(sin(2*x), x, 2) =", diff(sin(2 * x), x, 2)
print "diff(sin(2*x), x, 3) =", diff(sin(2 * x), x, 3)
print '''2.10.3.3'''
print "series(cos(x), x) =", series(cos(x), x)
print "series(1/cos(x), x) = ", series(1 / cos(x), x)
import pylab
x_deg = pylab.arange(-90, 90 + 1)
x_rad = pylab.deg2rad(x_deg)
y_cos = pylab.cos(x_rad)
y_series_1 = 1 * pylab.ones_like(x_rad)
y_series_2 = 1 - x_rad**2 / 2
y_series_3 = 1 - x_rad**2 / 2 + x_rad**4 / 24
pylab.plot(x_deg, y_cos, label='cos')
pylab.plot(x_deg, y_series_1, label='series 1')
pylab.plot(x_deg, y_series_2, label='series 2')
pylab.plot(x_deg, y_series_3, label='series 3')
pylab.grid()
pylab.legend(loc=0)
#pylab.show()
print '''2.10.3.5'''
print "integrate(6*x**5, x) =", integrate(6 * x**5, x)
print "integrate(sin(x), x) =", integrate(sin(x), x)
print "integrate(log(x), x) =", integrate(log(x), x)
# +-----+
#
# In equations this represents
# $$S' = b-m,$$
# $$b = h_b S,$$
# $$m = h_m S.$$
#
# And then if $h_b$ and $h_m$ are constants, the closed form solution is
# $$S = S_0 e^{(h_b-h_m)t}.$$
# <codecell>
t = 1.*pl.arange(101)
S_0 = 1.
h_b = .02*pl.ones_like(t)
h_m = .01*pl.ones_like(t)
# <codecell>
def one_compartment_ode(S, t, h_b, h_m):
# piecewise-constant functions of time implementend as array
t = int(pl.clip(t, 0, len(h_b)-1))
return (h_b[t]-h_m[t])*S
# <markdowncell>
# SciPy has two wrappers for the odepack solver, <code>scipy.integrate.odeint</code> (function-based) and <code>scipy.integrate.ode</code> (object-based).
#
# The function-based wrapper has less overhead, so I'll use it here. The object-based wrapper has a nicer format for some applications.
dm.data = [d for d in dm.data if dm.relevant_to(d, 'all-cause_mortality', region, year, sex)] + \
[d for d in dm.data if dm.relevant_to(d, 'prevalence_x_excess-mortality', region, year, sex)] + \
[d for d in dm.data if d.get('country_iso3_code') == 'GBR']
dm.params['global_priors']['level_bounds']['excess_mortality'] = dict(lower=.2, upper=10.)
fit_posterior.fit_posterior(dm, region, sex, year, map_only=True, store_results=False)
models.append(dm)
dm = initialize_model()
dm.description = 'With excess-mortality data and priors removed'
dm.data = [d for d in dm.data if dm.relevant_to(d, 'all-cause_mortality', region, year, sex)] + \
[d for d in dm.data if dm.relevant_to(d, 'prevalence_x_excess-mortality', region, year, sex)] + \
[d for d in dm.data if d.get('country_iso3_code') == 'GBR']
dm.params['global_priors']['level_bounds']['excess_mortality'] = dict(lower=0., upper=10.)
dm.params['global_priors']['smoothness']['prevalence']['amount'] = 'No Prior'
dm.params['gamma_effect_excess-mortality'] = dict(mean=list(pl.zeros_like(dm.get_estimate_age_mesh())),
std=list(10.*pl.ones_like(dm.get_estimate_age_mesh())))
fit_posterior.fit_posterior(dm, region, sex, year, map_only=True, store_results=False)
models.append(dm)
dm = initialize_model()
dm.description = 'Without increasing prior on excess-mortality, but with "Very" smoothing'
dm.data = [d for d in dm.data if dm.relevant_to(d, 'all-cause_mortality', region, year, sex)] + \
[d for d in dm.data if dm.relevant_to(d, 'prevalence_x_excess-mortality', region, year, sex)] + \
[d for d in dm.data if d.get('country_iso3_code') == 'GBR'] + \
[d for d in dm.data if dm.relevant_to(d, 'excess-mortality', 'all', 'all', 'all')]
dm.params['global_priors']['smoothness']['prevalence']['amount'] = 'Very'
fit_posterior.fit_posterior(dm, region, sex, year, map_only=True, store_results=False)
models.append(dm)
dm = initialize_model()
def delta(eta=eta):
return pl.exp(eta) * pl.ones_like(input_data.index)
dismod3.neg_binom_model.fit_emp_prior(dm, 'prevalence', map_only=True)
models[-1].append(dm)
models.append([])
for level in [.1, 1, 10.]:
dm = initialize_model()
dm.params['beta_effect_prevalence'] = dict(mean=[0.], std=[level])
dismod3.neg_binom_model.fit_emp_prior(dm, 'prevalence', map_only=True)
models[-1].append(dm)
models.append([])
for level in [.2, 2., 20.]:
dm = initialize_model()
dm.params['gamma_effect_prevalence'] = dict(
mean=list(pl.zeros_like(dm.get_estimate_age_mesh())),
std=list(level * pl.ones_like(dm.get_estimate_age_mesh())))
dismod3.neg_binom_model.fit_emp_prior(dm, 'prevalence', map_only=True)
models[-1].append(dm)
# this should change uncertainty, although it turns out not to change levels
models.append([])
for level in [.025, .25, 2.5]:
dm = initialize_model()
dm.params['delta_effect_prevalence'] = dict(mean=3., std=level)
dismod3.neg_binom_model.fit_emp_prior(dm, 'prevalence', map_only=True)
models[-1].append(dm)
models.append([])
for level in [1, 2, 3]:
dm = initialize_model()
dm.params['delta_effect_prevalence'] = dict(mean=level, std=.25)
pl.figure(figsize=(11, 8.5), dpi=120)
pl.subplots_adjust(wspace=0.4)
pl.subplot(2, 2, 1)
plot_funnel(0.004, "5")
pl.subplot(2, 2, 2)
plot_funnel(0.004, "50")
pl.subplot(2, 1, 2)
r = pl.array(schiz["r"])
n = pl.array(schiz["n"])
pi = mc.Uniform("pi", 0, 1, value=0.001)
delta = mc.Uniform("delta", 0, 100, value=0.0001)
nb = rate_model.neg_binom_model("funnel", pi * pl.ones_like(n), delta * pl.ones_like(n), r, n)
# old way:
# nb = rate_model.neg_binom_model('funnel', pi, delta*r*n, r, n)
mcmc = mc.MCMC([pi, delta, nb])
mcmc.sample(20000, 10000, 10, verbose=False, progress_bar=False)
sorted_indices = r.argsort().argsort()
jitter = mc.rnormal(0, 0.1 ** -2, len(nb["p_pred"].trace()))
for i, s_i in enumerate(sorted_indices):
pl.plot(s_i + jitter, nb["p_pred"].trace()[:, i], "ko", mew=0, alpha=0.25, zorder=-99)
pl.errorbar(
sorted_indices, r, yerr=1.96 * pl.sqrt(r * (1 - r) / n), fmt="ws", mew=1, mec="white", ms=5, elinewidth=3, capsize=0
)
pl.errorbar(sorted_indices, r, yerr=1.96 * pl.sqrt(r * (1 - r) / n), fmt="ks", mew=1, mec="white", ms=5)
def set_birth_prev(value):
model.vars['logit_C0'].value = mc.logit(pl.maximum(1.e-9, value))
### @export 'congenital'
set_rate('remission', pl.zeros_like(ages))
set_rate('incidence', pl.zeros_like(ages))
set_birth_prev(.2)
set_rate('excess-mortality', .5*(ages/100.)**2)
book_graphics.plot_age_patterns(model, yticks=[0,.2,.4], xticks=[0,25,50,75,100], panel='b')
pl.savefig('book/graphics/forward-sim-congenital.pdf')
### @export 'mental'
set_rate('remission', pl.ones_like(ages)*.25)
set_rate('incidence', pl.where((ages > 14) & (ages < 100), (100-ages)*.001, 0.))
set_birth_prev(0)
set_rate('excess-mortality', 1e-4*pl.ones_like(ages))
book_graphics.plot_age_patterns(model, yticks=[0,.2,.4], panel='a')
pl.savefig('book/graphics/forward-sim-mental.pdf')
### @export 'old_age'
set_rate('remission', pl.ones_like(ages)*0.)
set_rate('incidence', pl.where(ages > 50, pl.exp((ages-50.)/25.)*.01, 0.))
set_birth_prev(0)
set_rate('excess-mortality', pl.exp(ages/25.)*.01)
book_graphics.plot_age_patterns(model, yticks=[0,.2,.4], panel='c')
def age_specific_rate(
model,
data_type,
reference_area="all",
reference_sex="total",
reference_year="all",
mu_age=None,
mu_age_parent=None,
sigma_age_parent=None,
rate_type="neg_binom",
lower_bound=None,
interpolation_method="linear",
include_covariates=True,
zero_re=False,
):
# TODO: expose (and document) interface for alternative rate_type as well as other options,
# record reference values in the model
""" Generate PyMC objects for model of epidemological age-interval data
:Parameters:
- `model` : data.ModelData
- `data_type` : str, one of 'i', 'r', 'f', 'p', or 'pf'
- `reference_area, reference_sex, reference_year` : the node of the model to fit consistently
- `mu_age` : pymc.Node, will be used as the age pattern, set to None if not needed
- `mu_age_parent` : pymc.Node, will be used as the age pattern of the parent of the root area, set to None if not needed
- `sigma_age_parent` : pymc.Node, will be used as the standard deviation of the age pattern, set to None if not needed
- `rate_type` : str, optional. One of 'beta_binom', 'binom', 'log_normal_model', 'neg_binom', 'neg_binom_lower_bound_model', 'neg_binom_model', 'normal_model', 'offest_log_normal', or 'poisson'
- `lower_bound` :
- `interpolation_method` : str, optional, one of 'linear', 'nearest', 'zero', 'slinear', 'quadratic, or 'cubic'
- `include_covariates` : boolean
- `zero_re` : boolean, change one stoch from each set of siblings in area hierarchy to a 'sum to zero' deterministic
:Results:
- Returns dict of PyMC objects, including 'pi', the covariate adjusted predicted values for each row of data
"""
name = data_type
import data
result = data.ModelVars()
if (mu_age_parent != None and pl.any(pl.isnan(mu_age_parent))) or (
sigma_age_parent != None and pl.any(pl.isnan(sigma_age_parent))
):
mu_age_parent = None
sigma_age_parent = None
print "WARNING: nan found in parent mu/sigma. Ignoring"
ages = pl.array(model.parameters["ages"])
data = model.get_data(data_type)
if lower_bound:
lb_data = model.get_data(lower_bound)
parameters = model.parameters.get(data_type, {})
area_hierarchy = model.hierarchy
vars = dismod3.data.ModelVars()
vars += dict(data=data)
if "parameter_age_mesh" in parameters:
knots = pl.array(parameters["parameter_age_mesh"])
else:
knots = pl.arange(ages[0], ages[-1] + 1, 5)
smoothing_dict = {"No Prior": pl.inf, "Slightly": 0.5, "Moderately": 0.05, "Very": 0.005}
if "smoothness" in parameters:
smoothing = smoothing_dict[parameters["smoothness"]["amount"]]
else:
smoothing = 0.0
if mu_age == None:
vars.update(
age_pattern.age_pattern(
name, ages=ages, knots=knots, smoothing=smoothing, interpolation_method=interpolation_method
)
)
else:
vars.update(dict(mu_age=mu_age, ages=ages))
vars.update(expert_prior_model.level_constraints(name, parameters, vars["mu_age"], ages))
vars.update(expert_prior_model.derivative_constraints(name, parameters, vars["mu_age"], ages))
if mu_age_parent != None:
# setup a hierarchical prior on the simliarity between the
# consistent estimate here and (inconsistent) estimate for its
# parent in the areas hierarchy
# weight_dict = {'Unusable': 10., 'Slightly': 10., 'Moderately': 1., 'Very': .1}
# weight = weight_dict[parameters['heterogeneity']]
vars.update(
similarity_prior_model.similar(
"parent_similarity_%s" % name, vars["mu_age"], mu_age_parent, sigma_age_parent, 0.0
)
)
# also use this as the initial value for the age pattern, if it is not already specified
if mu_age == None:
if isinstance(mu_age_parent, mc.Node): # TODO: test this code
initial_mu = mu_age_parent.value
else:
initial_mu = mu_age_parent
for i, k_i in enumerate(knots):
vars["gamma"][i].value = (pl.log(initial_mu[k_i - ages[0]])).clip(-12, 6)
age_weights = pl.ones_like(vars["mu_age"].value) # TODO: use age pattern appropriate to the rate type
if len(data) > 0:
vars.update(
age_integrating_model.age_standardize_approx(
name, age_weights, vars["mu_age"], data["age_start"], data["age_end"], ages
)
)
# uncomment the following to effectively remove alleffects
# if 'random_effects' in parameters:
# for i in range(5):
# effect = 'sigma_alpha_%s_%d' % (name, i)
# parameters['random_effects'][effect] = dict(dist='TruncatedNormal', mu=.0001, sigma=.00001, lower=.00009, upper=.00011)
# if 'fixed_effects' in parameters:
# for effect in ['x_sex', 'x_LDI_id_Updated_7July2011']:
# parameters['fixed_effects'][effect] = dict(dist='normal', mu=.0001, sigma=.00001)
if include_covariates:
vars.update(
covariate_model.mean_covariate_model(
name,
vars["mu_interval"],
data,
parameters,
model,
reference_area,
reference_sex,
reference_year,
zero_re=zero_re,
)
)
else:
vars.update({"pi": vars["mu_interval"]})
## ensure that all data has uncertainty quantified appropriately
# first replace all missing se from ci
missing_se = pl.isnan(data["standard_error"]) | (data["standard_error"] < 0)
data["standard_error"][missing_se] = (data["upper_ci"][missing_se] - data["lower_ci"][missing_se]) / (2 * 1.96)
# then replace all missing ess with se
missing_ess = pl.isnan(data["effective_sample_size"])
data["effective_sample_size"][missing_ess] = (
data["value"][missing_ess] * (1 - data["value"][missing_ess]) / data["standard_error"][missing_ess] ** 2
)
if rate_type == "neg_binom":
# warn and drop data that doesn't have effective sample size quantified, or is is non-positive
missing_ess = pl.isnan(data["effective_sample_size"]) | (data["effective_sample_size"] < 0)
if sum(missing_ess) > 0:
print "WARNING: %d rows of %s data has invalid quantification of uncertainty." % (
sum(missing_ess),
name,
)
data["effective_sample_size"][missing_ess] = 0.0
# warn and change data where ess is unreasonably huge
large_ess = data["effective_sample_size"] >= 1.0e10
if sum(large_ess) > 0:
print "WARNING: %d rows of %s data have effective sample size exceeding 10 billion." % (
sum(large_ess),
name,
)
data["effective_sample_size"][large_ess] = 1.0e10
if "heterogeneity" in parameters:
lower_dict = {"Slightly": 9.0, "Moderately": 3.0, "Very": 1.0}
lower = lower_dict[parameters["heterogeneity"]]
else:
lower = 1.0
# special case, treat pf data as poisson
if data_type == "pf":
lower = 1.0e12
vars.update(covariate_model.dispersion_covariate_model(name, data, lower, lower * 9.0))
vars.update(
rate_model.neg_binom_model(
name, vars["pi"], vars["delta"], data["value"], data["effective_sample_size"]
)
)
elif rate_type == "log_normal":
# warn and drop data that doesn't have effective sample size quantified
missing = pl.isnan(data["standard_error"]) | (data["standard_error"] < 0)
if sum(missing) > 0:
print "WARNING: %d rows of %s data has no quantification of uncertainty." % (sum(missing), name)
data["standard_error"][missing] = 1.0e6
# TODO: allow options for alternative priors for sigma
vars["sigma"] = mc.Uniform("sigma_%s" % name, lower=0.0001, upper=1.0, value=0.01)
# vars['sigma'] = mc.Exponential('sigma_%s'%name, beta=100., value=.01)
vars.update(
rate_model.log_normal_model(name, vars["pi"], vars["sigma"], data["value"], data["standard_error"])
)
elif rate_type == "normal":
# warn and drop data that doesn't have standard error quantified
missing = pl.isnan(data["standard_error"]) | (data["standard_error"] < 0)
if sum(missing) > 0:
print "WARNING: %d rows of %s data has no quantification of uncertainty." % (sum(missing), name)
data["standard_error"][missing] = 1.0e6
vars["sigma"] = mc.Uniform("sigma_%s" % name, lower=0.0001, upper=0.1, value=0.01)
vars.update(rate_model.normal_model(name, vars["pi"], vars["sigma"], data["value"], data["standard_error"]))
elif rate_type == "binom":
missing_ess = pl.isnan(data["effective_sample_size"]) | (data["effective_sample_size"] < 0)
if sum(missing_ess) > 0:
print "WARNING: %d rows of %s data has invalid quantification of uncertainty." % (
sum(missing_ess),
name,
)
data["effective_sample_size"][missing_ess] = 0.0
vars += rate_model.binom(name, vars["pi"], data["value"], data["effective_sample_size"])
elif rate_type == "beta_binom":
vars += rate_model.beta_binom(name, vars["pi"], data["value"], data["effective_sample_size"])
elif rate_type == "poisson":
missing_ess = pl.isnan(data["effective_sample_size"]) | (data["effective_sample_size"] < 0)
if sum(missing_ess) > 0:
print "WARNING: %d rows of %s data has invalid quantification of uncertainty." % (
sum(missing_ess),
name,
)
data["effective_sample_size"][missing_ess] = 0.0
vars += rate_model.poisson(name, vars["pi"], data["value"], data["effective_sample_size"])
elif rate_type == "offset_log_normal":
vars["sigma"] = mc.Uniform("sigma_%s" % name, lower=0.0001, upper=10.0, value=0.01)
vars += rate_model.offset_log_normal(name, vars["pi"], vars["sigma"], data["value"], data["standard_error"])
else:
raise Exception, 'rate_model "%s" not implemented' % rate_type
else:
if include_covariates:
vars.update(
covariate_model.mean_covariate_model(
name, [], data, parameters, model, reference_area, reference_sex, reference_year, zero_re=zero_re
)
)
if include_covariates:
vars.update(expert_prior_model.covariate_level_constraints(name, model, vars, ages))
if lower_bound and len(lb_data) > 0:
vars["lb"] = age_integrating_model.age_standardize_approx(
"lb_%s" % name, age_weights, vars["mu_age"], lb_data["age_start"], lb_data["age_end"], ages
)
if include_covariates:
vars["lb"].update(
covariate_model.mean_covariate_model(
"lb_%s" % name,
vars["lb"]["mu_interval"],
lb_data,
parameters,
model,
reference_area,
reference_sex,
reference_year,
zero_re=zero_re,
)
)
else:
vars["lb"].update({"pi": vars["lb"]["mu_interval"]})
vars["lb"].update(
covariate_model.dispersion_covariate_model("lb_%s" % name, lb_data, 1e12, 1e13) # treat like poisson
)
## ensure that all data has uncertainty quantified appropriately
# first replace all missing se from ci
missing_se = pl.isnan(lb_data["standard_error"]) | (lb_data["standard_error"] <= 0)
lb_data["standard_error"][missing_se] = (lb_data["upper_ci"][missing_se] - lb_data["lower_ci"][missing_se]) / (
2 * 1.96
)
# then replace all missing ess with se
missing_ess = pl.isnan(lb_data["effective_sample_size"])
lb_data["effective_sample_size"][missing_ess] = (
lb_data["value"][missing_ess]
* (1 - lb_data["value"][missing_ess])
/ lb_data["standard_error"][missing_ess] ** 2
)
# warn and drop lb_data that doesn't have effective sample size quantified
missing_ess = pl.isnan(lb_data["effective_sample_size"]) | (lb_data["effective_sample_size"] <= 0)
if sum(missing_ess) > 0:
print "WARNING: %d rows of %s lower bound data has no quantification of uncertainty." % (
sum(missing_ess),
name,
)
lb_data["effective_sample_size"][missing_ess] = 1.0
vars["lb"].update(
rate_model.neg_binom_lower_bound_model(
"lb_%s" % name,
vars["lb"]["pi"],
vars["lb"]["delta"],
lb_data["value"],
lb_data["effective_sample_size"],
)
)
result[data_type] = vars
return result
def delta(eta=eta):
return pl.exp(eta) * pl.ones_like(input_data.index)
