def test_bad_model(self):
X = data.sim_data(10)
Y = models.bad_model(X)
assert pl.allclose(Y.sum(axis=2), 1), 'should be all ones, (%s found)' % str(Y.sum(axis=2))
# test again for 10x2x3 dataset
X = data.sim_data(10, [[.1, .4, .5]], [.1, .1, .1])
Y = models.bad_model(X)
assert pl.allclose(Y.sum(axis=2), 1), 'should be all ones, (%s found)' % str(Y.sum(axis=2))
def test_bad_model(self):
X = data.sim_data(10)
Y = models.bad_model(X)
assert pl.allclose(
Y.sum(axis=2),
1), 'should be all ones, (%s found)' % str(Y.sum(axis=2))
# test again for 10x2x3 dataset
X = data.sim_data(10, [[.1, .4, .5]], [.1, .1, .1])
Y = models.bad_model(X)
assert pl.allclose(
Y.sum(axis=2),
1), 'should be all ones, (%s found)' % str(Y.sum(axis=2))
def find_data(items, values):
for subdir, dirs, files in os.walk("hole-results-old/"):
if not os.path.isfile(subdir + "/parameters.xml"):
continue
# Extract the parameters used
param = Parameters()
File(subdir + "/parameters.xml") >> param
# It matched, extract information
if pl.allclose(pl.array([eval("param." + item) for item in items]),
pl.array(values)):
data = pl.loadtxt(subdir + "/energies.txt")
t = data[:, 0]
S = data[:, 2]
# Umtimate stress
tf = (t[-1] + t[-2]) / 2
# Elastic limit
ind = pl.find(S > 0)[0]
te = (t[ind] + t[ind - 1]) / 2
return te, tf
def sim_data(N,
true_cf=[[.3, .6, .1], [.3, .5, .2]],
true_std=[[.2, .05, .05], [.3, 0.1, 0.1]],
sum_to_one=True):
"""
Create an NxTxJ matrix of simulated data (T is determined by the length
of true_cf, J by the length of the elements of true_cf).
true_cf - a list of lists of true cause fractions (each must sum to one)
true_std - a list of lists of the standard deviations corresponding to the true csmf's
for each time point. Can either be a list of length J inside a list of length
1 (in this case, the same standard deviation is used for all time points) or
can be T lists of length J (in this case, the a separate standard deviation
is specified and used for each time point).
"""
if sum_to_one == True:
assert pl.allclose(pl.sum(true_cf, 1),
1), 'The sum of elements of true_cf must equal 1'
T = len(true_cf)
J = len(true_cf[0])
## if only one std provided, duplicate for all time points
if len(true_std) == 1 and len(true_cf) > 1:
true_std = [true_std[0] for i in range(len(true_cf))]
## transform the mean and std to logit space
transformed_std = []
for t in range(T):
pi_i = pl.array(true_cf[t])
sigma_pi_i = pl.array(true_std[t])
transformed_std.append(
((1 / (pi_i * (pi_i - 1)))**2 * sigma_pi_i**2)**0.5)
## find minimum standard deviation (by cause across time) and draw from this
min = pl.array(transformed_std).min(0)
common_perturbation = [
pl.ones([T, J]) * mc.rnormal(mu=0, tau=min**-2) for n in range(N)
]
## draw from remaining variation
tau = pl.array(transformed_std)**2 - min**2
tau[tau == 0] = 0.000001
additional_perturbation = [
[mc.rnormal(mu=0, tau=tau[t]**-1) for t in range(T)] for n in range(N)
]
result = pl.zeros([N, T, J])
for n in range(N):
result[n, :, :] = [
mc.invlogit(
mc.logit(true_cf[t]) + common_perturbation[n][t] +
additional_perturbation[n][t]) for t in range(T)
]
return result
def test_sim_data_2(self):
sims = 10000
return # skip for now
test1 = pl.zeros(3, dtype='f').view(pl.recarray)
for i in range(sims):
temp = data.sim_data(1, [0.1,0.1,0.8], [0.01,0.01,0.01])
test1 = pl.vstack((test1, temp))
test1 = test1[1:,]
test2 = data.sim_data(sims, [0.1,0.1,0.8], [0.01, 0.01, 0.01])
diff = (test1.mean(0) - test2.mean(0))/test1.mean(0)
assert pl.allclose(diff, 0, atol=0.01), 'should be close to zero, (%s found)' % str(diff)
def test_sim_data_2(self):
sims = 10000
return # skip for now
test1 = pl.zeros(3, dtype='f').view(pl.recarray)
for i in range(sims):
temp = data.sim_data(1, [0.1, 0.1, 0.8], [0.01, 0.01, 0.01])
test1 = pl.vstack((test1, temp))
test1 = test1[1:, ]
test2 = data.sim_data(sims, [0.1, 0.1, 0.8], [0.01, 0.01, 0.01])
diff = (test1.mean(0) - test2.mean(0)) / test1.mean(0)
assert pl.allclose(
diff, 0,
atol=0.01), 'should be close to zero, (%s found)' % str(diff)
def sim_data(N, true_cf=[[.3, .6, .1],
[.3, .5, .2]],
true_std=[[.2, .05, .05],
[.3, 0.1, 0.1]],
sum_to_one=True):
"""
Create an NxTxJ matrix of simulated data (T is determined by the length
of true_cf, J by the length of the elements of true_cf).
true_cf - a list of lists of true cause fractions (each must sum to one)
true_std - a list of lists of the standard deviations corresponding to the true csmf's
for each time point. Can either be a list of length J inside a list of length
1 (in this case, the same standard deviation is used for all time points) or
can be T lists of length J (in this case, the a separate standard deviation
is specified and used for each time point).
"""
if sum_to_one == True:
assert pl.allclose(pl.sum(true_cf, 1), 1), 'The sum of elements of true_cf must equal 1'
T = len(true_cf)
J = len(true_cf[0])
## if only one std provided, duplicate for all time points
if len(true_std)==1 and len(true_cf)>1:
true_std = [true_std[0] for i in range(len(true_cf))]
## transform the mean and std to logit space
transformed_std = []
for t in range(T):
pi_i = pl.array(true_cf[t])
sigma_pi_i = pl.array(true_std[t])
transformed_std.append( ((1/(pi_i*(pi_i-1)))**2 * sigma_pi_i**2)**0.5 )
## find minimum standard deviation (by cause across time) and draw from this
min = pl.array(transformed_std).min(0)
common_perturbation = [pl.ones([T,J])*mc.rnormal(mu=0, tau=min**-2) for n in range(N)]
## draw from remaining variation
tau=pl.array(transformed_std)**2 - min**2
tau[tau==0] = 0.000001
additional_perturbation = [[mc.rnormal(mu=0, tau=tau[t]**-1) for t in range(T)] for n in range(N)]
result = pl.zeros([N, T, J])
for n in range(N):
result[n, :, :] = [mc.invlogit(mc.logit(true_cf[t]) + common_perturbation[n][t] + additional_perturbation[n][t]) for t in range(T)]
return result
import pylab as p
from matplotlib.toolkits.basemap import Basemap
import os
import Scientific.IO.NetCDF as S
#--- Get data (assuming lon goes from 0 to 360): Make arrays cyclic
# in longitude:
time_idx = 5 #@@@ USER ADJUSTABLE
f = S.NetCDFFile('qm_seasonal_1yr.nc', mode='r')
lat = f.variables['lat'].getValue()
lon = f.variables['lon'].getValue()
u1_all = f.variables['u1'].getValue()
if p.allclose(lon[0], 0):
u1 = p.zeros( (p.size(lat), p.size(lon)+1) )
u1[:,0:-1] = u1_all[time_idx,:,:]
u1[:,-1] = u1_all[time_idx,:,0]
tmp = copy.copy(lon)
lon = p.zeros((p.size(tmp)+1,))
lon[0:-1] = tmp[:]
lon[-1] = tmp[0]+360
del tmp
f.close()
#--- Mapping information:
def test_data_model_sim():
# generate simulated data
n = 50
sigma_true = .025
# start with truth
a = pl.arange(0, 100, 1)
pi_age_true = .0001 * (a * (100. - a) + 100.)
# choose age intervals to measure
age_start = pl.array(mc.runiform(0, 100, n), dtype=int)
age_start.sort() # sort to make it easy to discard the edges when testing
age_end = pl.array(mc.runiform(age_start+1, pl.minimum(age_start+10,100)), dtype=int)
# find truth for the integral across the age intervals
import scipy.integrate
pi_interval_true = [scipy.integrate.trapz(pi_age_true[a_0i:(a_1i+1)]) / (a_1i - a_0i)
for a_0i, a_1i in zip(age_start, age_end)]
# generate covariates that add explained variation
X = mc.rnormal(0., 1.**2, size=(n,3))
beta_true = [-.1, .1, .2]
Y_true = pl.dot(X, beta_true)
# calculate the true value of the rate in each interval
pi_true = pi_interval_true*pl.exp(Y_true)
# simulate the noisy measurement of the rate in each interval
p = mc.rnormal(pi_true, 1./sigma_true**2.)
# store the simulated data in a pandas DataFrame
data = pandas.DataFrame(dict(value=p, age_start=age_start, age_end=age_end,
x_0=X[:,0], x_1=X[:,1], x_2=X[:,2]))
data['effective_sample_size'] = pl.maximum(p*(1-p)/sigma_true**2, 1.)
data['standard_error'] = pl.nan
data['upper_ci'] = pl.nan
data['lower_ci'] = pl.nan
data['year_start'] = 2005. # TODO: make these vary
data['year_end'] = 2005.
data['sex'] = 'total'
data['area'] = 'all'
# generate a moderately complicated hierarchy graph for the model
hierarchy = nx.DiGraph()
hierarchy.add_node('all')
hierarchy.add_edge('all', 'super-region-1', weight=.1)
hierarchy.add_edge('super-region-1', 'NAHI', weight=.1)
hierarchy.add_edge('NAHI', 'CAN', weight=.1)
hierarchy.add_edge('NAHI', 'USA', weight=.1)
output_template=pandas.DataFrame(dict(year=[1990, 1990, 2005, 2005, 2010, 2010]*2,
sex=['male', 'female']*3*2,
x_0=[.5]*6*2,
x_1=[0.]*6*2,
x_2=[.5]*6*2,
pop=[50.]*6*2,
area=['CAN']*6 + ['USA']*6))
# create model and priors
vars = data_model.data_model('test', data, hierarchy, 'all')
# fit model
mc.MAP(vars).fit(method='fmin_powell', verbose=1)
m = mc.MCMC(vars)
m.use_step_method(mc.AdaptiveMetropolis, [m.gamma_bar, m.gamma, m.beta])
m.sample(30000, 15000, 15)
# check estimates
pi_usa = data_model.predict_for(output_template, hierarchy, 'all', 'USA', 'male', 1990, vars)
assert pl.allclose(pi_usa.mean(), (m.mu_age.trace()*pl.exp(.05)).mean(), rtol=.1)
# check convergence
print 'gamma mc error:', m.gamma_bar.stats()['mc error'].round(2), m.gamma.stats()['mc error'].round(2)
# plot results
for a_0i, a_1i, p_i in zip(age_start, age_end, p):
pl.plot([a_0i, a_1i], [p_i,p_i], 'rs-', mew=1, mec='w', ms=4)
pl.plot(a, pi_age_true, 'g-', linewidth=2)
pl.plot(pl.arange(101), m.mu_age.stats()['mean'], 'k-', drawstyle='steps-post', linewidth=3)
pl.plot(pl.arange(101), m.mu_age.stats()['95% HPD interval'], 'k', linestyle='steps-post:')
pl.plot(pl.arange(101), pi_usa.mean(0), 'r-', linewidth=2, drawstyle='steps-post')
pl.savefig('age_integrating_sim.png')
# compare estimate to ground truth (skip endpoints, because they are extra hard to get right)
assert pl.allclose(m.pi.stats()['mean'][10:-10], pi_true[10:-10], rtol=.2)
lb, ub = m.pi.stats()['95% HPD interval'].T
assert pl.mean((lb <= pi_true)[10:-10] & (pi_true <= ub)[10:-10]) > .75
import copy
import pylab as p
from matplotlib.toolkits.basemap import Basemap
import os
import Scientific.IO.NetCDF as S
#--- Get data (assuming lon goes from 0 to 360): Make arrays cyclic
# in longitude:
time_idx = 5 #@@@ USER ADJUSTABLE
f = S.NetCDFFile('qm_seasonal_1yr.nc', mode='r')
lat = f.variables['lat'].getValue()
lon = f.variables['lon'].getValue()
u1_all = f.variables['u1'].getValue()
if p.allclose(lon[0], 0):
u1 = p.zeros((p.size(lat), p.size(lon) + 1))
u1[:, 0:-1] = u1_all[time_idx, :, :]
u1[:, -1] = u1_all[time_idx, :, 0]
tmp = copy.copy(lon)
lon = p.zeros((p.size(tmp) + 1, ))
lon[0:-1] = tmp[:]
lon[-1] = tmp[0] + 360
del tmp
f.close()
#--- Mapping information:
map = Basemap(projection='cyl',
def SLIP_step3D(IC, SLIP_params, use_legacy=False):
"""
simulates the SLIP in 3D
:args:
IC: initial state vector, containing y0, vx0, vz0
(x0 is assumed to be 0;
z0 is assumed to be 0;
vy0 = 0 (apex);
also ground level = 0)
SLIP_params(dict):
k
L0
m
alpha : "original" angle of attack
beta : lateral leg turn
foot position relative to CoM in flight:
xF = vx0*t_flight + L0*cos(alpha)*cos(beta)
yF = -L0*sin(alpha)
zF = vz0*t_flight - L0*cos(alpha)*sin(beta)
dE: energy change in "midstance" by changing k and L0
g: gravity (negative! should be ~ -9.81 for SI)
use_legacy (bool): Use legacy code or SLIP model in C
:returns:
sim_data, sim_state
sim_data : *dict*
the results of the simulation
sim_states: *dict*
information about the simulation (e.g. errors)
"""
alpha = SLIP_params['alpha']
beta = SLIP_params['beta']
k = SLIP_params['k']
L0 = SLIP_params['L0']
dE = SLIP_params['dE']
g = SLIP_params['g']
m = SLIP_params['m']
y0 = IC[0]
vx0 = IC[1]
vz0 = IC[2]
# model code in C
#if models == None:
# models = []
# models.append(mo.makeODE(model1_code, '_slip3D_1',skiphash=skiphash))
# models.append(mo.makeODE(model2_code, '_slip3D_2',skiphash=skiphash))
if not allclose(g, -9.81):
raise ValueError, "currently gravity is hard-coded to -9.81"
# concatenate state vector of four elements:
# (1) time to touchdown
# (2) time to vy = 0
# (3) time to takeoff
# (4) time to apex
# (1) and (4) are analytically described
y_land = L0 * sin(alpha)
if y0 < y_land:
raise ValueError, "invalid starting condition"
# before starting, define the model:
def SLIP_ode(y, t, params):
"""
defines the ODE of the SLIP, under stance condition
state:
[x
y
z
vx
vy
vz]
params:
{'L0' : leg rest length
'x0' : leg touchdown position
'k' : spring stiffness
'm' : mass
'xF' : anterior foot position
'zF' : lateral foot position }
"""
dy0 = y[3]
dy1 = y[4]
dy2 = y[5]
L = sqrt((y[0] - params['xF'])**2 + y[1]**2 + (y[2] - params['zF'])**2)
F = params['k'] * (params['L0'] - L)
Fx = F * (y[0] - params['xF']) / L
Fy = F * y[1] / L
Fz = F * (y[2] - params['zF']) / L
dy3 = Fx / m
dy4 = Fy / m + params['g']
dy5 = Fz / m
return hstack([dy0, dy1, dy2, dy3, dy4, dy5])
def sim_until(IC, params, stop_fcn, tmax=2.):
"""
simulated the SLIP_ode until stop_fcn has a zero-crossing
includes a refinement of the time at this instant
stop_fcn must be a function of the system state, e.g.
stop_fcn(IC) must exist
this function is especially adapted to the SLIP state,
so it uses dot(x1) = x3, dot(x2) = x4
tmax: maximal simulation time [s]
"""
init_sign = sign(stop_fcn(IC))
#1st: evaluate a certain fraction
tvec_0 = .001 * arange(50)
sim_results = []
sim_tvecs = []
newIC = IC
sim_results.append(
odeint(SLIP_ode, newIC, tvec_0, args=(params, ), rtol=1e-9))
sim_tvecs.append(tvec_0)
check_vec = [init_sign * stop_fcn(x) for x in sim_results[-1]]
t_tot = 0.
while min(check_vec) > 0:
newIC = sim_results[-1][-1, :]
sim_results.append(
odeint(SLIP_ode, newIC, tvec_0, args=(params, ), rtol=1e-9))
sim_tvecs.append(tvec_0)
check_vec = [init_sign * stop_fcn(x) for x in sim_results[-1]]
t_tot += tvec_0[-1]
# time exceeded or ground hit
if t_tot > tmax or min(sim_results[-1][:, 1] < 0):
raise SimFailError, "simulation failed"
# now: zero-crossing detected
# -> refine!
minidx = find(array(check_vec) < 0)[0]
if minidx == 0:
# this should not happen because the first value in
# check_vec should be BEFORE the zero_crossing by
# construction
raise ValueError, "ERROR: this should not happen!"
# refine simulation by factor 50, but only for two
# adjacent original time frames
newIC = sim_results[-1][minidx - 1, :]
sim_results[-1] = sim_results[-1][:minidx, :]
sim_tvecs[-1] = sim_tvecs[-1][:minidx]
# avoid that last position can be the zero-crossing
n_refine = 100
tvec_0 = linspace(tvec_0[0], tvec_0[1] + 2. / n_refine, n_refine + 2)
sim_results.append(
odeint(SLIP_ode, newIC, tvec_0, args=(params, ), rtol=1e-9))
sim_tvecs.append(tvec_0)
# linearly interpolate to zero
check_vec = [init_sign * stop_fcn(x) for x in sim_results[-1]]
minidx = find(array(check_vec) < 0)[0]
if minidx == 0:
# this should not happen because the first value in
# check_vec should be BEFORE the zero_crossing by
# construction
raise ValueError, "ERROR: this should not happen! (2)"
# compute location of zero-crossing
y0 = sim_results[-1][minidx - 1, :]
y1 = sim_results[-1][minidx, :]
fcn0 = stop_fcn(y0)
fcn1 = stop_fcn(y1)
t0 = tvec_0[minidx - 1]
t1 = tvec_0[minidx]
t_zero = t0 - (t1 - t0) * fcn0 / (fcn1 - fcn0)
# cut last simulation result and replace last values
# by interpolated values
sim_results[-1] = sim_results[-1][:minidx + 1, :]
sim_tvecs[-1] = sim_tvecs[-1][:minidx + 1]
for coord in arange(sim_results[-1].shape[1]):
sim_results[-1][-1, coord] = interp(
t_zero, [t0, t1],
[sim_results[-1][-2, coord], sim_results[-1][-1, coord]])
sim_tvecs[-1][-1] = t_zero
#newIC = sim_results[-1][minidx-1,:]
#sim_results[-1] = sim_results[-1][:minidx,:]
#sim_tvecs[-1] = sim_tvecs[-1][:minidx]
#tvec_0 = linspace(tvec_0[0],tvec_0[1],100)
#sim_results.append ( odeint(SLIP_ode, newIC, tvec_0,
# args=(params,),rtol=1e-9))
#sim_tvecs.append(tvec_0)
# concatenate lists
sim_data = vstack(
[x[:-1, :] for x in sim_results[:-1] if x.shape[0] > 1] + [
sim_results[-1],
])
sim_time = [
sim_tvecs[0],
]
for idx in arange(1, len(sim_tvecs)):
sim_time.append(sim_tvecs[idx] + sim_time[-1][-1])
sim_time = hstack([x[:-1] for x in sim_time[:-1]] + [
sim_time[-1],
])
return sim_data, sim_time
# --- Section 1: time to touchdown
# TODO: make sampling frequency regular
t_flight1 = sqrt(-2. * (y0 - y_land) / g)
#t_flight = sqrt()
tvec_flight1 = .01 * arange(t_flight1 * 100.)
vy_flight1 = tvec_flight1 * g
y_flight1 = y0 + .5 * g * (tvec_flight1**2)
x_flight1 = vx0 * tvec_flight1
vx_flight1 = vx0 * ones_like(tvec_flight1)
z_flight1 = vz0 * tvec_flight1
vz_flight1 = vz0 * ones_like(tvec_flight1)
x_TD = vx0 * t_flight1
z_TD = vz0 * t_flight1
# --- Section 2: time to vy = 0
# approach: calculate forward -> estimate interval of
# zero position of vy -> refine simulation in that interval
# until a point with vy sufficiently close to zero is in the
# resulting vector
params = {
'L0': L0,
'xF': t_flight1 * vx0 + L0 * cos(alpha) * cos(beta),
'zF': t_flight1 * vz0 - L0 * cos(alpha) * sin(beta),
'k': k,
'm': m,
'g': g
}
buffsize = 8000
buf = zeros((buffsize, _slipmdl1.WIDTH), dtype=np.float64)
IC = array([x_TD, y_land, z_TD, vx0, t_flight1 * g, vz0])
buf[0, 1:] = IC
buf[0, 0] = t_flight1
# for the model in C
xF = t_flight1 * vx0 + L0 * cos(alpha) * cos(beta)
zF = t_flight1 * vz0 - L0 * cos(alpha) * sin(beta)
#print k, L0, m, xF, zF
#print "IC=", IC
#print "t0:", t_flight1
if use_legacy: # skip original code
# initial guess: L0*cos(alpha)/vx0
#t_sim1 = L0*cos(alpha)/vx0
# TODO: implement sim_fail check!
sim_fail = False
try:
sim_phase2, t_phase2 = sim_until(IC, params, lambda x: x[4])
t_phase2 += t_flight1
except SimFailError:
print 'simulation aborted (phase 2)\n'
sim_fail = True
#print "t_min:", t_phase2[-1]
#print "y_min:", sim_phase2[-1,1]
else:
pars = array([k, L0, m, xF, zF], dtype=np.float64)
#print "pars=", pars
sim_fail = False
N = _slipmdl1.odeOnce(buf, 2., dt=5e-3, pars=pars)
#print "FS (landing): ", buf[N, [0,2,5]]
#print "N=", N
if N >= buffsize - 1 or buf[N, 0] >= 2.:
sim_fail = True
print "simulation aborted (phase 2)"
if not allclose(buf[N, 5], 0):
print "WARNING: nadir not found"
sim_phase2 = buf[:N, 1:].copy()
t_phase2 = buf[:N, 0].copy()
# Phase 3:
#return t_phase2, sim_phase2
if not sim_fail:
L = sqrt(sim_phase2[-1, 1]**2 + (sim_phase2[-1, 0] - params['xF'])**2 +
(sim_phase2[-1, 2] - params['zF'])**2)
if use_legacy: # skip original code
dk, dL = dk_dL(L0, k, L, dE)
params2 = deepcopy(params)
params2['k'] += dk
params2['L0'] += dL
IC = sim_phase2[-1, :]
compression = (lambda x: sqrt(
(x[0] - params2['xF'])**2 + x[1]**2 +
(x[2] - params['zF'])**2) - params2['L0'])
#print ('L:', L, 'dk', dk, 'dL', dL, 'dE', dE, '\ncompression:', compression(IC),
# 'IC', IC)
try:
sim_phase3, t_phase3 = sim_until(IC, params2, compression)
sim_phase3 = sim_phase3[1:, :]
t_phase3 = t_phase3[1:] + t_phase2[-1]
except SimFailError:
print 'simulation aborted (phase 3)\n'
sim_fail = True
else:
dk, dL = dk_dL(L0, k, L, dE)
pars[0] += dk
pars[1] += dL
# new IC and start time is final state and time from previous
# simulation step
buf[0, :] = buf[N, :]
buf[0, 5] = 0. # hard-code vertical velocity set to 0.
N = _slipmdl2.odeOnce(buf, 2., dt=5e-3, pars=pars)
if N >= buffsize - 1 or buf[N, 0] >= 2.:
sim_fail = True
print "simulation aborted (phase 3)"
else:
sim_phase3 = buf[1:N, 1:]
t_phase3 = buf[1:N, 0]
# Phase 4:
if not sim_fail:
# time to apex
# TODO: make sampling frequency regular
vy_liftoff = sim_phase3[-1, 4]
#vz_liftoff = sim_phase3[-1,5]
t_flight2 = -1. * vy_liftoff / g
#t_flight = sqrt()
tvec_flight2 = arange(t_flight2, 0, -.01)[::-1]
vy_flight2 = tvec_flight2 * g + vy_liftoff
y_flight2 = (sim_phase3[-1, 1] + vy_liftoff * tvec_flight2 + .5 * g *
(tvec_flight2**2))
x_flight2 = sim_phase3[-1, 0] + sim_phase3[-1, 3] * tvec_flight2
vx_flight2 = sim_phase3[-1, 3] * ones_like(tvec_flight2)
z_flight2 = sim_phase3[-1, 2] + sim_phase3[-1, 5] * tvec_flight2
vz_flight2 = sim_phase3[-1, 5] * ones_like(tvec_flight2)
#print tvec_flight2
tvec_flight2 += t_phase3[-1]
# todo: return data until error
if sim_fail:
return {
't': None,
'x': None,
'y': None,
'z': None,
'vx': None,
'vy': None,
'vz': None,
'sim_fail': sim_fail,
'dk': None,
'dL': None
}
# finally: concatenate phases
x_final = hstack(
[x_flight1, sim_phase2[:, 0], sim_phase3[:, 0], x_flight2])
y_final = hstack(
[y_flight1, sim_phase2[:, 1], sim_phase3[:, 1], y_flight2])
z_final = hstack(
[z_flight1, sim_phase2[:, 2], sim_phase3[:, 2], z_flight2])
vx_final = hstack(
[vx_flight1, sim_phase2[:, 3], sim_phase3[:, 3], vx_flight2])
vy_final = hstack(
[vy_flight1, sim_phase2[:, 4], sim_phase3[:, 4], vy_flight2])
vz_final = hstack(
[vz_flight1, sim_phase2[:, 5], sim_phase3[:, 5], vz_flight2])
tvec_final = hstack([tvec_flight1, t_phase2, t_phase3, tvec_flight2])
return {
't': tvec_final,
'x': x_final,
'y': y_final,
'z': z_final,
'vx': vx_final,
'vy': vy_final,
'vz': vz_final,
'sim_fail': sim_fail,
'dk': dk,
'dL': dL,
#'sim_res':sim_res,
#'sim_phase2': sim_phase2_cut,
#'t_phase2': t_phase2_cut
}
def intersect(h1, h2):
if h1.ambiant_dimension != h2.ambiant_dimension:
raise ValueError("Different ambiant\
spaces dimensions ({}!={})".format(len(w1), len(w2)))
# elif pl.allclose(w1,w2) and pl.allclose(b1,b2):
# print("return same object")
# return copy(h1)
# elif pl.allclose(w1,w2) and not pl.allclose(b1,b2):
# print("return empty set")
# return None
#Now that the trivial cases have been removed there exists an
#intersection of the two hyperplanes to obtain the new form of
#the hyperplane we perform the following 3 steps:
# (1)consider the intersection of two affine spaces as the intersection
# of a linear space and an affine space with modified bias
# (2)express the bias of the affine space in the linear space,
# this gives us the bias that will be for the new affine space
# (3)compute the basis of the intersection of the two linear spaces
# this gives us the span of the linear space, jointly with the above
# bias corresponding to the affine space from the intersection
#(1)
# we set h2 to be without bias and thus we alter h1 bias as follows
bp = h1.bias - h2.bias
#(2)
A = sp.hstack([h1.basis, h2.basis])
output = sparse_lsqr(A, bp)
alpha_beta_star, istop, itn = output[:3]
if (istop == 2):
warnings.warn("In Intersection of {} with {},\
least square solution is approximate".format(h1.name, h2.name))
# since we set h2 to be without bias we need to express the bias vector w.r.t.
# the h2 basis as follows
bpp = h2.basis.dot(alpha_beta_star[h1.dim:])
print("New bias is {}".format(bpp))
# (3) --------- Implement the Zassenhaus algorithm -------------
# assumes each space basis is in the same basis
# first need to create the original matrix
print(h1.basis.todense(), '\n\n')
print(h2.basis.todense(), '\n\n')
matrix = sp.bmat([[h1.basis, h2.basis], [h1.basis, None]]).T.todense()
# Now we need to put the top left half in row echelon form
# loop over the columns of the left half
rows = range(matrix.shape[0])
for column in range(h1.dim + h2.dim):
# print('start column:{}\nmatrix:\n{}\n\n'.format(column,matrix))
# compute current index and value of pivot A_{column,column}
pivot = matrix[rows[column], column]
# check for active pivoting to switch current row
# with the one with maximum value
maximum_index = matrix[rows[column:], column].argmax() + column
if maximum_index > column:
print('needs preventive partial pivoting')
rows[column] = maximum_index
rows[maximum_index] = column
pivot = matrix[rows[maximum_index], column]
# Loop over the rows
multiplicators = matrix[rows[column + 1:], column] / pivot
matrix[rows[column + 1:]] -= multiplicators.reshape(
(-1, 1)) * matrix[rows[column]].reshape((1, -1))
if (pl.allclose(matrix[rows[column + 1:]], 0)): return matrix
def test_data_model_sim():
# generate simulated data
n = 50
sigma_true = .025
# start with truth
a = pl.arange(0, 100, 1)
pi_age_true = .0001 * (a * (100. - a) + 100.)
# choose age intervals to measure
age_start = pl.array(mc.runiform(0, 100, n), dtype=int)
age_start.sort() # sort to make it easy to discard the edges when testing
age_end = pl.array(mc.runiform(age_start + 1,
pl.minimum(age_start + 10, 100)),
dtype=int)
# find truth for the integral across the age intervals
import scipy.integrate
pi_interval_true = [
scipy.integrate.trapz(pi_age_true[a_0i:(a_1i + 1)]) / (a_1i - a_0i)
for a_0i, a_1i in zip(age_start, age_end)
]
# generate covariates that add explained variation
X = mc.rnormal(0., 1.**2, size=(n, 3))
beta_true = [-.1, .1, .2]
Y_true = pl.dot(X, beta_true)
# calculate the true value of the rate in each interval
pi_true = pi_interval_true * pl.exp(Y_true)
# simulate the noisy measurement of the rate in each interval
p = mc.rnormal(pi_true, 1. / sigma_true**2.)
# store the simulated data in a pandas DataFrame
data = pandas.DataFrame(
dict(value=p,
age_start=age_start,
age_end=age_end,
x_0=X[:, 0],
x_1=X[:, 1],
x_2=X[:, 2]))
data['effective_sample_size'] = pl.maximum(p * (1 - p) / sigma_true**2, 1.)
data['standard_error'] = pl.nan
data['upper_ci'] = pl.nan
data['lower_ci'] = pl.nan
data['year_start'] = 2005. # TODO: make these vary
data['year_end'] = 2005.
data['sex'] = 'total'
data['area'] = 'all'
# generate a moderately complicated hierarchy graph for the model
hierarchy = nx.DiGraph()
hierarchy.add_node('all')
hierarchy.add_edge('all', 'super-region-1', weight=.1)
hierarchy.add_edge('super-region-1', 'NAHI', weight=.1)
hierarchy.add_edge('NAHI', 'CAN', weight=.1)
hierarchy.add_edge('NAHI', 'USA', weight=.1)
output_template = pandas.DataFrame(
dict(year=[1990, 1990, 2005, 2005, 2010, 2010] * 2,
sex=['male', 'female'] * 3 * 2,
x_0=[.5] * 6 * 2,
x_1=[0.] * 6 * 2,
x_2=[.5] * 6 * 2,
pop=[50.] * 6 * 2,
area=['CAN'] * 6 + ['USA'] * 6))
# create model and priors
vars = data_model.data_model('test', data, hierarchy, 'all')
# fit model
mc.MAP(vars).fit(method='fmin_powell', verbose=1)
m = mc.MCMC(vars)
m.use_step_method(mc.AdaptiveMetropolis, [m.gamma_bar, m.gamma, m.beta])
m.sample(30000, 15000, 15)
# check estimates
pi_usa = data_model.predict_for(output_template, hierarchy, 'all', 'USA',
'male', 1990, vars)
assert pl.allclose(pi_usa.mean(), (m.mu_age.trace() * pl.exp(.05)).mean(),
rtol=.1)
# check convergence
print 'gamma mc error:', m.gamma_bar.stats()['mc error'].round(
2), m.gamma.stats()['mc error'].round(2)
# plot results
for a_0i, a_1i, p_i in zip(age_start, age_end, p):
pl.plot([a_0i, a_1i], [p_i, p_i], 'rs-', mew=1, mec='w', ms=4)
pl.plot(a, pi_age_true, 'g-', linewidth=2)
pl.plot(pl.arange(101),
m.mu_age.stats()['mean'],
'k-',
drawstyle='steps-post',
linewidth=3)
pl.plot(pl.arange(101),
m.mu_age.stats()['95% HPD interval'],
'k',
linestyle='steps-post:')
pl.plot(pl.arange(101),
pi_usa.mean(0),
'r-',
linewidth=2,
drawstyle='steps-post')
pl.savefig('age_integrating_sim.png')
# compare estimate to ground truth (skip endpoints, because they are extra hard to get right)
assert pl.allclose(m.pi.stats()['mean'][10:-10], pi_true[10:-10], rtol=.2)
lb, ub = m.pi.stats()['95% HPD interval'].T
assert pl.mean((lb <= pi_true)[10:-10] & (pi_true <= ub)[10:-10]) > .75
