def _set_C(self, new_C):
self._C = new_C
new_C.flags['WRITEABLE'] = False
# This will catch the first time C's diagonal is negative fo sho.
if np.any(np.diag(new_C) < 0):
from IPython.Debugger import Pdb
Pdb(color_scheme='Linux').set_trace()
def make_pt_fig(cur_val,
samps,
output_fname,
output_path,
outfigs_transparent=False,
hist_color=(0, .1, .5),
line_color='r-'):
"""Creates a png file from a point, writes it to disk and returns the path."""
output_fname += '.png'
pl.figure()
try:
h, b, p = pl.hist(samps,
10,
normed=True,
facecolor=hist_color,
histtype='stepfilled')
except:
from IPython.Debugger import Pdb
Pdb(color_scheme='Linux').set_trace()
pl.xlabel(r'$x$')
pl.ylabel(r'$p(x)$')
pl.plot([cur_val, cur_val], [0, h.max()],
line_color,
linewidth=2,
label='Current value')
pl.title('Utility function: standard deviation')
l = pl.legend(loc=0)
l.legendPatch.set_alpha(0)
pl.savefig(output_path + '/' + output_fname,
transparent=outfigs_transparent)
return '/'.join([os.getcwd(), output_path, output_fname])
def post_mortem(tb):
ip = ipapi.get()
def_colors = ip.options.colors
p = Pdb(def_colors)
p.reset()
while tb.tb_next is not None:
tb = tb.tb_next
p.interaction(tb.tb_frame, tb)
def magma_spotrf_gpu(uplo, n, A, lda, work):
info = 1
from IPython.Debugger import Pdb
Pdb(color_scheme='LightBG').set_trace()
_magma_spotrf_gpu(char_pointer(uplo),
int_pointer(n),
# ====================================================================
# = I think this argument is responsible for the segmentation fault. =
# ====================================================================
ctypes.POINTER(ctypes.c_float).from_address(get_gpu_pointer(A)),
int_pointer(lda),
work.ctypes.data_as(ctypes.POINTER(ctypes.c_float)),
int_pointer(info))
return info
def robust_observe_and_eval(lm, M, C, samp_mesh, nug, pred_mesh):
"""
Evaluates and observes a mean and covariance, and returns their evaluation on a
prediction mesh. Works even for negative 'observation variance'.
"""
nsm = samp_mesh.shape[0]
if lm is not None:
# Observe according to EP likelihood
try:
pm.gp.observe(M, C, samp_mesh, lm, nug)
C_mesh = C(pred_mesh, pred_mesh)
M_mesh = M(pred_mesh)
except np.linalg.LinAlgError:
# If there's a problem with the Cholesky-based algorithm (and there easily might be
# because lv may be large and negative) then do it the boneheaded way.
C_mesh = C(pred_mesh, pred_mesh)
M_mesh = M(pred_mesh)
C_samp = C(samp_mesh, samp_mesh).copy()
M_samp = M(samp_mesh)
np.ravel(
C_samp)[::nsm +
1] += nug # Add lv+V to diagonal of C_samp in place
C_samp_I = C_samp.I
C_off = C(samp_mesh, pred_mesh)
C_mesh -= C_off.T * C_samp_I * C_off
M_mesh += np.ravel(np.dot(C_off.T * C_samp_I, (lm - M_samp)))
else:
C_mesh = C(pred_mesh, pred_mesh)
M_mesh = M(pred_mesh)
if np.any(np.diag(C_mesh) < 0):
from IPython.Debugger import Pdb
Pdb(color_scheme='Linux').set_trace()
return M_mesh, C_mesh
def postmortem_hook(etype, value, tb): # pragma no cover
import sys
import pdb
import traceback
try:
from IPython.ipapi import make_session
make_session()
from IPython.Debugger import Pdb
sys.stderr.write('Entering post-mortem IPDB shell\n')
p = Pdb(color_scheme='Linux')
p.reset()
p.setup(None, tb)
p.print_stack_trace()
sys.stderr.write('%s: %s\n' % (etype, value))
p.cmdloop()
p.forget()
# p.interaction(None, tb)
except ImportError:
sys.stderr.write('Entering post-mortem PDB shell\n')
traceback.print_exception(etype, value, tb)
pdb.post_mortem(tb)
def set_trace():
ip = ipapi.get()
def_colors = ip.options.colors
Pdb(def_colors).set_trace(sys._getframe().f_back)
def pm(etype, value, tb): # pragma no cover
import pdb, traceback
try:
from IPython.ipapi import make_session; make_session()
from IPython.Debugger import Pdb
sys.stderr.write('Entering post-mortem IPDB shell\n')
p = Pdb(color_scheme='Linux')
p.reset()
p.setup(None, tb)
p.print_stack_trace()
sys.stderr.write('%s: %s\n' % ( etype, value))
p.cmdloop()
p.forget()
# p.interaction(None, tb)
except ImportError:
sys.stderr.write('Entering post-mortem PDB shell\n')
traceback.print_exception(etype, value, tb)
pdb.post_mortem(tb)
M.sample(100)
t2 = time.time()
print 'Class %s: %s seconds, accepted %i, rejected %i' % (
kls.__name__, t2 - t1, sm.accepted, sm.rejected)
import pylab as pl
pl.clf()
pl.plot(M.trace('x')[:, 0],
M.trace('y')[:, 0],
linestyle='none',
marker='.',
markersize=8,
color=(
.1,
0,
.8,
),
alpha=10. / len(M.trace('x')[:]),
markeredgewidth=0)
pl.title(kls.__name__)
from IPython.Debugger import Pdb
Pdb(color_scheme='Linux').set_trace()
# a = np.empty((1000,2))
# for i in xrange(1000):
# M.step_method_dict[x][0].step()
# a[i,0] = x.value
# a[i,1] = y.value
#
# import pylab as pl
# pl.plot(a[:,0], a[:,1], 'b.')
def debug(f, *args, **kwargs):
from IPython.Debugger import Pdb
Pdb(color_scheme='Linux').set_trace()
return f(*args, **kwargs)
def fit(self, N, tol=.01):
"""
Stores approximate likelihood parameters mu, V
and returns approximation of log p(D).
"""
# Make initial observations (usually trivial)
self.C = self.C_pri.copy()
self.M = self.M_pri.copy()
self.coefs = simps_coefs(N)
for i in xrange(self.Nx):
self.observe(i)
if np.any(np.isinf(self.C)) or np.any(np.isinf(self.M)):
# Pdb(color_scheme='Linux').set_trace()
raise RuntimeError, 'C or M is infinite.'
if np.any(np.diag(self.C) < 0):
# Pdb(color_scheme='Linux').set_trace()
raise RuntimeError, 'C has negative diagonal.'
# To help convergence, use same iid normal samples
# in all iterations.
# self.normal_samps = np.random.normal(size=N_samps)
# self.nug_samps = np.random.normal(size=N_samps)
# Record last values of mu and V to assess convergence.
last_mu = np.copy(self.mu)
last_V = np.copy(self.V)
sw = 0
dmu = np.Inf
dV = np.Inf
# print self.mu, self.V
while max(dmu, dV) > tol:
sw += 1
# Update
self.update_sweep(N)
# Assess convergence and possibly terminate.
dmu = np.max(np.abs((self.mu - last_mu) / self.mu))
if np.isnan(dmu):
dmu = np.max(np.abs((self.mu - last_mu) * 10))
dV = np.max(np.abs((self.V - last_V) / self.V))
if sw > 10000 and sw % 100 == 0:
print 'Too many iterations, randomizing.'
# Pdb(color_scheme='Linux').set_trace()
print 'mu: %s' % self.mu
print 'V: %s' % self.V
self.V = last_V + np.random.random(
size=len(self.V)) * dV * self.V
self.mu = last_mu + np.random.random(
size=len(self.mu)) * dmu * self.mu
last_V[:] = self.V[:]
last_mu[:] = self.mu[:]
# print self.mu, self.V
V = np.array((self.V + self.nug))
V_ind = V + np.diag(self.C_pri)
C_joint = self.C_pri + np.diag(V)
if np.all(V_ind > 0):
joint_term = pm.mv_normal_cov_like(self.mu, self.M_pri, C_joint)
ind_term = pm.normal_like(self.mu, self.M_pri, 1 / V_ind)
log_ratio = joint_term - ind_term
# Protect against negative 'variances' (which are acceptable)
else:
V = V.astype('complex')
V_ind = V_ind.astype('complex')
C_joint = C_joint.astype('complex')
dev = self.mu - self.M_pri
ind_term = -.5 * np.sum(
(np.log(2. * np.pi * V_ind) + dev**2 / V_ind))
val, vec = np.linalg.eig(C_joint)
sq = np.asarray(np.dot(dev, vec / np.sqrt(val))).ravel()
joint_term = -.5 * (np.sum(
(np.log(2. * np.pi * val))) + np.dot(sq, sq))
log_ratio = np.real(joint_term - ind_term)
if np.sum(self.lp) + log_ratio > 10000:
print 'Warning, HUGE p'
if np.any(np.isnan(log_ratio)):
from IPython.Debugger import Pdb
Pdb(color_scheme='Linux').set_trace()
return np.sum(self.lp) + log_ratio
newdp=DataPoint(value=line[8:13], time=datetime.datetime(2009,12,int(line[0:2]),int(line[3:4]),int(line[4:5])))
print newdp
for line in cones_09:
newdp=DataPoint(value=line[8:13], time=datetime.datetime(2009,12,int(line[0:2]),int(line[3:4]),int(line[4:5])))
print newdp
for line in cones_09:
newdp=DataPoint(value=line[8:13], time=datetime.datetime(2009,12,int(line[0:2]),int(line[3:4]),int(line[4:5])))
print newdp
newts.datapoint_set.add(newdp)
newts
cones_09=open('/fs/raid/users/aarongc/cones_aws_dec09.txt','r')
cones_09.next()
cones_09.next()
cones_09.next()
cones_09.next()
cones_09.next()
cones_09.next()
for line in cones_09:
newts.datapoint_set.add(DataPoint(value=line[8:13], time=datetime.datetime(2009,12,int(line[0:2]),int(line[3:4]),int(line[4:5]))))
from IPython.Debugger import Pdb
Pdb.pm)_
Pdb.pm()
_ip.magic("pdb ")
fump
_ip.magic("debug ")
_ip.magic("prun 5+2")
_ip.magic("prun 5+2")
exit90
exit()
def create_realization(outfile_root, real_index, C, C_straightfromtrace,
mean_ondata, M, covariate_mesh, tdata, data_locs, grids,
axes, data_mesh_indices, where_in, where_out,
n_blocks_x, n_blocks_y, relp, mask, thinning, indices,
paramfileINDEX, NinThinnedBlock):
# define only realizations chunk of hdf5 realization file
out_arr = outfile_root.realizations
"""
Creates a single realization from the predictive distribution over specified space-time mesh.
"""
grid_shape = tuple([grid[2] for grid in grids])
#r.X11(width=8,height=4)
#r.par(mfrow=(1,2))
#r.plot(data_mesh_indices[:,0],data_mesh_indices[:,1],xlab="",ylab="",main="",cex=0.5)
#r.plot(data_locs[:,0],data_locs[:,1],xlab="",ylab="",main="",cex=0.5)
#from IPython.Debugger import Pdb
#Pdb(color_scheme='Linux').set_trace()
thin_grids = tuple([grid[:2] + (grid[2] / thinning, ) for grid in grids])
thin_grid_shape = tuple([thin_grid[2] for thin_grid in thin_grids])
thin_axes = tuple([np.linspace(*thin_grid) for thin_grid in thin_grids])
mapgrid = np.array(mgrid[0:grid_shape[0], 0:grid_shape[1]], dtype=float)
for i in xrange(2):
normalize_for_mapcoords(mapgrid[i], thin_grid_shape[i] - 1)
thin_mapgrid = np.array(mgrid[0:thin_grid_shape[0], 0:thin_grid_shape[1]],
dtype=float)
for i in xrange(2):
normalize_for_mapcoords(thin_mapgrid[i], grid_shape[i] - 1)
def thin_to_full(thin_row):
return ndimage.map_coordinates(thin_row, mapgrid)
def full_to_thin(row):
return ndimage.map_coordinates(row, thin_mapgrid)
thin_mask = np.array(np.round(full_to_thin(mask)), dtype='bool')
# Container for x
thin_x = np.empty(thin_grid_shape[:2] + (3, ))
mlon, mlat = np.meshgrid(*thin_axes[:2])
thin_x[:, :, 0] = mlon.T
thin_x[:, :, 1] = mlat.T
thin_x[:, :, 2] = 0
x = np.empty(grid_shape[:2] + (3, ))
mlon, mlat = np.meshgrid(*axes[:2])
x[:, :, 0] = mlon.T
x[:, :, 1] = mlat.T
x[:, :, 2] = 0
del mlon, mlat
Cp = C.params
# Prepare input dictionaries
covParamObj = {
'Scale': Cp['scale'][0],
'amp': Cp['amp'][0],
'inc': Cp['inc'][0],
'ecc': Cp['ecc'][0],
't.lim.corr': Cp['tlc'][0],
'scale.t': Cp['st'][0],
'sin.frac': Cp['sf'][0]
}
gridParamObj = {
'YLLCORNER': grids[1][0] * rad_to_deg,
'CELLSIZE':
(grids[1][1] - grids[1][0]) / (grids[1][2] - 1.) * rad_to_deg,
'NROWS': grid_shape[1],
'NCOLS': grid_shape[0]
}
monthParamObj = {'Nmonths': grid_shape[2], 'StartMonth': grids[2][0]}
################################~TEMP
# Call R preprocessing function and check to make sure no screwy re-casting has taken place.
#t1 = time.time()
#os.chdir(r_path)
#preLoopObj = r.CONDSIMpreloop(covParamObj,gridParamObj,monthParamObj,indices.min(), indices.max(),paramfileINDEX)
#tree_reader = reader(file('listSummary_preLoopObj_original_%i_%i.txt'%(indices.min(), indices.max())),delimiter=' ')
#preLoopClassTree, junk = parse_tree(tree_reader)
#preLoopObj = compare_tree(preLoopObj, preLoopClassTree)
#OutMATlist = preLoopObj['OutMATlist']
#tree_reader = reader(file('listSummary_OutMATlist_original_%i_%i.txt'%(indices.min(), indices.max())),delimiter=' ')
#OutMATClassTree, junk = parse_tree(tree_reader)
#OutMATlist = compare_tree(OutMATlist, OutMATClassTree)
#os.chdir(curpath)
#preLoop_time = time.time()-t1
#print "preLoop_time :"+str(preLoop_time)
## Create and store unconditional realizations
#print '\tGenerating unconditional realizations.'
#t1 = time.time()
#for i in xrange(grid_shape[2]):
# print 'On month :'+str(i)
# #print 'OutMATlist:'
# #print OutMATlist
# os.chdir(r_path)
# monthObject = r.CONDSIMmonthloop(i+1,preLoopObj,OutMATlist, indices.min(), indices.max(),paramfileINDEX)
# #monthObject = r.CONDSIMmonthloop(i+1,preLoopObj,OutMATlist,paramfileINDEX)
# os.chdir(curpath)
# OutMATlist= monthObject['OutMATlist']
# MonthGrid = monthObject['MonthGrid']
# out_arr[real_index,:,:,i] = MonthGrid[:grid_shape[1],:grid_shape[0]]
#t2 = time.time()
#print '\t\tDone in %f'%(t2-t1)
#print "monthloop_time :"+str(t2-t1)+" for "+str(grid_shape[2])+" months"
# delete unneeded R products
#del OutMATlist, preLoopObj, MonthGrid, monthObject
################################~TEMP
################################~TEMP DIRECTLY JOIN SIMULATE UNCODITIONED BLOCK FOR TESTING
getUnconditionedBlock(out_arr,
real_index,
grids,
C_straightfromtrace,
NinThinnedBlock=None,
relp=None,
FULLRANK=False)
print 'variance of unconditioned block = ' + str(round(
np.var(out_arr), 10))
print 'variance of unconditioned block month 6 = ' + str(
round(np.var(out_arr[:, :, :, 6]), 10))
examineRealization(outfile_root,
0,
6,
15,
None,
None,
conditioned=False,
flipVertical="FALSE",
SPACE=True,
TIME=True)
################################~TEMP
# Figure out pdata
pdata = np.empty(tdata.shape)
for i in xrange(len(where_in)):
pdata[where_in[i]] = out_arr[real_index, grid_shape[1] - 1 -
data_mesh_indices[i, 1],
data_mesh_indices[i, 0],
data_mesh_indices[i, 2]]
# jointly simulate at data points conditional on block
## first get XYZT list of locations of a regular thinned sample from block
#array3d = out_arr[real_index,:,:,:]
ThinnedBlockXYTZlists = getThinnedBlockXYTZlists(out_arr, real_index,
grids, NinThinnedBlock)
xyt_in = ThinnedBlockXYTZlists['xyt_in']
z_in = ThinnedBlockXYTZlists['z_in']
## get locations of data outside block (referenced by grid location)
#XYT_out_gridlocs = data_mesh_indices[where_out]
#
### convert these grid lcoations into actual position in radians and months
#coordsDict = gridParams_2_XYTmarginallists(grids)
#xcoords = coordsDict['xcoords']
#ycoords = coordsDict['ycoords']
#tcoords = coordsDict['tcoords']
#
#x_out = xcoords[XYT_out_gridlocs[:,0]]
#y_out = ycoords[XYT_out_gridlocs[:,1]]
#t_out = tcoords[XYT_out_gridlocs[:,2]]
#
#xyt_out = np.vstack((x_out,y_out,t_out)).T
# get locations of data outside block
xyt_out = data_locs[where_out]
# now we have locations and values of thinned sample from the block, and locations we want to predict at outside the block,go ahead and
# get simulated values of the latter, conditonal on the former
print '\tsimulating over ' + str(
len(xyt_out[:, 0])
) + ' locations outside block using thinned block sample of ' + str(
len(z_in)) + ' points'
t1 = time.time()
z_out = predictPointsFromBlock(xyt_in, z_in, xyt_out, C_straightfromtrace,
relp)
print '\ttime for simulation: ' + str(time.time() - t1)
#########################################CHECK COVARIANCE STRUCTURE
#r.X11(width=12,height=12)
#r.par(mfrow=(3,2))
# test marginal space and time covariance structures of points outside block
#cfdict_out = getEmpiricalCovarianceFunction_STmarginals(xyt_out,z_out,mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8)
#plotEmpiricalCovarianceFunction(cfdict_out['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points outside (S) "+str(paramfileINDEX))
#plotEmpiricalCovarianceFunction(cfdict_out['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points outside (T)"+str(paramfileINDEX))
# test marginal space and time covariance structures of points inside block
#cfdict_in = getEmpiricalCovarianceFunction_STmarginals(xyt_in,z_in,mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8)
#plotEmpiricalCovarianceFunction(cfdict_in['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points inside (S)"+str(paramfileINDEX))
#plotEmpiricalCovarianceFunction(cfdict_in['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points inside (T)"+str(paramfileINDEX))
# test marginal space and time covariance structures of points inside and outside block
#cfdict_inout = getEmpiricalCovarianceFunction_STmarginals(np.vstack((xyt_in,xyt_out)),np.hstack((z_in,z_out)),mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8)
#plotEmpiricalCovarianceFunction(cfdict_inout['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points inside (S)"+str(paramfileINDEX))
#plotEmpiricalCovarianceFunction(cfdict_inout['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points inside (T)"+str(paramfileINDEX))
#########################################CHECK COVARIANCE STRUCTURE
# assign these values to pdata
pdata[where_out] = z_out
###############################~~TEMP
#return()
#####################################
# Bring in data.
print '\tKriging to bring in data.'
print '\tPreprocessing.'
t1 = time.time()
dev_posdef, xbi, ybi, dl_posdef = preprocess(C, data_locs, thin_grids,
thin_x, n_blocks_x,
n_blocks_y, tdata, pdata,
relp, mean_ondata)
t2 = time.time()
print '\t\tDone in %f' % (t2 - t1)
###############################~~TEMP
#from IPython.Debugger import Pdb
#Pdb(color_scheme='Linux').set_trace()
#####################################
thin_row = np.empty(thin_grid_shape[:2], dtype=np.float32)
print '\tKriging.'
t1 = time.time()
C1 = pm.gp.NearlyFullRankCovariance(C_straightfromtrace.eval_fun,
**C_straightfromtrace.params)
C2 = pm.gp.NearlyFullRankCovariance(C_straightfromtrace.eval_fun,
**C_straightfromtrace.params)
M1 = pm.gp.Mean(lambda x: np.zeros(x.shape[:-1]))
M2 = pm.gp.Mean(lambda x: np.zeros(x.shape[:-1]))
cholfac = C1.cholesky(data_locs, apply_pivot=False)
U = cholfac['U']
piv = cholfac['pivots']
m = U.shape[0]
w = np.linalg.solve(
U[:m, :m], np.linalg.solve(U[:m, :m].T,
(tdata - mean_ondata)[piv[:m]]))
pm.gp.observe(M2, C2, data_locs, pdata)
for i in xrange(grid_shape[2] - 1, -1, -1):
thin_row.fill(0.)
thin_x[:, :, 2] = axes[2][i]
x[:, :, 2] = axes[2][i]
simkrige = M2(x)
dkrige = covariate_mesh + M(x) + np.dot(
C1(x, data_locs[piv[:m]]).view(ndarray), w).reshape(x.shape[:-1])
simsurf = grid_convert(out_arr[real_index, :, :, i], 'y-x+', 'x+y+')
import pylab as pl
pl.imshow(simsurf)
pl.colorbar()
pl.title('Simulated')
pl.figure()
pl.imshow(simkrige)
pl.colorbar()
pl.title('Sim krige')
pl.figure()
pl.imshow(simsurf - simkrige)
pl.colorbar()
pl.title('Adjusted sim')
pl.figure()
pl.imshow(dkrige)
pl.colorbar()
pl.title('Data krige')
pl.figure()
pl.imshow(simsurf - simkrige + dkrige)
pl.colorbar()
pl.title('Kriged sim')
krige_month(C, i, dl_posdef, thin_grid_shape, n_blocks_x, n_blocks_y,
xbi, ybi, thin_x, dev_posdef, thin_row, thin_mask)
row = ndimage.map_coordinates(thin_row, mapgrid)
row += covariate_mesh
row += M(x)
row += grid_convert(out_arr[real_index, :, :, i], 'y-x+', 'x+y+')
pl.figure()
pl.imshow(row)
pl.colorbar()
pl.title('Original method')
from IPython.Debugger import Pdb
Pdb(color_scheme='Linux').set_trace()
# NaN the oceans to save storage
row[np.where(1 - mask)] = missing_val
out_arr[real_index, :, :, i] = grid_convert(row, 'x+y+', 'y-x+')
####################################TEMP
print 'variance of conditioned block month 6 = ' + str(
round(np.var(out_arr[:, :, :, 6]), 10))
print 'variance of conditioned block = ' + str(round(np.var(out_arr), 10))
examineRealization(outfile_root,
0,
6,
15,
None,
None,
conditioned=True,
flipVertical="FALSE",
SPACE=True,
TIME=True)
########################################
print '\t\tDone in %f' % (time.time() - t1)
def pdb_builder():
return Pdb()
def pdb_builder():
return Pdb(ipapi.get().options.colors)