def __init__(self, order_range = [0, 30], crit = 'sbc'):
"""
"""
GeoField.__init__(self)
self.sd = None
self.order_range = order_range
self.crit = crit
def __init__(self, order_range=[0, 30], crit='sbc'):
"""
"""
GeoField.__init__(self)
self.sd = None
self.order_range = order_range
self.crit = crit
def make_model_geofield(S, ts):
M, N = S.shape
T = ts.shape[0]
gf = GeoField()
gf.lats = np.arange(M)
gf.lons = np.arange(N)
gf.tm = np.arange(T)
gf.d = np.reshape(ts, [T, M, N])
return gf
def plot_components_slp():
gf = GeoField()
gf.load('data/pres.mon.mean.nc', 'pres')
gf.slice_spatial(None, [20, 89])
with open('results/slp_nh_var_bootstrap_results_b1000_cosweights.bin', 'r') as f:
d = cPickle.load(f)
mn = d['mean']
mn_gf = gf.reshape_flat_field(mn)
t_start = datetime.now()
thr = 1.0 / mn.shape[0]**0.5
pool = Pool(4)
render_list_triples = [ ([mn_gf[i, ...],
mn_gf[i, ...] * (np.abs(mn_gf[i,...]) > thr)],
[ 'Mean', 'Mean:Thr'],
gf.lats, gf.lons, False,
# 'figs/nhemi_comp%02d_varimax_mn.png' % (i+1),
'figs/nhemi_comp%02d_varimax_mn.pdf' % (i+1),
'Component %d' % (i+1))
for i in range(mn.shape[1])]
# use less nodes to render the maps due to memory constraints
pool.map(render_set_stub, render_list_triples)
pool.close()
# clean up memory
del render_list_triples
print("DONE after [%s]." % (datetime.now() - t_start))
def render_slp_components():
with open('results/slp_nh_var_bootstrap_results_b1000.bin', 'r') as f:
d = cPickle.load(f)
mn = d['mean']
mn_mask = (np.abs(mn) > 1.0 / mn.shape[0]**0.5)
mn_thr = mn * mn_mask
print np.sum(np.sum(mn_mask, axis= 1) == 0)
cid = np.argmax(np.abs(mn) * mn_mask, axis = 1)[:, np.newaxis] + 1
gf = GeoField()
gf.load('data/pres.mon.mean.nc', 'pres')
gf.slice_spatial(None, [20, 89])
mnd = gf.reshape_flat_field(cid)
# plt.figure()
# plt.hist(cid, bins = 43)
# plt.show()
f = render_component_single(mnd[0, :, :], gf.lats, gf.lons, False, None,
'NH Extratropical Components',
cmap = plt.get_cmap('gist_ncar'))
plt.show()
def render_slp_component_element_values():
with open(FILE_NAME_COMPS, 'r') as f:
d = cPickle.load(f)
mn = d['mean']
cid = np.amax(np.abs(mn), axis=1)[:, np.newaxis]
gf = GeoField()
gf.load('data/pres.mon.mean.nc', 'pres')
gf.slice_spatial(None, [20, 89])
mnd = gf.reshape_flat_field(cid)
f = render_component_single(mnd[0, :, :], gf.lats, gf.lons, (0, 0.15),
None,
'NH Extratropical Components - max values')
f.savefig('figs/slp_nh_component_maxima_sameaxis.pdf')
f = render_component_single(mnd[0, :, :], gf.lats, gf.lons, None, None,
'NH Extratropical Components - max values')
f.savefig('figs/slp_nh_component_maxima.pdf')
f = plt.figure()
plt.hist(cid, bins=40)
plt.title('Histogram of max values @ grid points across components')
plt.xlabel('Maximum value [-]')
plt.ylabel('Frequency [-]')
f.savefig('figs/slp_nh_compmax_hist.pdf')
return f
def render_slp_model_orders():
with open(FILE_NAME_EIGS, 'r') as f:
d = cPickle.load(f)
print(d.keys())
o = d['orders'][:, np.newaxis]
print(o.shape)
gf = GeoField()
gf.load('data/pres.mon.mean.nc', 'pres')
gf.slice_spatial(None, [20, 89])
# od = gf.reshape_flat_field(o)
f = render_component_single(o[:, 0, :], gf.lats, gf.lons, False, None,
'NH Extratropical Components - AR model order')
f.savefig('figs/slp_nh_model_order.pdf')
def render_slp_component_element_values():
with open(FILE_NAME_COMPS, 'r') as f:
d = cPickle.load(f)
mn = d['mean']
cid = np.amax(np.abs(mn), axis = 1)[:, np.newaxis]
gf = GeoField()
gf.load('data/pres.mon.mean.nc', 'pres')
gf.slice_spatial(None, [20, 89])
mnd = gf.reshape_flat_field(cid)
f = render_component_single(mnd[0, :, :], gf.lats, gf.lons, (0, 0.15), None,
'NH Extratropical Components - max values')
f.savefig('figs/slp_nh_component_maxima_sameaxis.pdf')
f = render_component_single(mnd[0, :, :], gf.lats, gf.lons, None, None,
'NH Extratropical Components - max values')
f.savefig('figs/slp_nh_component_maxima.pdf')
f = plt.figure()
plt.hist(cid, bins = 40)
plt.title('Histogram of max values @ grid points across components')
plt.xlabel('Maximum value [-]')
plt.ylabel('Frequency [-]')
f.savefig('figs/slp_nh_compmax_hist.pdf')
return f
def make_model_geofield(S, ts):
M, N = S.shape
T = ts.shape[0]
gf = GeoField()
gf.lats = np.arange(M)
gf.lons = np.arange(N)
gf.tm = np.arange(T)
gf.d = np.reshape(ts, [T, M, N])
return gf
def render_slp_component_element_values():
with open('results/slp_nh_var_bootstrap_results_b1000_cosweights.bin', 'r') as f:
d = cPickle.load(f)
mn = d['mean']
cid = np.amax(np.abs(mn), axis = 1)[:, np.newaxis]
gf = GeoField()
gf.load('data/pres.mon.mean.nc', 'pres')
gf.slice_spatial(None, [20, 89])
mnd = gf.reshape_flat_field(cid)
f = render_component_single(mnd[0, :, :], gf.lats, gf.lons, False, None,
'NH Extratropical Components - max values')
plt.show()
def plot_slp_model_orders():
with open('results/slp_eigvals_multi_surrogates_nh_var.bin', 'r') as f:
d = cPickle.load(f)
print(d.keys())
o = d['orders'][:, np.newaxis]
print(o.shape)
gf = GeoField()
gf.load('data/pres.mon.mean.nc', 'pres')
gf.slice_spatial(None, [20, 89])
# od = gf.reshape_flat_field(o)
f = render_component_single(o[:, 0, :], gf.lats, gf.lons, False, None,
'NH Extratropical Components - AR model order')
plt.show()
def render_slp_model_orders():
with open(FILE_NAME_EIGS, 'r') as f:
d = cPickle.load(f)
print(d.keys())
o = d['orders'][:, np.newaxis]
print(o.shape)
gf = GeoField()
gf.load('data/pres.mon.mean.nc', 'pres')
gf.slice_spatial(None, [20, 89])
# od = gf.reshape_flat_field(o)
f = render_component_single(
o[:, 0, :], gf.lats, gf.lons, False, None,
'NH Extratropical Components - AR model order')
f.savefig('figs/slp_nh_model_order.pdf')
plt.figure()
plt.imshow(v.A, interpolation='nearest')
plt.colorbar()
plt.title('AR structural')
plt.figure()
plt.plot(ts)
plt.title('Simulated time series')
C = np.corrcoef(ts, None, rowvar=0)
plt.figure()
plt.imshow(C, interpolation='nearest')
plt.title('Correlation matrix')
plt.colorbar()
gf = GeoField()
gf.lons = np.arange(S.shape[1])
gf.lats = np.arange(S.shape[0])
gf.tm = np.arange(768)
gf.d = np.reshape(ts, [768, S.shape[0], S.shape[1]])
plt.figure()
plt.plot(ts2[:, 0])
plt.plot(gf.d[:, 0, 0])
with open('data/test_gf.bin', 'w') as f:
cPickle.dump(gf, f)
plt.show()
from datetime import date
from geo_field import GeoField
from geo_rendering import render_component_single
import numpy as np
import cPickle
import matplotlib.pyplot as plt
import scipy.io as sio
if __name__ == '__main__':
# load geo-field
gf = GeoField()
gf.load('data/pres.mon.mean.nc', 'pres')
gf.transform_to_anomalies()
gf.normalize_monthly_variance()
# gf.slice_spatial(None, [20, 89])
gf.slice_date_range(date(1950, 1, 1), date(2012, 3, 1))
# unroll the data
ts = gf.data()
# load the monthly NAO index
nao = np.loadtxt('data/nao_index.tim.txt', skiprows = 0)
naoh = np.loadtxt('data/nao_index_hurrel.tim.txt', skiprows = 0)
naoh_ndx = naoh[:, 2]
from datetime import datetime, date
from surr_geo_field_ar import SurrGeoFieldAR
from geo_field import GeoField
from multiprocessing import Pool
# load netCDF SLP field
d = GeoField()
d.load("/home/martin/Work/Geo/data/netcdf/pres.mon.mean.nc", 'pres')
d.slice_date_range(date(1948, 1, 1), date(2012, 1, 1))
#d.slice_months([12, 1, 2])
d.slice_spatial(None, [-89, 89])
# copy into surrogate field
sd = SurrGeoFieldAR()
sd.copy_field(d)
# create the Pool
pool = Pool(4)
t1 = datetime.now()
sd.prepare_surrogates(pool)
print("Prep: elapsed time %s" % str(datetime.now() - t1))
t1 = datetime.now()
sd.construct_surrogate()
print("Gen: elapsed time %s" % str(datetime.now() - t1))
import numpy as np
import fastcluster
from scipy.cluster.hierarchy import dendrogram, fcluster
#
# Current evaluation params
#
POOL_SIZE = None
if __name__ == "__main__":
print("Visualization of correlation between neighbors 1.0")
print("Loading data ...")
gf = GeoField()
gf.load("data/pres.mon.mean.nc", 'pres')
gf.transform_to_anomalies()
gf.slice_date_range(date(1948, 1, 1), date(2012, 1, 1))
gf.slice_spatial(None, [20, 89])
# initialize a parallel pool
pool = Pool(POOL_SIZE)
# get the correlation
num_lats, num_lons = gf.spatial_dims()
num_gpoints = num_lats * num_lons
dfC = np.zeros([1, (num_lats - 1) * 2, num_lons], dtype = np.float64)
# compute the neighbor correlations
y = np.ones(shape=(num_gpoints, num_gpoints)) * 10e6
def load_daily_data_general(fname, varname, from_date, to_date,
slice_lon, slice_lat, level):
# the daily data is stored in yearly files
yr_start = from_date.year
yr_end = to_date.year
# load each NC dataset
gflist = []
Ndays = 0
for yr in range(yr_start, yr_end+1):
g = GeoField()
g.load(fname % yr, varname)
if level is not None:
g.slice_level(level)
g.slice_spatial(slice_lon, slice_lat)
g.slice_date_range(from_date, to_date)
Ndays += len(g.tm)
gflist.append(g)
# now append all of the records together
g = GeoField()
d = np.zeros((Ndays, len(gflist[0].lats), len(gflist[0].lons)))
tm = np.zeros((Ndays,))
n = 0
for g_i in gflist:
Ndays_i = len(g_i.tm)
d[n:Ndays_i + n, :, :] = g_i.d
tm[n:Ndays_i + n] = g_i.tm
n += Ndays_i
# load geo_fields and then append them together
g.use_existing(d, gflist[0].lons, gflist[0].lats, tm)
g.transform_to_anomalies()
g.normalize_variance()
return g
def load_monthly_data_general(fname, varname, from_date, to_date, months, slice_lon, slice_lat, level, var_norm = True):
g = GeoField()
g.load(fname, varname)
if level is not None:
g.slice_level(level)
g.transform_to_anomalies()
if var_norm:
g.normalize_variance()
g.slice_spatial(slice_lon, slice_lat)
g.slice_date_range(from_date, to_date)
if months is not None:
g.slice_months(months)
return g
plt.figure()
plt.imshow(v.A, interpolation = 'nearest')
plt.colorbar()
plt.title('AR structural')
plt.figure()
plt.plot(ts)
plt.title('Simulated time series')
C = np.corrcoef(ts, None, rowvar = 0)
plt.figure()
plt.imshow(C, interpolation = 'nearest')
plt.title('Correlation matrix')
plt.colorbar()
gf = GeoField()
gf.lons = np.arange(S.shape[1])
gf.lats = np.arange(S.shape[0])
gf.tm = np.arange(768)
gf.d = np.reshape(ts, [768, S.shape[0], S.shape[1]])
plt.figure()
plt.plot(ts2[:,0])
plt.plot(gf.d[:, 0, 0])
with open('data/test_gf.bin', 'w') as f:
cPickle.dump(gf, f)
plt.show()
Ur, sr, _ = pca_components_gf(d)
# perm, sf = match_components_munkres(U, Ur)
# Ur = Ur[:, perm[:NUM_EIGS]]
# Ur *= sf
# return sr[perm[:NUM_EIGS]]
return sr[:NUM_EIGS], np.amax(np.abs(Ur[:, :NUM_EIGS]), axis = 0)
if __name__ == "__main__":
print("Estimate PCA components script version 1.0")
print("Loading data ...")
gf = GeoField()
gf.load("data/pres.mon.mean.nc", 'pres')
gf.transform_to_anomalies()
gf.normalize_monthly_variance()
gf.slice_date_range(date(1948, 1, 1), date(2012, 1, 1))
gf.slice_spatial(None, [20, 89])
# gf.slice_spatial(None, [-88, 88])
# S = np.zeros(shape = (5, 10), dtype = np.int32)
# S[1:4, 0:2] = 1
# S[0:3, 6:9] = 2
# v, Sr = constructVAR(S, [0.0, 0.7, 0.6], [-0.3, 0.3], [0.2, 0.27])
# ts = v.simulate(768)
# d = make_model_geofield(S, ts)
# S = np.zeros(shape = (20, 50), dtype = np.int32)
perm[m_i[0]] = m_i[1]
# flip the sign in the matched boostrap component if the correlation was negative
Ur[m_i[1]] = -Ur[m_i[1]] if C[m_i[0], m_i[1]] < 0.0 else Ur[m_i[1]]
# reorder the bootstrap components according to the best matching
Ur = Ur[:, perm]
return Ur
def render_set_par(x):
render_component_set(*x)
# load up the monthly SLP geo-field
gf = GeoField()
gf.load("data/pres.mon.mean.nc", 'pres')
gf.transform_to_anomalies()
gf.normalize_monthly_variance()
gf.slice_date_range(date(1948, 1, 1), date(2012, 1, 1)) # years 1948-2012
#gf.slice_spatial(None, [20, 87]) # northern hemisphere, extratropical
gf.slice_spatial(None, [-88, 88])
#gf.slice_months([12, 1, 2])
#S = np.zeros(shape = (5, 10), dtype = np.int32)
#S[1:4, 0:2] = 1
#S[0:3, 6:9] = 2
#v, Sr = constructVAR(S, [0.0, 0.191, 0.120], [-0.1, 0.1], [0.00, 0.00], [0.01, 0.01])
#ts = v.simulate(768)
#gf = make_model_geofield(S, ts)
from datetime import date
from geo_field import GeoField
from geo_rendering import render_component_single
import numpy as np
import cPickle
import matplotlib.pyplot as plt
import scipy.io as sio
if __name__ == '__main__':
# load geo-field
gf = GeoField()
gf.load('data/pres.mon.mean.nc', 'pres')
gf.transform_to_anomalies()
gf.normalize_monthly_variance()
# gf.slice_spatial(None, [20, 89])
gf.slice_date_range(date(1950, 1, 1), date(2012, 3, 1))
# unroll the data
ts = gf.data()
# load the monthly NAO index
nao = np.loadtxt('data/nao_index.tim.txt', skiprows=0)
naoh = np.loadtxt('data/nao_index_hurrel.tim.txt', skiprows=0)
naoh_ndx = naoh[:, 2]
nao_ndx = nao[:naoh_ndx.shape[0], 2]
print nao_ndx.shape
from datetime import date
from geo_field import GeoField
import numpy as np
import cPickle
import matplotlib.pyplot as plt
import scipy.io as sio
if __name__ == '__main__':
# load geo-field
gf = GeoField()
gf.load('data/pres.mon.mean.nc', 'pres')
gf.transform_to_anomalies()
gf.normalize_monthly_variance()
gf.slice_spatial(None, [20, 89])
gf.slice_date_range(date(1950, 1, 1), date(2012, 3, 1))
# load the components
with open('results/slp_nh_var_bootstrap_results_b1000_cosweights.bin', 'r') as f:
d = cPickle.load(f)
# convert to unit vectors
mn = d['mean']
mn = mn / np.sum(mn**2, axis = 0) ** 0.5
# mark maxima
mx_pos = np.argmax(mn**2, axis = 0)
def plot_nao_correlations():
# load geo-field
gf = GeoField()
gf.load('data/pres.mon.mean.nc', 'pres')
gf.transform_to_anomalies()
gf.normalize_monthly_variance()
gf.slice_spatial(None, [20, 89])
gf.slice_date_range(date(1950, 1, 1), date(2012, 3, 1))
with open(FILE_NAME_COMPS, 'r') as f:
d = cPickle.load(f)
# unroll the data
data = gf.data()
data = np.transpose(
np.reshape(data, (data.shape[0], data.shape[1] * data.shape[2])))
# load the monthly NAO index
nao = np.loadtxt('data/nao_index.tim.txt', skiprows=0)
naoh = np.loadtxt('data/nao_index_hurrel.tim.txt', skiprows=0)
naoh_ndx = naoh[:, 2]
nao_ndx = nao[:, 2]
print nao_ndx.shape
print naoh_ndx.shape
ts_len = min(len(nao_ndx), len(naoh_ndx))
nao_ndx = nao_ndx[:ts_len]
naoh_ndx = naoh_ndx[:ts_len]
mn = d['mean']
mn = mn / np.sum(mn**2, axis=0)**0.5
Nc = mn.shape[1]
ts = np.transpose(np.dot(mn.T, data))
ts = ts[:ts_len, :]
Cnao = np.zeros((Nc, ))
Cnaoh = np.zeros((Nc, ))
for i in range(Nc):
Cnao[i] = np.corrcoef(nao_ndx, ts[:, i], rowvar=False)[0, 1]
Cnaoh[i] = np.corrcoef(naoh_ndx, ts[:, i], rowvar=False)[0, 1]
f = plt.figure()
# plt.plot(nao_ndx, 'r-')
# plt.plot(naoh_ndx, 'b-')
plt.plot(np.arange(Nc) + 1, Cnao, 'ro-')
plt.plot(np.arange(Nc) + 1, Cnaoh, 'go-')
plt.legend(('NAO/PC', 'NAO/Stat.'))
plt.xlabel('Component index [-]')
plt.ylabel('NAO correlation [-]')
f.savefig('figs/slp_nh_nao_correlation.pdf')
print('Max station NAO correlation: %g at %d' %
(np.amax(np.abs(Cnaoh)), np.argmax(np.abs(Cnaoh))))
print('Max PC/NAO correlation: %g at %d' %
(np.amax(np.abs(Cnao)), np.argmax(np.abs(Cnao))))
def plot_nao_correlations():
# load geo-field
gf = GeoField()
gf.load('data/pres.mon.mean.nc', 'pres')
gf.transform_to_anomalies()
gf.normalize_monthly_variance()
gf.slice_spatial(None, [20, 89])
gf.slice_date_range(date(1950, 1, 1), date(2012, 3, 1))
with open(FILE_NAME_COMPS, 'r') as f:
d = cPickle.load(f)
# unroll the data
data = gf.data()
data = np.transpose(np.reshape(data, (data.shape[0], data.shape[1] * data.shape[2])))
# load the monthly NAO index
nao = np.loadtxt('data/nao_index.tim.txt', skiprows = 0)
naoh = np.loadtxt('data/nao_index_hurrel.tim.txt', skiprows = 0)
naoh_ndx = naoh[:, 2]
nao_ndx = nao[:, 2]
print nao_ndx.shape
print naoh_ndx.shape
ts_len = min(len(nao_ndx), len(naoh_ndx))
nao_ndx = nao_ndx[:ts_len]
naoh_ndx = naoh_ndx[:ts_len]
mn = d['mean']
mn = mn / np.sum(mn**2, axis = 0) ** 0.5
Nc = mn.shape[1]
ts = np.transpose(np.dot(mn.T, data))
ts = ts[:ts_len, :]
Cnao = np.zeros((Nc,))
Cnaoh = np.zeros((Nc,))
for i in range(Nc):
Cnao[i] = np.corrcoef(nao_ndx, ts[:, i], rowvar = False)[0,1]
Cnaoh[i] = np.corrcoef(naoh_ndx, ts[:, i], rowvar = False)[0,1]
f = plt.figure()
# plt.plot(nao_ndx, 'r-')
# plt.plot(naoh_ndx, 'b-')
plt.plot(np.arange(Nc) + 1, Cnao, 'ro-')
plt.plot(np.arange(Nc) + 1, Cnaoh, 'go-')
plt.legend(('NAO/PC', 'NAO/Stat.'))
plt.xlabel('Component index [-]')
plt.ylabel('NAO correlation [-]')
f.savefig('figs/slp_nh_nao_correlation.pdf')
print('Max station NAO correlation: %g at %d' % (np.amax(np.abs(Cnaoh)), np.argmax(np.abs(Cnaoh))))
print('Max PC/NAO correlation: %g at %d' % (np.amax(np.abs(Cnao)), np.argmax(np.abs(Cnao))))
perm[m_i[0]] = m_i[1]
# flip the sign in the matched boostrap component if the correlation was negative
Ur[m_i[1]] = - Ur[m_i[1]] if C[m_i[0], m_i[1]] < 0.0 else Ur[m_i[1]]
# reorder the bootstrap components according to the best matching
Ur = Ur[:, perm]
return Ur
def render_set_par(x):
render_component_set(*x)
# load up the monthly SLP geo-field
gf = GeoField()
gf.load("data/pres.mon.mean.nc", 'pres')
gf.transform_to_anomalies()
gf.normalize_monthly_variance()
gf.slice_date_range(date(1948, 1, 1), date(2012, 1, 1)) # years 1948-2012
#gf.slice_spatial(None, [20, 87]) # northern hemisphere, extratropical
gf.slice_spatial(None, [-88, 88])
#gf.slice_months([12, 1, 2])
#S = np.zeros(shape = (5, 10), dtype = np.int32)
#S[1:4, 0:2] = 1
#S[0:3, 6:9] = 2
#v, Sr = constructVAR(S, [0.0, 0.191, 0.120], [-0.1, 0.1], [0.00, 0.00], [0.01, 0.01])
#ts = v.simulate(768)
#gf = make_model_geofield(S, ts)
from datetime import datetime, date
from surr_geo_field_ar import SurrGeoFieldAR
from geo_field import GeoField
from multiprocessing import Pool
# load netCDF SLP field
d = GeoField()
d.load("/home/martin/Work/Geo/data/netcdf/pres.mon.mean.nc", 'pres')
d.slice_date_range(date(1948, 1, 1), date(2012, 1, 1))
#d.slice_months([12, 1, 2])
d.slice_spatial(None, [-89, 89])
# copy into surrogate field
sd = SurrGeoFieldAR()
sd.copy_field(d)
# create the Pool
pool = Pool(4)
t1 = datetime.now()
sd.prepare_surrogates(pool)
print("Prep: elapsed time %s" % str(datetime.now() - t1))
t1 = datetime.now()
sd.construct_surrogate()
print("Gen: elapsed time %s" % str(datetime.now() - t1))
t1 = datetime.now()
sd.construct_surrogate()
print("Gen: elapsed time %s" % str(datetime.now() - t1))
# -*- coding: utf-8 -*-
"""
Created on Thu Mar 1 11:17:39 2012
@author: martin
"""
from datetime import date
from geo_field import GeoField
from var_model import VARModel
from geo_rendering import render_component_single
import matplotlib.pyplot as plt
d = GeoField()
d.load("data/pres.mon.mean.nc", 'pres')
d.transform_to_anomalies()
d.normalize_monthly_variance()
d.slice_date_range(date(1948, 1, 1), date(2012, 1, 1))
#d.slice_months([12, 1, 2])
d.slice_spatial(None, [-89, 89])
render_component_single(d.d[0, :, :], d.lats, d.lons, False, None, 'SLP anomalies Jan 1948')
render_component_single(d.d[-1, :, :], d.lats, d.lons, False, None, 'SLP anomalies Jan 2012')
plt.show()
