def load_enso_SSTs(CMIP5model = None, num_ts = None, PROmodel = False):
# load enso SSTs
print("[%s] Loading monthly ENSO SSTs..." % str(datetime.now()))
enso_raw = np.loadtxt("nino34m13.txt") # length x 2 as 1st column is continuous year, second is SST in degC
enso = DataField()
enso.data = enso_raw[:, 1]
time = np.zeros_like(enso.data, dtype = np.int32)
y = np.int(enso_raw[0, 0])
start_date = date(y, 1, 1)
delta = relativedelta(months = +1)
d = start_date
for i in range(time.shape[0]):
time[i] = d.toordinal()
d += delta
enso.time = time
enso.location = 'NINO3.4 SSTs'
if CMIP5model is not None and num_ts is not None:
print("Loading %s model, time series %d..." % (CMIP5model, num_ts))
fname = CMIP5model + '.txt'
model = np.loadtxt('N34_CMIP5/' + fname)
enso.data = model[:, num_ts]
# varnorm, not seasonal
enso.data /= np.std(enso.data, axis = 0, ddof = 1)
if PROmodel:
print("[%s] Integrating PRO model which will be used instead of ENSO SSTs..." % (str(datetime.now())))
from parametric_recharge_oscillator import ENSO_PROmodel
PROmodel_enso = ENSO_PROmodel(length = enso.data.shape[0], daily = False, damped = True, ENSOperiod = 3.75, modulation = 2, lambda0 = 0.4)
PROmodel_enso.integrate_PROmodel()
enso.data = PROmodel_enso.data.copy()
if NUM_SURR > 0:
a = list(enso.get_seasonality(DETREND = False))
enso_sg = SurrogateField()
_, _, idx = enso.get_data_of_precise_length(length = 1024, end_date = date(2014, 1, 1), copy = False)
enso_sg.copy_field(enso)
enso_sg.data = enso_sg.data[idx[0]:idx[1]]
enso.return_seasonality(a[0], a[1], None)
a[0] = a[0][idx[0]:idx[1]]
a[1] = a[1][idx[0]:idx[1]]
# select 1024 data points
enso.get_data_of_precise_length(length = 1024, end_date = date(2014, 1, 1), copy = True)
print("[%s] Data loaded with shape %s" % (str(datetime.now()), enso.data.shape))
return enso, enso_sg, a
def load_enso_index(fname, nino_type, start_date, end_date, anom = False):
"""
Data loader for various ENSO indices.
"""
from dateutil.relativedelta import relativedelta
g = DataField()
enso_raw = np.loadtxt(fname, skiprows = 1)
y = int(enso_raw[0,0])
start_date_ts = date(y, 1, 1)
enso_raw = enso_raw[:, 1:]
enso_raw = enso_raw.reshape(np.prod(enso_raw.shape))
g.data = enso_raw.copy()
time = np.zeros_like(enso_raw, dtype = np.int32)
delta = relativedelta(months = +1)
d = start_date_ts
for i in range(time.shape[0]):
time[i] = d.toordinal()
d += delta
g.time = time.copy()
g.location = ('NINO%s SSTs' % nino_type)
print("** loaded")
g.select_date(start_date, end_date)
if anom:
print("** anomalising")
g.anomalise()
_, month, year = g.extract_day_month_year()
print("[%s] Nino%s data loaded with shape %s. Date range is %d/%d - %d/%d inclusive."
% (str(datetime.now()), nino_type, str(g.data.shape), month[0],
year[0], month[-1], year[-1]))
return g
amplitude = m * amplitude + c
print("Oscillatory series fitted to SAT data with coeff. %.3f and intercept %.3f" % (m, c))
cond_means = np.zeros((BINS, 2, 1))
def get_equidistant_bins(num):
return np.array(np.linspace(-np.pi, np.pi, num+1))
# start_cut = date(1962,1,1)
start_cut = date(1958,1,1)
# l = 17532
l = '16k'
g_data.data, g_data.time, idx = g.get_data_of_precise_length(l, start_cut, None, False) # 16k
print g_data.get_date_from_ndx(0), g_data.get_date_from_ndx(-1)
phase = phase[idx[0] : idx[1]]
if AMPLITUDE:
amplitude = amplitude[idx[0] : idx[1]]
phase_bins = get_equidistant_bins(BINS)
for i in range(cond_means.shape[0]):
ndx = ((phase >= phase_bins[i]) & (phase <= phase_bins[i+1]))
cond_means[i, 0, 0] = np.mean(g_data.data[ndx])
cond_means[i, 1, 0] = np.std(g_data.data[ndx], ddof = 1)
data_diff = cond_means[:, 0, 0].max() - cond_means[:, 0, 0].min()
def load_enso_SSTs(num_ts = None, PROmodel = False, EMRmodel = None, DDEmodel = None):
# load enso SSTs
print("[%s] Loading monthly ENSO SSTs..." % str(datetime.now()))
# enso_raw = np.loadtxt("nino34m13.txt") # length x 2 as 1st column is continuous year, second is SST in degC
# enso = DataField()
# enso = load_enso_index("nino34raw.txt", '3.4', date(1900, 1, 1), date(2001, 1, 1))
enso = DataField()
enso.data = np.loadtxt("nino34_CESMhigh.dat")
enso.create_time_array(date_from = date(1900, 1, 1), sampling = 'm')
# fname = "conceptualRossler1:2monthlysampling_100eps0-0.25.dat"
# r = read_rossler(fname)
# x, y = r[0.0707][20000:52768, 0], r[0.0707][20000:52768, 1] #
# x, y = r[0.1465][20000:21024, 0], r[0.1465][20000:21024, 1] # x is annual, y is biennal
# print("rossler data read")
# add noise
# x += np.random.normal(0, scale = 0.1*np.std(x, ddof = 1), size = (x.shape[0],))
# y += np.random.normal(0, scale = 0.1*np.std(y, ddof = 1), size = (y.shape[0],))
# print("added noise to rossler data...")
# exa = np.loadtxt("ExA-comb-mode-20CR-1900-2010-PC2-stand.txt")
# enso.data = exa.copy()
# enso.data = enso_raw[:, 1]
# enso.data = np.zeros((1200,))
# if '4k' in EMRmodel:
# enso.data = np.zeros((4096,))
# elif '8k' in EMRmodel:
# enso.data = np.zeros((8192,))
# elif '16k' in EMRmodel:
# enso.data = np.zeros((16384,))
# elif '32k' in EMRmodel:
# enso.data = np.zeros((32768,))
# enso.data = x.copy()
# enso.create_time_array(date(1900, 1, 1), sampling = 'm')
print enso.data.shape
print enso.get_date_from_ndx(0), enso.get_date_from_ndx(-1)
# enso.select_date(date(1884,1,1), date(2014,1,1))
# if CMIP5model is not None and num_ts is not None:
# fname = CMIP5model + '.txt'
# model = np.loadtxt('N34_CMIP5/' + fname)
# enso.data = model[:, num_ts]
# with open("SST-EOFanalysis-inputPCs.bin", "rb") as f:
# pcs = cPickle.load(f)
# pcs = pcs['PC']
# if num_ts != 2:
# enso.data = pcs[0, :]
# elif num_ts == 2:
# enso.data = pcs[3, :]
if PROmodel:
print("[%s] Integrating PRO model which will be used instead of ENSO SSTs..." % (str(datetime.now())))
from parametric_recharge_oscillator import ENSO_PROmodel
PROmodel_enso = ENSO_PROmodel(length = enso.data.shape[0], daily = False, damped = False, ENSOperiod = 3.75, modulation = 2., lambda0 = 0.4)
PROmodel_enso.integrate_PROmodel()
enso.data = PROmodel_enso.data.copy()
if DDEmodel is not None:
print("[%s] Integrating DDE model which will be used instead of ENSO SSTs..." % (str(datetime.now())))
from enso_dde_model import enso_dde_model
kappa, tau, b, subsample = DDEmodel
t, h = enso_dde_model(kappa = kappa, tau = tau, b = b, nyears = enso.time.shape[0]//12+10, subsample_to_monthly = subsample)
enso.data = h.copy()
if EMRmodel is not None:
print("[%s] Loading EMR (%s) simulated syntethic ENSO time series..." % (str(datetime.now()), EMRmodel))
# raw = sio.loadmat("Nino34-%s.mat" % (EMRmodel))['N34s']
raw = sio.loadmat("Nino34-linear-10PCsel-L3-seasonal-surrs.mat")['N34s']
enso.data = raw[num_ts, -1332:] # same length as nino3.4 data
# if '4k' in EMRmodel:
# enso.data = raw[-4096:, num_ts].copy()
# elif '8k' in EMRmodel:
# enso.data = raw[-8192:, num_ts].copy()
# elif '16k' in EMRmodel:
# enso.data = raw[-16384:, num_ts].copy()
# elif '32k' in EMRmodel:
# enso.data = raw[-32768:, num_ts].copy()
# if num_ts%3 == 0:
# dat = enso.get_date_from_ndx(12423)
# elif num_ts%3 == 1:
# dat = enso.get_date_from_ndx(2434)
# elif num_ts%3 == 2:
# dat = enso.get_date_from_ndx(7354)
# enso.get_data_of_precise_length(length = 1200, end_date = enso.get_date_from_ndx(-1), apply_to_data = True)
if NUM_SURR > 0:
a = list(enso.get_seasonality(detrend = False))
enso_sg = SurrogateField()
# _, _, idx = enso.get_data_of_precise_length(length = 1024, end_date = date(2014, 1, 1), COPY = False)
enso_sg.copy_field(enso)
# enso_sg.data = enso_sg.data[idx[0]:idx[1]]
enso.return_seasonality(a[0], a[1], None)
# a[0] = a[0][idx[0]:idx[1]]
# a[1] = a[1][idx[0]:idx[1]]
# select 1024 data points
# enso.get_data_of_precise_length(length = 1024, end_date = date(2014, 1, 1), COPY = True)
print("[%s] Data loaded with shape %s" % (str(datetime.now()), enso.data.shape))
return enso, enso_sg, a#, y
def load_ERA_data_daily(filename, varname, start_date, end_date, lats, lons, anom, parts = 1, logger_function = None):
"""
Data loader for daily ERA-40 / ERA-Interim data.
If more than one file, filename should be all letters they have got in common without suffix.
"""
if logger_function is None:
def logger(msg):
print("[%s] %s" % (str(datetime.now()), msg))
else:
logger = logger_function
logger("Loading daily ERA-40 / ERA-Interim data...")
# if in one file, just load it
if parts == 1:
path, name = split(filename)
if path != '':
path += '/'
g = DataField(data_folder = path)
else:
g = DataField()
g.load(name, varname, dataset = 'ERA', print_prog = False)
# if in more files, find them all and load them
else:
fnames = []
glist = []
Ndays = 0
path, name = split(filename)
if path != '':
path += '/'
else:
path = '../data'
for root, _, files in os.walk(path):
if root == path:
for f in files:
if name in f:
fnames.append(f)
if parts != len(fnames):
logger("Something went wrong since %d files matching your filename were found instead of %d." % (len(fnames), parts))
raise Exception('Check your files and enter correct number of files you want to load.')
for f in fnames:
g = DataField(data_folder = path + '/')
g.load(f, varname, dataset = 'ERA', print_prog = False)
Ndays += g.time.shape[0]
glist.append(g)
data = np.zeros((Ndays, len(glist[0].lats), len(glist[0].lons)))
time = np.zeros((Ndays,))
n = 0
for g in glist:
Ndays_i = len(g.time)
data[n:Ndays_i + n, ...] = g.data
time[n:Ndays_i + n] = g.time
n += Ndays_i
g = DataField(data = data, lons = glist[0].lons, lats = glist[0].lats, time = time)
del glist
if not np.all(np.unique(g.time) == g.time):
logger('**WARNING: Some fields are overlapping, trying to fix this... (please note: experimental feature)')
doubles = []
for i in range(g.time.shape[0]):
if np.where(g.time == g.time[i])[0].shape[0] == 1:
# if there is one occurence of time value do nothing
pass
else:
# remember the indices of other occurences
doubles.append(np.where(g.time == g.time[i])[0][1:])
logger("... found %d multiple values (according to the time field)..." % (len(doubles)/4))
delete_mask = np.squeeze(np.array(doubles)) # mask with multiple indices
# time
g.time = np.delete(g.time, delete_mask)
# data
g.data = np.delete(g.data, delete_mask, axis = 0)
logger("** loaded")
g.select_date(start_date, end_date)
g.select_lat_lon(lats, lons)
g.average_to_daily()
if anom:
logger("** anomalising")
g.anomalise()
day, month, year = g.extract_day_month_year()
logger("ERA-40 / ERA-Interim data loaded with shape %s. Date range is %d.%d.%d - %d.%d.%d inclusive."
% (str(g.data.shape), day[0], month[0],
year[0], day[-1], month[-1], year[-1]))
return g