def __init__(self, no_levels, verbose = False):
"""
Init function.
"""
DataField.__init__(self)
self.no_levels = no_levels
self.low_freq = None
self.input_pcs = None
self.input_eofs = None
self.verbose = verbose
self.combined = False
self.delay_model = False
def __init__(self,
fname,
varname,
start_date,
end_date,
lats,
lons,
level=None,
dataset="NCEP",
sampling='monthly',
anom=False,
pickled=False,
verbose=False):
"""
Initialisation of the class.
"""
# if sampling == 'monthly':
# self.g = load_NCEP_data_monthly(fname, varname, start_date, end_date, None, None, None, anom)
# elif sampling == 'daily':
# self.g = load_NCEP_data_daily(fname, varname, start_date, end_date, None, None, None, anom)
DataField.__init__(self)
if not pickled:
self.load(fname, varname, dataset=dataset, print_prog=False)
else:
self.load_field(fname)
self.data_mask = None
self.var_name = varname
self.select_date(start_date, end_date)
self.select_lat_lon(lats, lons)
if level is not None:
self.select_level(level)
if anom:
self.anomalise()
day, month, year = self.extract_day_month_year()
if verbose:
print(
"[%s] NCEP data loaded with shape %s. Date range is %d.%d.%d - %d.%d.%d inclusive."
% (str(datetime.now()), str(self.data.shape), day[0], month[0],
year[0], day[-1], month[-1], year[-1]))
self.phase = None
self.amplitude = None
self.wave = None
self.adjacency_matrix = None
self.num_lats = self.lats.shape[0]
self.num_lons = self.lons.shape[0]
self.sampling = sampling
def load_enso_index(fname, nino_type, start_date, end_date, anom = False):
"""
Data loader for various ENSO indices.
"""
from dateutil.relativedelta import relativedelta
g = DataField()
enso_raw = np.loadtxt(fname, skiprows = 1)
y = int(enso_raw[0,0])
start_date_ts = date(y, 1, 1)
enso_raw = enso_raw[:, 1:]
enso_raw = enso_raw.reshape(np.prod(enso_raw.shape))
g.data = enso_raw.copy()
time = np.zeros_like(enso_raw, dtype = np.int32)
delta = relativedelta(months = +1)
d = start_date_ts
for i in range(time.shape[0]):
time[i] = d.toordinal()
d += delta
g.time = time.copy()
g.location = ('NINO%s SSTs' % nino_type)
print("** loaded")
g.select_date(start_date, end_date)
if anom:
print("** anomalising")
g.anomalise()
_, month, year = g.extract_day_month_year()
print("[%s] Nino%s data loaded with shape %s. Date range is %d/%d - %d/%d inclusive."
% (str(datetime.now()), nino_type, str(g.data.shape), month[0],
year[0], month[-1], year[-1]))
return g
def __init__(self, fname, varname, start_date, end_date, lats, lons, level = None, dataset = "NCEP", sampling = 'monthly', anom = False, pickled = False, verbose = False):
"""
Initialisation of the class.
"""
# if sampling == 'monthly':
# self.g = load_NCEP_data_monthly(fname, varname, start_date, end_date, None, None, None, anom)
# elif sampling == 'daily':
# self.g = load_NCEP_data_daily(fname, varname, start_date, end_date, None, None, None, anom)
DataField.__init__(self)
if not pickled:
self.load(fname, varname, dataset = dataset, print_prog = False)
else:
self.load_field(fname)
self.data_mask = None
self.var_name = varname
self.select_date(start_date, end_date)
self.select_lat_lon(lats, lons)
if level is not None:
self.select_level(level)
if anom:
self.anomalise()
day, month, year = self.extract_day_month_year()
if verbose:
print("[%s] NCEP data loaded with shape %s. Date range is %d.%d.%d - %d.%d.%d inclusive."
% (str(datetime.now()), str(self.data.shape), day[0], month[0],
year[0], day[-1], month[-1], year[-1]))
self.phase = None
self.amplitude = None
self.wave = None
self.adjacency_matrix = None
self.num_lats = self.lats.shape[0]
self.num_lons = self.lons.shape[0]
self.sampling = sampling
def load_enso_SSTs(CMIP5model = None, num_ts = None, PROmodel = False):
# load enso SSTs
print("[%s] Loading monthly ENSO SSTs..." % str(datetime.now()))
enso_raw = np.loadtxt("nino34m13.txt") # length x 2 as 1st column is continuous year, second is SST in degC
enso = DataField()
enso.data = enso_raw[:, 1]
time = np.zeros_like(enso.data, dtype = np.int32)
y = np.int(enso_raw[0, 0])
start_date = date(y, 1, 1)
delta = relativedelta(months = +1)
d = start_date
for i in range(time.shape[0]):
time[i] = d.toordinal()
d += delta
enso.time = time
enso.location = 'NINO3.4 SSTs'
if CMIP5model is not None and num_ts is not None:
print("Loading %s model, time series %d..." % (CMIP5model, num_ts))
fname = CMIP5model + '.txt'
model = np.loadtxt('N34_CMIP5/' + fname)
enso.data = model[:, num_ts]
# varnorm, not seasonal
enso.data /= np.std(enso.data, axis = 0, ddof = 1)
if PROmodel:
print("[%s] Integrating PRO model which will be used instead of ENSO SSTs..." % (str(datetime.now())))
from parametric_recharge_oscillator import ENSO_PROmodel
PROmodel_enso = ENSO_PROmodel(length = enso.data.shape[0], daily = False, damped = True, ENSOperiod = 3.75, modulation = 2, lambda0 = 0.4)
PROmodel_enso.integrate_PROmodel()
enso.data = PROmodel_enso.data.copy()
if NUM_SURR > 0:
a = list(enso.get_seasonality(DETREND = False))
enso_sg = SurrogateField()
_, _, idx = enso.get_data_of_precise_length(length = 1024, end_date = date(2014, 1, 1), copy = False)
enso_sg.copy_field(enso)
enso_sg.data = enso_sg.data[idx[0]:idx[1]]
enso.return_seasonality(a[0], a[1], None)
a[0] = a[0][idx[0]:idx[1]]
a[1] = a[1][idx[0]:idx[1]]
# select 1024 data points
enso.get_data_of_precise_length(length = 1024, end_date = date(2014, 1, 1), copy = True)
print("[%s] Data loaded with shape %s" % (str(datetime.now()), enso.data.shape))
return enso, enso_sg, a
def __init__(self, data=None):
DataField.__init__(self)
self.data = None
self.model_grid = None
self.original_data = data
# amp_to_plot_surr = amp_to_plot_surr[idx[0] : idx[1]]
phase_amp = phase_amp[idx[0] : idx[1]]
sg.data = sg.data[idx[0] : idx[1]]
cond_temp = np.zeros((BINS,2))
for i in range(cond_means.shape[0]):
ndx = ((phase_amp >= phase_bins[i]) & (phase_amp <= phase_bins[i+1]))
cond_temp[i,1] = np.mean(sg.data[ndx])
data_diff = cond_temp[:, 1].max() - cond_temp[:, 1].min()
resq.put([data_diff, np.mean(amplitude2)])
g = load_station_data('../data/ECAstation-TG/TG_STAID000027.txt', date(1958, 1, 1), date(2013, 11, 10), True)
g_amp = load_station_data('../data/ECAstation-TG/TG_STAID000027.txt', date(1958, 1, 1), date(2013, 11, 10), True)
g_data = DataField()
g.wavelet(PERIOD, period_unit='y', cut=None)
phase = g.phase.copy()
if AMPLITUDE:
g_amp.wavelet(8, period_unit='y', cut=None)
amplitude = g_amp.amplitude.copy()
phase_amp = g_amp.phase.copy()
# fitting oscillatory phase / amplitude to actual SAT
reconstruction = amplitude * np.cos(phase_amp)
fit_x = np.vstack([reconstruction, np.ones(reconstruction.shape[0])]).T
m, c = np.linalg.lstsq(fit_x, g_amp.data)[0]
amplitude = m * amplitude + c
print("Oscillatory series fitted to SAT data with coeff. %.3f and intercept %.3f" % (m, c))
from datetime import date
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
PERIOD = 8
BINS = 8
NUM_SURR = 1000
def get_equidistant_bins(num):
return np.array(np.linspace(-np.pi, np.pi, num+1))
g = load_station_data('../data/ECAstation-TG/TG_STAID000027.txt', date(1924,1,14), date(2013,10,1), anom=True)
g_amp = load_station_data('../data/ECAstation-TG/TG_STAID000027.txt', date(1924,1,14), date(2013, 10, 1), anom=False)
g_data = DataField()
mean, var, trend = g.get_seasonality(True)
mean2, var2, trend2 = g_amp.get_seasonality(True)
sg_amp = SurrogateField()
sg_amp.copy_field(g_amp)
sg = SurrogateField()
sg.copy_field(g)
g.return_seasonality(mean, var, trend)
g_amp.return_seasonality(mean2, var2, trend2)
g.wavelet(PERIOD, period_unit='y', cut=None)
# wave, _, _, _ = wavelet_analysis.continous_wavelet(g.data, 1, False, wavelet_analysis.morlet, dj = 0, s0 = s0, j1 = 0, k0 = k0) # perform wavelet
phase = g.phase.copy()
NUM_SURR = 1000
def get_equidistant_bins(num):
return np.array(np.linspace(-np.pi, np.pi, num + 1))
g = load_station_data('../data/ECAstation-TG/TG_STAID000027.txt',
date(1924, 1, 14),
date(2013, 10, 1),
anom=True)
g_amp = load_station_data('../data/ECAstation-TG/TG_STAID000027.txt',
date(1924, 1, 14),
date(2013, 10, 1),
anom=False)
g_data = DataField()
mean, var, trend = g.get_seasonality(True)
mean2, var2, trend2 = g_amp.get_seasonality(True)
sg_amp = SurrogateField()
sg_amp.copy_field(g_amp)
sg = SurrogateField()
sg.copy_field(g)
g.return_seasonality(mean, var, trend)
g_amp.return_seasonality(mean2, var2, trend2)
g.wavelet(PERIOD, period_unit='y', cut=None)
# wave, _, _, _ = wavelet_analysis.continous_wavelet(g.data, 1, False, wavelet_analysis.morlet, dj = 0, s0 = s0, j1 = 0, k0 = k0) # perform wavelet
phase = g.phase.copy()
def load_enso_SSTs(num_ts = None, PROmodel = False, EMRmodel = None, DDEmodel = None):
# load enso SSTs
print("[%s] Loading monthly ENSO SSTs..." % str(datetime.now()))
# enso_raw = np.loadtxt("nino34m13.txt") # length x 2 as 1st column is continuous year, second is SST in degC
# enso = DataField()
# enso = load_enso_index("nino34raw.txt", '3.4', date(1900, 1, 1), date(2001, 1, 1))
enso = DataField()
enso.data = np.loadtxt("nino34_CESMhigh.dat")
enso.create_time_array(date_from = date(1900, 1, 1), sampling = 'm')
# fname = "conceptualRossler1:2monthlysampling_100eps0-0.25.dat"
# r = read_rossler(fname)
# x, y = r[0.0707][20000:52768, 0], r[0.0707][20000:52768, 1] #
# x, y = r[0.1465][20000:21024, 0], r[0.1465][20000:21024, 1] # x is annual, y is biennal
# print("rossler data read")
# add noise
# x += np.random.normal(0, scale = 0.1*np.std(x, ddof = 1), size = (x.shape[0],))
# y += np.random.normal(0, scale = 0.1*np.std(y, ddof = 1), size = (y.shape[0],))
# print("added noise to rossler data...")
# exa = np.loadtxt("ExA-comb-mode-20CR-1900-2010-PC2-stand.txt")
# enso.data = exa.copy()
# enso.data = enso_raw[:, 1]
# enso.data = np.zeros((1200,))
# if '4k' in EMRmodel:
# enso.data = np.zeros((4096,))
# elif '8k' in EMRmodel:
# enso.data = np.zeros((8192,))
# elif '16k' in EMRmodel:
# enso.data = np.zeros((16384,))
# elif '32k' in EMRmodel:
# enso.data = np.zeros((32768,))
# enso.data = x.copy()
# enso.create_time_array(date(1900, 1, 1), sampling = 'm')
print enso.data.shape
print enso.get_date_from_ndx(0), enso.get_date_from_ndx(-1)
# enso.select_date(date(1884,1,1), date(2014,1,1))
# if CMIP5model is not None and num_ts is not None:
# fname = CMIP5model + '.txt'
# model = np.loadtxt('N34_CMIP5/' + fname)
# enso.data = model[:, num_ts]
# with open("SST-EOFanalysis-inputPCs.bin", "rb") as f:
# pcs = cPickle.load(f)
# pcs = pcs['PC']
# if num_ts != 2:
# enso.data = pcs[0, :]
# elif num_ts == 2:
# enso.data = pcs[3, :]
if PROmodel:
print("[%s] Integrating PRO model which will be used instead of ENSO SSTs..." % (str(datetime.now())))
from parametric_recharge_oscillator import ENSO_PROmodel
PROmodel_enso = ENSO_PROmodel(length = enso.data.shape[0], daily = False, damped = False, ENSOperiod = 3.75, modulation = 2., lambda0 = 0.4)
PROmodel_enso.integrate_PROmodel()
enso.data = PROmodel_enso.data.copy()
if DDEmodel is not None:
print("[%s] Integrating DDE model which will be used instead of ENSO SSTs..." % (str(datetime.now())))
from enso_dde_model import enso_dde_model
kappa, tau, b, subsample = DDEmodel
t, h = enso_dde_model(kappa = kappa, tau = tau, b = b, nyears = enso.time.shape[0]//12+10, subsample_to_monthly = subsample)
enso.data = h.copy()
if EMRmodel is not None:
print("[%s] Loading EMR (%s) simulated syntethic ENSO time series..." % (str(datetime.now()), EMRmodel))
# raw = sio.loadmat("Nino34-%s.mat" % (EMRmodel))['N34s']
raw = sio.loadmat("Nino34-linear-10PCsel-L3-seasonal-surrs.mat")['N34s']
enso.data = raw[num_ts, -1332:] # same length as nino3.4 data
# if '4k' in EMRmodel:
# enso.data = raw[-4096:, num_ts].copy()
# elif '8k' in EMRmodel:
# enso.data = raw[-8192:, num_ts].copy()
# elif '16k' in EMRmodel:
# enso.data = raw[-16384:, num_ts].copy()
# elif '32k' in EMRmodel:
# enso.data = raw[-32768:, num_ts].copy()
# if num_ts%3 == 0:
# dat = enso.get_date_from_ndx(12423)
# elif num_ts%3 == 1:
# dat = enso.get_date_from_ndx(2434)
# elif num_ts%3 == 2:
# dat = enso.get_date_from_ndx(7354)
# enso.get_data_of_precise_length(length = 1200, end_date = enso.get_date_from_ndx(-1), apply_to_data = True)
if NUM_SURR > 0:
a = list(enso.get_seasonality(detrend = False))
enso_sg = SurrogateField()
# _, _, idx = enso.get_data_of_precise_length(length = 1024, end_date = date(2014, 1, 1), COPY = False)
enso_sg.copy_field(enso)
# enso_sg.data = enso_sg.data[idx[0]:idx[1]]
enso.return_seasonality(a[0], a[1], None)
# a[0] = a[0][idx[0]:idx[1]]
# a[1] = a[1][idx[0]:idx[1]]
# select 1024 data points
# enso.get_data_of_precise_length(length = 1024, end_date = date(2014, 1, 1), COPY = True)
print("[%s] Data loaded with shape %s" % (str(datetime.now()), enso.data.shape))
return enso, enso_sg, a#, y
def load_ERA_data_daily(filename, varname, start_date, end_date, lats, lons, anom, parts = 1, logger_function = None):
"""
Data loader for daily ERA-40 / ERA-Interim data.
If more than one file, filename should be all letters they have got in common without suffix.
"""
if logger_function is None:
def logger(msg):
print("[%s] %s" % (str(datetime.now()), msg))
else:
logger = logger_function
logger("Loading daily ERA-40 / ERA-Interim data...")
# if in one file, just load it
if parts == 1:
path, name = split(filename)
if path != '':
path += '/'
g = DataField(data_folder = path)
else:
g = DataField()
g.load(name, varname, dataset = 'ERA', print_prog = False)
# if in more files, find them all and load them
else:
fnames = []
glist = []
Ndays = 0
path, name = split(filename)
if path != '':
path += '/'
else:
path = '../data'
for root, _, files in os.walk(path):
if root == path:
for f in files:
if name in f:
fnames.append(f)
if parts != len(fnames):
logger("Something went wrong since %d files matching your filename were found instead of %d." % (len(fnames), parts))
raise Exception('Check your files and enter correct number of files you want to load.')
for f in fnames:
g = DataField(data_folder = path + '/')
g.load(f, varname, dataset = 'ERA', print_prog = False)
Ndays += g.time.shape[0]
glist.append(g)
data = np.zeros((Ndays, len(glist[0].lats), len(glist[0].lons)))
time = np.zeros((Ndays,))
n = 0
for g in glist:
Ndays_i = len(g.time)
data[n:Ndays_i + n, ...] = g.data
time[n:Ndays_i + n] = g.time
n += Ndays_i
g = DataField(data = data, lons = glist[0].lons, lats = glist[0].lats, time = time)
del glist
if not np.all(np.unique(g.time) == g.time):
logger('**WARNING: Some fields are overlapping, trying to fix this... (please note: experimental feature)')
doubles = []
for i in range(g.time.shape[0]):
if np.where(g.time == g.time[i])[0].shape[0] == 1:
# if there is one occurence of time value do nothing
pass
else:
# remember the indices of other occurences
doubles.append(np.where(g.time == g.time[i])[0][1:])
logger("... found %d multiple values (according to the time field)..." % (len(doubles)/4))
delete_mask = np.squeeze(np.array(doubles)) # mask with multiple indices
# time
g.time = np.delete(g.time, delete_mask)
# data
g.data = np.delete(g.data, delete_mask, axis = 0)
logger("** loaded")
g.select_date(start_date, end_date)
g.select_lat_lon(lats, lons)
g.average_to_daily()
if anom:
logger("** anomalising")
g.anomalise()
day, month, year = g.extract_day_month_year()
logger("ERA-40 / ERA-Interim data loaded with shape %s. Date range is %d.%d.%d - %d.%d.%d inclusive."
% (str(g.data.shape), day[0], month[0],
year[0], day[-1], month[-1], year[-1]))
return g
def load_NCEP_data_monthly(filename, varname, start_date, end_date, lats, lons, level, anom):
"""
Data loader for monthly reanalyses data.
"""
print("[%s] Loading monthly NCEP/NCAR data..." % str(datetime.now()))
g = DataField()
g.load(filename, varname, dataset = 'NCEP', print_prog = False)
print("** loaded")
g.select_date(start_date, end_date)
g.select_lat_lon(lats, lons)
if level is not None:
g.select_level(level)
if anom:
print("** anomalising")
g.anomalise()
day, month, year = g.extract_day_month_year()
print("[%s] NCEP data loaded with shape %s. Date range is %d.%d.%d - %d.%d.%d inclusive."
% (str(datetime.now()), str(g.data.shape), day[0], month[0],
year[0], day[-1], month[-1], year[-1]))
return g
def load_NCEP_data_daily(filename, varname, start_date, end_date, lats, lons, level, anom):
"""
Data loader for daily reanalyses data. Filename in form path/air.sig995.%d.nc
"""
print("[%s] Loading daily NCEP/NCAR data..." % str(datetime.now()))
start_year = start_date.year
end_year = end_date.year - 1
glist = []
Ndays = 0
path, name = split(filename)
path += "/"
for year in range(start_year, end_year+1):
g = DataField(data_folder = path)
fname = name % year
g.load(fname, varname, dataset = 'NCEP', print_prog = False)
Ndays += len(g.time)
glist.append(g)
if glist[0].data.ndim == 3:
data = np.zeros((Ndays, len(glist[0].lats), len(glist[0].lons)))
else:
data = np.zeros((Ndays, len(glist[0].level), len(glist[0].lats), len(glist[0].lons)))
time = np.zeros((Ndays,))
n = 0
for g in glist:
Ndays_i = len(g.time)
data[n:Ndays_i + n, ...] = g.data
time[n:Ndays_i + n] = g.time
n += Ndays_i
g = DataField(data = data, lons = glist[0].lons, lats = glist[0].lats, time = time)
del glist
print("** loaded")
g.select_date(start_date, end_date)
g.select_lat_lon(lats, lons)
if level is not None:
g.select_level(level)
if anom:
print("** anomalising")
g.anomalise()
day, month, year = g.extract_day_month_year()
print("[%s] NCEP data loaded with shape %s. Date range is %d.%d.%d - %d.%d.%d inclusive."
% (str(datetime.now()), str(g.data.shape), day[0], month[0],
year[0], day[-1], month[-1], year[-1]))
return g
def load_ECA_D_data_daily(filename, varname, start_date, end_date, lats, lons, anom, logger_function = None):
"""
Data loader for daily ECA&D reanalysis data.
"""
if logger_function is None:
def logger(msg):
print("[%s] %s" % (str(datetime.now()), msg))
else:
logger = logger_function
logger("Loading daily ECA&D data...")
g = DataField()
g.load(filename, varname, dataset = 'ECA-reanalysis', print_prog = False)
logger("** loaded")
g.select_date(start_date, end_date)
g.select_lat_lon(lats, lons)
if anom:
logger("** anomalising")
g.anomalise()
day, month, year = g.extract_day_month_year()
logger("ECA&D data loaded with shape %s. Date range is %d.%d.%d - %d.%d.%d inclusive."
% (str(g.data.shape), day[0], month[0],
year[0], day[-1], month[-1], year[-1]))
return g
def load_station_data(filename, start_date, end_date, anom, to_monthly=False, dataset='ECA-station', offset=1):
"""
Data loader for station data.
"""
print("[%s] Loading station data..." % (str(datetime.now())))
g = DataField()
g.load_station_data(filename, dataset, print_prog=False, offset_in_file=offset)
print("** loaded")
g.select_date(start_date, end_date)
if anom:
print("** anomalising")
g.anomalise()
if to_monthly:
g.get_monthly_data()
day, month, year = g.extract_day_month_year()
print("[%s] Data from %s loaded with shape %s. Date range is %d.%d.%d - %d.%d.%d inclusive."
% (str(datetime.now()), g.location, str(g.data.shape), day[0], month[0],
year[0], day[-1], month[-1], year[-1]))
return g