def load_neutron_NESDIS_data(fname, start_date, end_date, anom=True):
raw = np.loadtxt(fname, skiprows=2)
data = []
time = []
for year in range(raw.shape[0]):
for month in range(1, 13):
dat = float(raw[year, month])
if dat == 9999.:
dat = (float(raw[year, month - 2]) + float(
raw[year, month - 1]) + float(raw[year, month + 1]) +
float(raw[year, month + 2])) / 4.
data.append(dat)
time.append(date(int(raw[year, 0]), month, 1).toordinal())
g = DataField(data=np.array(data), time=np.array(time))
g.location = ('%s cosmic data' % (fname[32].upper() + fname[33:-4]))
g.select_date(start_date, end_date)
if anom:
g.anomalise()
if NUM_SURR != 0:
g_surr = SurrogateField()
seasonality = g.get_seasonality()
g_surr.copy_field(g)
g.return_seasonality(seasonality[0], seasonality[1], None)
else:
g_surr, seasonality = None, None
return g, g_surr, seasonality
def load_CR_climax_daily_data(fname, start_date, end_date, anom=False):
from dateutil.relativedelta import relativedelta
raw = np.loadtxt(fname)
time = []
datenow = date(1994, 1, 1)
delta = timedelta(days=1)
for t in range(raw.shape[0]):
time.append(datenow.toordinal())
datenow += delta
print raw.shape
print len(time)
g = DataField(data=np.array(raw), time=np.array(time))
g.location = 'Climax, CO cosmic data'
g.select_date(start_date, end_date)
if anom:
g.anomalise()
if NUM_SURR != 0:
g_surr = SurrogateField()
seasonality = g.get_seasonality(True)
g_surr.copy_field(g)
g.return_seasonality(seasonality[0], seasonality[1], seasonality[2])
else:
g_surr, seasonality = None, None
return g, g_surr, seasonality
def load_neutron_NESDIS_data(fname, start_date, end_date, anom = True):
raw = np.loadtxt(fname, skiprows = 2)
data = []
time = []
for year in range(raw.shape[0]):
for month in range(1,13):
dat = float(raw[year, month])
if dat == 9999.:
dat = (float(raw[year, month-2]) + float(raw[year, month-1]) + float(raw[year, month+1]) + float(raw[year, month+2])) / 4.
data.append(dat)
time.append(date(int(raw[year,0]), month, 1).toordinal())
g = DataField(data = np.array(data), time = np.array(time))
g.location = ('%s cosmic data' % (fname[32].upper() + fname[33:-4]))
g.select_date(start_date, end_date)
if anom:
g.anomalise()
if NUM_SURR != 0:
g_surr = SurrogateField()
seasonality = g.get_seasonality()
g_surr.copy_field(g)
g.return_seasonality(seasonality[0], seasonality[1], None)
else:
g_surr, seasonality = None, None
return g, g_surr, seasonality
def load_CR_climax_daily_data(fname, start_date, end_date, anom = False):
from dateutil.relativedelta import relativedelta
raw = np.loadtxt(fname)
time = []
datenow = date(1994, 1, 1)
delta = timedelta(days = 1)
for t in range(raw.shape[0]):
time.append(datenow.toordinal())
datenow += delta
print raw.shape
print len(time)
g = DataField(data = np.array(raw), time = np.array(time))
g.location = 'Climax, CO cosmic data'
g.select_date(start_date, end_date)
if anom:
g.anomalise()
if NUM_SURR != 0:
g_surr = SurrogateField()
seasonality = g.get_seasonality(True)
g_surr.copy_field(g)
g.return_seasonality(seasonality[0], seasonality[1], seasonality[2])
else:
g_surr, seasonality = None, None
return g, g_surr, seasonality
def load_cosmic_data(fname,
start_date,
end_date,
anom=True,
daily=False,
corrected=True):
# corrected stands for if use corrected data or not
from dateutil.relativedelta import relativedelta
raw = open(fname).read()
lines = raw.split('\n')
data = []
time = []
d = date(int(lines[0][:4]), int(lines[0][5:7]), 1)
if not daily:
delta = relativedelta(months=+1)
elif daily:
delta = timedelta(days=1)
for line in lines:
row = line.split(' ')
if len(row) < 6:
continue
time.append(d.toordinal())
if corrected:
data.append(float(row[4]))
else:
data.append(float(row[5]))
d += delta
g = DataField(data=np.array(data), time=np.array(time))
g.location = 'Oulu cosmic data'
g.select_date(start_date, end_date)
if anom:
g.anomalise()
if NUM_SURR != 0:
g_surr = SurrogateField()
seasonality = g.get_seasonality(True)
g_surr.copy_field(g)
g.return_seasonality(seasonality[0], seasonality[1], seasonality[2])
else:
g_surr, seasonality = None, None
return g, g_surr, seasonality
def load_nino34_wavelet_phase(start_date, end_date, anom=True):
raw = np.loadtxt('/home/nikola/Work/phd/data/nino34monthly.txt')
data = []
time = []
for y in range(raw.shape[0]):
for m in range(1, 13):
dat = float(raw[y, m])
data.append(dat)
time.append(date(int(raw[y, 0]), m, 1).toordinal())
g = DataField(data=np.array(data), time=np.array(time))
g.location = "NINO3.4"
g.select_date(start_date, end_date)
if anom:
g.anomalise()
k0 = 6. # wavenumber of Morlet wavelet used in analysis
fourier_factor = (4 * np.pi) / (k0 + np.sqrt(2 + np.power(k0, 2)))
per = PERIOD * 12 # frequency of interest
s0 = per / fourier_factor # get scale
wave, _, _, _ = wvlt.continous_wavelet(g.data,
1,
False,
wvlt.morlet,
dj=0,
s0=s0,
j1=0,
k0=6.)
phase = np.arctan2(np.imag(wave), np.real(wave))[0, :]
return phase
def load_cosmic_data(fname, start_date, end_date, anom = True, daily = False, corrected = True):
# corrected stands for if use corrected data or not
from dateutil.relativedelta import relativedelta
raw = open(fname).read()
lines = raw.split('\n')
data = []
time = []
d = date(int(lines[0][:4]), int(lines[0][5:7]), 1)
if not daily:
delta = relativedelta(months = +1)
elif daily:
delta = timedelta(days = 1)
for line in lines:
row = line.split(' ')
if len(row) < 6:
continue
time.append(d.toordinal())
if corrected:
data.append(float(row[4]))
else:
data.append(float(row[5]))
d += delta
g = DataField(data = np.array(data), time = np.array(time))
g.location = 'Oulu cosmic data'
g.select_date(start_date, end_date)
if anom:
g.anomalise()
g.data = X[:, 0].copy()
if NUM_SURR != 0:
g_surr = SurrogateField()
seasonality = g.get_seasonality(True)
g_surr.copy_field(g)
g.return_seasonality(seasonality[0], seasonality[1], seasonality[2])
else:
g_surr, seasonality = None, None
return g, g_surr, seasonality
def load_nino34_wavelet_phase(start_date, end_date, anom = True):
raw = np.loadtxt('/home/nikola/Work/phd/data/nino34monthly.txt')
data = []
time = []
for y in range(raw.shape[0]):
for m in range(1,13):
dat = float(raw[y, m])
data.append(dat)
time.append(date(int(raw[y, 0]), m, 1).toordinal())
g = DataField(data = np.array(data), time = np.array(time))
g.location = "NINO3.4"
g.select_date(start_date, end_date)
if anom:
g.anomalise()
k0 = 6. # wavenumber of Morlet wavelet used in analysis
fourier_factor = (4 * np.pi) / (k0 + np.sqrt(2 + np.power(k0,2)))
per = PERIOD * 12 # frequency of interest
s0 = per / fourier_factor # get scale
wave, _, _, _ = wvlt.continous_wavelet(g.data, 1, False, wvlt.morlet, dj = 0, s0 = s0, j1 = 0, k0 = 6.)
phase = np.arctan2(np.imag(wave), np.real(wave))[0, :]
return phase
def __init__(self, fname, varname, start_date, end_date, lats, lons, level = None, dataset = "NCEP", sampling = 'monthly', anom = False, pickled = False, verbose = False):
"""
Initialisation of the class.
"""
# if sampling == 'monthly':
# self.g = load_NCEP_data_monthly(fname, varname, start_date, end_date, None, None, None, anom)
# elif sampling == 'daily':
# self.g = load_NCEP_data_daily(fname, varname, start_date, end_date, None, None, None, anom)
DataField.__init__(self)
if not pickled:
self.load(fname, varname, dataset = dataset, print_prog = False)
else:
self.load_field(fname)
self.var_name = varname
self.select_date(start_date, end_date)
self.select_lat_lon(lats, lons)
if level is not None:
self.select_level(level)
if anom:
self.anomalise()
day, month, year = self.extract_day_month_year()
if verbose:
print("[%s] NCEP data loaded with shape %s. Date range is %d.%d.%d - %d.%d.%d inclusive."
% (str(datetime.now()), str(self.data.shape), day[0], month[0],
year[0], day[-1], month[-1], year[-1]))
self.phase = None
self.amplitude = None
self.wave = None
self.adjacency_matrix = None
self.num_lats = self.lats.shape[0]
self.num_lons = self.lons.shape[0]
self.sampling = sampling
net.get_continuous_phase(pool = pool)
net.get_phase_fluctuations(rewrite = True, pool = pool)
pool.close()
pool.join()
# index_correlations = {}
# index_datas = {}
# # SURROGATES
# for index, ndx_type, start_date, end_year in zip(INDICES, DATE_TYPE, START_DATES, END_YEARS):
# load index
# print index
# if index != 'NINO3.4':
index_data = DataField()
raw = np.loadtxt("%sNAO.station.monthly.1865-2016.txt" % (path_to_data))
raw = raw[:, 1:]
index_data.data = raw.reshape(-1)
index_data.create_time_array(date_from = date(1865, 1, 1), sampling = 'm')
index_data.select_date(date(1951, 1, 1), date(2014, 1, 1))
index_data.anomalise()
index_correlations = get_corrs(net, index_data)
# with open("20CRtemp-phase-fluct-corr-with-%sindex-1950-2014.bin" % index, "wb") as f:
# cPickle.dump({('%scorrs' % index) : index_correlations[index].reshape(np.prod(index_correlations[index].shape))}, f)
# # plotting
# tit = ("ECA&D annual phase SSA RC fluctuations x %s correlations" % index)
# fname = ("../scale-nets/ECAD-SAT-annual-phase-fluc-SSA-RC-%scorrs.png" % index)
# net.quick_render(field_to_plot = index_correlations[index], tit = tit, symm = True, whole_world = False, fname = fname)
PERIOD = 8
WINDOW_LENGTH = 13462 # 13462, 16384
MIDDLE_YEAR = 1965 # year around which the window will be deployed
DIFFS_LIM = [10, 25]
SAT = False
# load whole data - load SAT data
g = load_station_data('TG_STAID000027.txt', date(1775, 1, 1), date(2014, 1, 1),
False)
# save SAT data
tg_sat = g.copy_data()
# anomalise to obtain SATA data
g.anomalise()
g_temp = DataField()
# starting month and day
sm = 7
sd = 28
# starting year of final window
y = 365.25
sy = int(MIDDLE_YEAR - (WINDOW_LENGTH / y) / 2)
# get wvlt window
start = g.find_date_ndx(date(sy - 4, sm, sd))
end = start + 16384 if WINDOW_LENGTH < 16000 else start + 32768
g.data = g.data[start:end]
g.time = g.time[start:end]
tg_sat = tg_sat[start:end]
date(2015, 1, 1),
ANOMALISE) # 15-01-1924 if 32k, 28-04-1834 if 64k
if AMPLITUDE:
g_amp = load_station_data('../data/TG_STAID000027.txt', date(1775, 1, 1),
date(2015, 1, 1), False)
## HAMBURG -- TG_STAID000047, POTSDAM -- TG_STAID000054
# g = load_station_data('../data/TG_STAID000054.txt', date(1893,1,1), date(2014,1,1), ANOMALISE) # 15-01-1924 if 32k, 28-04-1834 if 64k
# if AMPLITUDE:
# g_amp = load_station_data('../data/TG_STAID000054.txt', date(1893,1,1), date(2014, 1, 1), False)
# ERA
#g = load_bin_data('../data/ERA_time_series_50.0N_15.0E.bin', date(1958,4,28), date(2013,10,1), ANOMALISE)
# ECA
#g = load_bin_data('../data/ECA&D_time_series_50.1N_14.4E.bin', date(1950,4,28), date(2013,10,1), ANOMALISE)
g_working = DataField()
g_surrs = DataField()
if AMPLITUDE:
g_working_amp = DataField()
g_surrs_amp = DataField()
if MOMENT == 'mean':
func = np.mean
if AMPLITUDE:
diff_ax = (0, 2) # means -> 0, 2, var -> 1, 8
mean_ax = (18, 22) # means -> -1, 1.5, var -> 9, 18
else:
diff_ax = (0, 5)
mean_ax = (-1, 1.5)
elif MOMENT == 'std':
func = np.var
diff_ax = (1, 15)
## -----------------
ANOMALISE = True
PERIOD = 8 # years, period of wavelet
WINDOW_LENGTH = 16384 # 13462, 16384
WINDOW_SHIFT = 1 # years, delta in the sliding window analysis
MEANS = True # if True, compute conditional means, if False, compute conditional variance
WORKERS = 4
NUM_SURR = 50 # how many surrs will be used to evaluate
SURR_TYPE = 'MF'
diff_ax = (0, 8) # means -> 0, 2, var -> 1, 8
mean_ax = (-1, 1) # means -> -1, 1.5, var -> 9, 18
g = load_station_data('TG_STAID000027.txt', date(1834, 7, 28),
date(2014, 1, 1), ANOMALISE)
g_working = DataField()
g_surrs = DataField()
TS_LEN = g.data.shape[0]
# map coeffs to numpy array
if RANDOM_COEFFS:
A_COEFFS = []
for i in range(k):
A_COEFFS.append((2 * np.random.rand(1) - 1)[0])
a_coeffs = np.array(A_COEFFS)
# initialize first k time points to noise
ts = np.zeros((TS_LEN, ))
for i in range(k):
ts[i] = np.random.normal(0, SIGMA_NOISE, 1)
# "grid" : "2.5/2.5",
# "time" : "00/06/12/18", ## daily
# "date" : "20010101/to/20131231",
# "area" : "50/-15/30/5", ## north/west/south/east
# "type" : "an",
# "class" : "e4",
# "format" : "netcdf",
# "padding" : "0",
# "target" : "test.nc"
# })
#==============================================================================
# load ERA-40 as g1 and ERA-Interim as g2
print("[%s] Loading data..." % (str(datetime.now())))
g1 = DataField()
g2 = DataField()
g1.load('Spain.ERA.58-01.nc', 't2m', 'ERA-40')
g2.load('Spain.ERA.01-13.nc', 't2m', 'ERA-40')
# concatenate
last = g1.time[-1]
idx = np.where(g2.time == last)[0] + 1
ndays = g1.time.shape[0] + g2.time[idx:].shape[0]
data = np.zeros((ndays, g1.lats.shape[0], g1.lons.shape[0]))
time = np.zeros((ndays, ))
data[:g1.time.shape[0], ...] = g1.data
data[g1.time.shape[0]:, ...] = g2.data[idx:]
net.get_continuous_phase(pool=pool)
net.get_phase_fluctuations(rewrite=True, pool=pool)
pool.close()
pool.join()
# index_correlations = {}
# index_datas = {}
# # SURROGATES
# for index, ndx_type, start_date, end_year in zip(INDICES, DATE_TYPE, START_DATES, END_YEARS):
# load index
# print index
# if index != 'NINO3.4':
index_data = DataField()
raw = np.loadtxt("%sNAO.station.monthly.1865-2016.txt" % (path_to_data))
raw = raw[:, 1:]
index_data.data = raw.reshape(-1)
index_data.create_time_array(date_from=date(1865, 1, 1), sampling='m')
index_data.select_date(date(1951, 1, 1), date(2014, 1, 1))
index_data.anomalise()
index_correlations = get_corrs(net, index_data)
# with open("20CRtemp-phase-fluct-corr-with-%sindex-1950-2014.bin" % index, "wb") as f:
# cPickle.dump({('%scorrs' % index) : index_correlations[index].reshape(np.prod(index_correlations[index].shape))}, f)
# # plotting
# tit = ("ECA&D annual phase SSA RC fluctuations x %s correlations" % index)
# fname = ("../scale-nets/ECAD-SAT-annual-phase-fluc-SSA-RC-%scorrs.png" % index)
# net.quick_render(field_to_plot = index_correlations[index], tit = tit, symm = True, whole_world = False, fname = fname)
#fontsize = 8.5)
#m.drawmeridians(np.arange(lons[0], lons[-1]+1, 10), dashes = [1,3], labels = [0,0,0,0], color = (.2, .2, .2),
#fontsize = 8.5)
plt.title(title)
cbar = plt.colorbar(format = r"%2.2f", shrink = 0.75, ticks = np.arange(mi, ma + step, (ma-mi)/8),
aspect = 25, drawedges = False)
cbar.set_label(cbar_label)
cbar_obj = plt.getp(cbar.ax.axes, 'yticklabels')
plt.setp(cbar_obj, fontsize = 10, color = (.1, .1, .1))
if filename != None:
plt.savefig(filename)
else:
plt.show()
g = DataField()
g.load('tg_0.25deg_reg_v9.0.nc', 'tg')
means = True
daily = False
if means:
idx = 0
y = 1950
while idx < g.data.shape[0]:
idx2 = g.find_date_ndx(date(y,1,1))
render_geo_field(g.data[idx:idx2, ...], g.lats, g.lons, None, None, False, 'Yearly mean temperature %s' % str(y), 'temperature [$^{\circ}C$]', 'imgs/temp_mean%s.png' % str(y))
y += 1
idx = idx2
if daily:
idx = g.find_date_ndx(date(2012,1,1))
SURR = True
NUM_SURR = 1000
WORKERS = 3
SURR_TYPE = 'FT'
# 65k
# g = load_station_data('TG_STAID000027.txt', date(1834,4,27), date(2013,10,1), True)
# g_amp = load_station_data('TG_STAID000027.txt', date(1834,4,27), date(2013,10,1), ANOMALISE)
# 32k
g = load_station_data('TG_STAID000027.txt', date(1958, 1,
1), date(2013, 11, 10),
True) # date(1924,1,14), date(2013,10,1)
g_amp = load_station_data('TG_STAID000027.txt', date(1958, 1, 1),
date(2013, 11, 10), ANOMALISE)
g_data = DataField()
k0 = 6. # wavenumber of Morlet wavelet used in analysis
y = 365.25 # year in days
fourier_factor = (4 * np.pi) / (k0 + np.sqrt(2 + np.power(k0, 2)))
period = PERIOD * y # frequency of interest
s0 = period / fourier_factor # get scale
# wavelet - data
wave, _, _, _ = wavelet_analysis.continous_wavelet(g.data,
1,
False,
wavelet_analysis.morlet,
dj=0,
s0=s0,
j1=0,
k0=k0) # perform wavelet
## -----------------
ANOMALISE = True
PERIOD = 8 # years, period of wavelet
WINDOW_LENGTH = 16384 # 13462, 16384
WINDOW_SHIFT = 1 # years, delta in the sliding window analysis
MEANS = True # if True, compute conditional means, if False, compute conditional variance
WORKERS = 4
NUM_SURR = 50 # how many surrs will be used to evaluate
SURR_TYPE = 'MF'
diff_ax = (0, 8) # means -> 0, 2, var -> 1, 8
mean_ax = (-1, 1) # means -> -1, 1.5, var -> 9, 18
g = load_station_data('TG_STAID000027.txt', date(1834,7,28), date(2014,1,1), ANOMALISE)
g_working = DataField()
g_surrs = DataField()
TS_LEN = g.data.shape[0]
# map coeffs to numpy array
if RANDOM_COEFFS:
A_COEFFS = []
for i in range(k):
A_COEFFS.append((2*np.random.rand(1) - 1)[0])
a_coeffs = np.array(A_COEFFS)
# initialize first k time points to noise
ts = np.zeros((TS_LEN,))
for i in range(k):
g_max = load_station_data('../data/TX_STAID000027.txt', date(1775, 1, 1),
date(2014, 1, 1), False)
g_min = load_station_data('../data/TN_STAID000027.txt', date(1775, 1, 1),
date(2014, 1, 1), False)
elif DATA == 1:
g_max = load_bin_data('../data/ERA_time_series_50.0N_15.0E.bin',
date(1940, 1, 1), date(2014, 1, 1), False)
g_min = load_bin_data('../data/ERA_time_series_50.0N_15.0E.bin',
date(1940, 1, 1), date(2014, 1, 1), False)
elif DATA == 2:
g_max = load_bin_data('../data/ECA&D_time_series_50.1N_14.4E.bin',
date(1940, 1, 1), date(2014, 1, 1), False)
g_min = load_bin_data('../data/ECA&D_time_series_50.1N_14.4E.bin',
date(1940, 1, 1), date(2014, 1, 1), False)
g_temp = DataField()
# starting month and day
sm = 7
sd = 28
# starting year of final window
y = 365.25
sy = int(MIDDLE_YEAR - (WINDOW_LENGTH / y) / 2)
# get wvlt window
start = g.find_date_ndx(date(sy - 4, sm, sd))
end = start + 16384 if WINDOW_LENGTH < 16000 else start + 32768
g.data = g.data[start:end]
g.time = g.time[start:end]
g_max.data = g_max.data[start:end]
g_max.time = g_max.time[start:end]
@author: Nikola Jajcay
"""
from src.oscillatory_time_series import OscillatoryTimeSeries
from src.data_class import DataField
from datetime import date, timedelta
import matplotlib.pyplot as plt
import numpy as np
from surrogates.surrogates import SurrogateField
import calendar
ts = OscillatoryTimeSeries('TG_STAID000027.txt', date(1834,7,28), date(2014,1,1), False)
sg = SurrogateField()
g = DataField()
daily_var = np.zeros((365,3))
mean, var_data, trend = ts.g.get_seasonality(True)
sg.copy_field(ts.g)
#MF
sg.construct_multifractal_surrogates()
sg.add_seasonality(mean, var_data, trend)
g.data = sg.surr_data.copy()
g.time = sg.time.copy()
_, var_surr_MF, _ = g.get_seasonality(True)
ev_start_year = 1861 if USE_SURR else 1802
for MIDDLE_YEAR in range(ev_start_year, 1988):
if USE_SURR:
result_temp_surr = np.zeros((NUM_SURR, 8,2))
for surr in range(NUM_SURR):
sg.construct_surrogates_with_residuals()
sg.add_seasonality(mean[:-1], var[:-1], trend[:-1]) # so SAT data
g.data = sg.surr_data.copy()
tg_sat = g.copy_data()
g.time = g.time[:-1]
g.anomalise()
g_temp = DataField()
tg_temp = tg_sat.copy()
sy = int(MIDDLE_YEAR - (WINDOW_LENGTH/year)/2)
g_temp.data = g.data.copy()
g_temp.time = g.time.copy()
start = g_temp.find_date_ndx(date(sy - 4, sm, sd))
end = start + 16384 if WINDOW_LENGTH < 16000 else start + 32768
g_temp.data = g_temp.data[start : end]
g_temp.time = g_temp.time[start : end]
tg_temp = tg_temp[start : end]
k0 = 6. # wavenumber of Morlet wavelet used in analysis
fourier_factor = (4 * np.pi) / (k0 + np.sqrt(2 + np.power(k0,2)))
period = PERIOD * year # frequency of interest
s0 = period / fourier_factor # get scale
# save SAT data
tg_sat = g.copy_data()
# anomalise to obtain SATA data
g.anomalise()
if DATA == 0:
g_max = load_station_data("../data/TX_STAID000027.txt", date(1775, 1, 1), date(2014, 1, 1), False)
g_min = load_station_data("../data/TN_STAID000027.txt", date(1775, 1, 1), date(2014, 1, 1), False)
elif DATA == 1:
g_max = load_bin_data("../data/ERA_time_series_50.0N_15.0E.bin", date(1940, 1, 1), date(2014, 1, 1), False)
g_min = load_bin_data("../data/ERA_time_series_50.0N_15.0E.bin", date(1940, 1, 1), date(2014, 1, 1), False)
elif DATA == 2:
g_max = load_bin_data("../data/ECA&D_time_series_50.1N_14.4E.bin", date(1940, 1, 1), date(2014, 1, 1), False)
g_min = load_bin_data("../data/ECA&D_time_series_50.1N_14.4E.bin", date(1940, 1, 1), date(2014, 1, 1), False)
g_temp = DataField()
# starting month and day
sm = 7
sd = 28
# starting year of final window
y = 365.25
sy = int(MIDDLE_YEAR - (WINDOW_LENGTH / y) / 2)
# get wvlt window
start = g.find_date_ndx(date(sy - 4, sm, sd))
end = start + 16384 if WINDOW_LENGTH < 16000 else start + 32768
g.data = g.data[start:end]
g.time = g.time[start:end]
g_max.data = g_max.data[start:end]
# "grid" : "2.5/2.5",
# "time" : "00/06/12/18", ## daily
# "date" : "20010101/to/20131231",
# "area" : "50/-15/30/5", ## north/west/south/east
# "type" : "an",
# "class" : "e4",
# "format" : "netcdf",
# "padding" : "0",
# "target" : "test.nc"
# })
#==============================================================================
# load ERA-40 as g1 and ERA-Interim as g2
print("[%s] Loading data..." % (str(datetime.now())))
g1 = DataField()
g2 = DataField()
g1.load('Spain.ERA.58-01.nc', 't2m', 'ERA-40')
g2.load('Spain.ERA.01-13.nc', 't2m', 'ERA-40')
# concatenate
last = g1.time[-1]
idx = np.where(g2.time == last)[0] + 1
ndays = g1.time.shape[0] + g2.time[idx:].shape[0]
data = np.zeros((ndays, g1.lats.shape[0], g1.lons.shape[0]))
time = np.zeros((ndays, ))
data[:g1.time.shape[0], ...] = g1.data
data[g1.time.shape[0]:, ...] = g2.data[idx:]
AMP_PERIOD = 8
BINS = 8
ANOMALISE = True # amplitude from SAT / SATA
SURR = True
NUM_SURR = 1000
WORKERS = 3
SURR_TYPE = 'FT'
# 65k
# g = load_station_data('TG_STAID000027.txt', date(1834,4,27), date(2013,10,1), True)
# g_amp = load_station_data('TG_STAID000027.txt', date(1834,4,27), date(2013,10,1), ANOMALISE)
# 32k
g = load_station_data('TG_STAID000027.txt', date(1958, 1, 1), date(2013, 11, 10), True) # date(1924,1,14), date(2013,10,1)
g_amp = load_station_data('TG_STAID000027.txt', date(1958, 1, 1), date(2013, 11, 10), ANOMALISE)
g_data = DataField()
k0 = 6. # wavenumber of Morlet wavelet used in analysis
y = 365.25 # year in days
fourier_factor = (4 * np.pi) / (k0 + np.sqrt(2 + np.power(k0,2)))
period = PERIOD * y # frequency of interest
s0 = period / fourier_factor # get scale
# wavelet - data
wave, _, _, _ = wavelet_analysis.continous_wavelet(g.data, 1, False, wavelet_analysis.morlet, dj = 0, s0 = s0, j1 = 0, k0 = k0) # perform wavelet
phase = np.arctan2(np.imag(wave), np.real(wave)) # get phases from oscillatory modes
period = AMP_PERIOD * 365.25 # frequency of interest
s0_amp = period / fourier_factor # get scale
wave, _, _, _ = wavelet_analysis.continous_wavelet(g_amp.data, 1, False, wavelet_analysis.morlet, dj = 0, s0 = s0_amp, j1 = 0, k0 = k0) # perform wavelet
g = load_station_data(
"../data/TG_STAID000027.txt", date(1775, 1, 1), date(2015, 1, 1), ANOMALISE
) # 15-01-1924 if 32k, 28-04-1834 if 64k
if AMPLITUDE:
g_amp = load_station_data("../data/TG_STAID000027.txt", date(1775, 1, 1), date(2015, 1, 1), False)
## HAMBURG -- TG_STAID000047, POTSDAM -- TG_STAID000054
# g = load_station_data('../data/TG_STAID000054.txt', date(1893,1,1), date(2014,1,1), ANOMALISE) # 15-01-1924 if 32k, 28-04-1834 if 64k
# if AMPLITUDE:
# g_amp = load_station_data('../data/TG_STAID000054.txt', date(1893,1,1), date(2014, 1, 1), False)
# ERA
# g = load_bin_data('../data/ERA_time_series_50.0N_15.0E.bin', date(1958,4,28), date(2013,10,1), ANOMALISE)
# ECA
# g = load_bin_data('../data/ECA&D_time_series_50.1N_14.4E.bin', date(1950,4,28), date(2013,10,1), ANOMALISE)
g_working = DataField()
g_surrs = DataField()
if AMPLITUDE:
g_working_amp = DataField()
g_surrs_amp = DataField()
if MOMENT == "mean":
func = np.mean
if AMPLITUDE:
diff_ax = (0, 2) # means -> 0, 2, var -> 1, 8
mean_ax = (18, 22) # means -> -1, 1.5, var -> 9, 18
else:
diff_ax = (0, 5)
mean_ax = (-1, 1.5)
elif MOMENT == "std":
func = np.var
diff_ax = (1, 15)
# [LON - 5, LON + 5], False, parts = 3)
GRID_POINTS = [[50, 15], [50, 12.5], [52.5, 12.5], [52.5, 15]]
for lat, lon in GRID_POINTS:
g = load_NCEP_data_monthly('../data/air.mon.mean.levels.nc',
'air',
date(1948, 1, 1),
date(2014, 1, 1), [lat - 1, lat + 1],
[lon - 1, lon + 1],
level=LEVEL,
anom=False)
print g.data.shape
lat_arg = np.argmin(np.abs(LAT - g.lats))
lon_arg = np.argmin(np.abs(LON - g.lons))
ts = g.data[:, lat_arg, lon_arg].copy()
time = g.time.copy()
loc = ("GRID | lat: %.1f, lon: %.1f" % (g.lats[lat_arg], g.lons[lon_arg]))
g_grid = DataField(data=ts, time=time)
g_grid.location = loc
with open("%s_time_series_%.1fN_%.1fE.bin" % ('NCEP30hPa', lat, lon),
'wb') as f:
cPickle.dump({'g': g_grid}, f, protocol=cPickle.HIGHEST_PROTOCOL)
print("[%s] Dumped time-series from %.1f N and %.1f E." %
(str(datetime.now()), g.lats[lat_arg], g.lons[lon_arg]))
plt.title(title)
cbar = plt.colorbar(format=r"%2.2f",
shrink=0.75,
ticks=np.arange(mi, ma + step, (ma - mi) / 8),
aspect=25,
drawedges=False)
cbar.set_label(cbar_label)
cbar_obj = plt.getp(cbar.ax.axes, 'yticklabels')
plt.setp(cbar_obj, fontsize=10, color=(.1, .1, .1))
if filename != None:
plt.savefig(filename)
else:
plt.show()
g = DataField()
g.load('tg_0.25deg_reg_v9.0.nc', 'tg')
means = True
daily = False
if means:
idx = 0
y = 1950
while idx < g.data.shape[0]:
idx2 = g.find_date_ndx(date(y, 1, 1))
render_geo_field(g.data[idx:idx2, ...], g.lats, g.lons, None, None,
False, 'Yearly mean temperature %s' % str(y),
'temperature [$^{\circ}C$]',
'imgs/temp_mean%s.png' % str(y))
y += 1
idx = idx2
# amp_to_plot_surr = amp_to_plot_surr[idx[0] : idx[1]]
phase_amp = phase_amp[idx[0] : idx[1]]
sg.surr_data = sg.surr_data[idx[0] : idx[1]]
cond_temp = np.zeros((BINS,2))
for i in range(cond_means.shape[0]):
ndx = ((phase_amp >= phase_bins[i]) & (phase_amp <= phase_bins[i+1]))
cond_temp[i,1] = np.mean(sg.surr_data[ndx])
data_diff = cond_temp[:, 1].max() - cond_temp[:, 1].min()
resq.put([data_diff, np.mean(amplitude2)])
g = load_station_data('../data/TG_STAID000027.txt', date(1958, 1, 1), date(2013, 11, 10), True)
g_amp = load_station_data('../data/TG_STAID000027.txt', date(1958, 1, 1), date(2013, 11, 10), True)
g_data = DataField()
k0 = 6. # wavenumber of Morlet wavelet used in analysis
y = 365.25 # year in days
fourier_factor = (4 * np.pi) / (k0 + np.sqrt(2 + np.power(k0,2)))
period = PERIOD * y # frequency of interest
s0 = period / fourier_factor # get scale
# wavelet - data
wave, _, _, _ = wavelet_analysis.continous_wavelet(g.data, 1, False, wavelet_analysis.morlet, dj = 0, s0 = s0, j1 = 0, k0 = k0) # perform wavelet
phase = np.arctan2(np.imag(wave), np.real(wave)) # get phases from oscillatory modes
if AMPLITUDE:
period = 8 * 365.25 # frequency of interest
def _corrs_surrs_ind(args):
index_surr = DataField()
index_surr.data = get_single_FT_surrogate(index_data.data)
index_correlations_surrs = get_corrs(net, index_surr)
return index_correlations_surrs
date(2015, 1, 1),
None,
None,
0,
dataset="NCEP",
sampling='monthly',
anom=False)
pool = Pool(NUM_WORKERS)
net.wavelet(1, 'y', pool=pool, cut=1)
net.get_continuous_phase(pool=pool)
net.get_phase_fluctuations(rewrite=True, pool=pool)
pool.close()
pool.join()
nao = DataField()
raw = np.loadtxt("%sWeMO.monthly.1821-2013.txt" % (path_to_data))
raw = raw[:, 1:]
nao.data = raw.reshape(-1)
nao.create_time_array(date_from=date(1821, 1, 1), sampling='m')
nao.select_date(date(1949, 1, 1), date(2014, 1, 1))
nao.anomalise()
jfm_index = nao.select_months([1, 2, 3], apply_to_data=False)
jfm_nao = nao.data[jfm_index]
_, _, y = nao.extract_day_month_year()
y = y[jfm_index]
ann_nao = []
for year in np.unique(y):
ann_nao.append(np.mean(jfm_nao[np.where(year == y)[0]]))
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
AMPLITUDE = True
PERIOD = 8
BINS = 8
SEASON = None#[[12, 1, 2], [6, 7, 8]]
STATIONS = None # ['TG_STAID000047.txt', 'TG_STAID000054.txt']
if STATIONS == None:
g = load_station_data('../data/TG_STAID000027.txt', date(1958, 1, 1), date(2013, 11, 10), True)
if AMPLITUDE:
g_amp = load_station_data('../data/TG_STAID000027.txt', date(1958, 1, 1), date(2013, 11, 10), False)
g_data = DataField()
else:
for i in range(len(STATIONS)):
locals()['g' + str(i)] = load_station_data(STATIONS[i], date(1924,1,15), date(2013,10,1), True)
if AMPLITUDE:
locals()['g_amp' + str(i)] = load_station_data(STATIONS[i], date(1924,1,15), date(2013, 10, 1), False)
locals()['g_data' + str(i)] = DataField()
k0 = 6. # wavenumber of Morlet wavelet used in analysis
y = 365.25 # year in days
fourier_factor = (4 * np.pi) / (k0 + np.sqrt(2 + np.power(k0,2)))
period = PERIOD * y # frequency of interest
s0 = period / fourier_factor # get scale
# wavelet - data
if STATIONS == None:
wave, _, _, _ = wavelet_analysis.continous_wavelet(g.data, 1, False, wavelet_analysis.morlet, dj = 0, s0 = s0, j1 = 0, k0 = k0) # perform wavelet
@author: Nikola Jajcay
"""
from src.oscillatory_time_series import OscillatoryTimeSeries
from src.data_class import DataField
from datetime import date, timedelta
import matplotlib.pyplot as plt
import numpy as np
from surrogates.surrogates import SurrogateField
import calendar
ts = OscillatoryTimeSeries('TG_STAID000027.txt', date(1834, 7, 28),
date(2014, 1, 1), False)
sg = SurrogateField()
g = DataField()
daily_var = np.zeros((365, 3))
mean, var_data, trend = ts.g.get_seasonality(True)
sg.copy_field(ts.g)
#MF
sg.construct_multifractal_surrogates()
sg.add_seasonality(mean, var_data, trend)
g.data = sg.surr_data.copy()
g.time = sg.time.copy()
_, var_surr_MF, _ = g.get_seasonality(True)
#FT
axplot = [2.5, 5.5]
elif MOMENT == 'skewness':
func = sts.skew
axplot = [-0.5, 1]
elif MOMENT == 'kurtosis':
func = sts.kurtosis
axplot = [0, 5]
# load data - at least 32k of data because of surrogates
# 00047 - Hamburg, 00054 - Potsdam
g = load_station_data('TG_STAID000027.txt', date(1834, 4, 28),
date(2013, 10, 1), ANOMALISE)
if AMPLITUDE:
g_amp = load_station_data('TG_STAID000027.txt', date(1834, 4, 28),
date(2013, 10, 1), False)
g_data = DataField()
print(
"[%s] Wavelet analysis in progress with %d year window shifted by %d year(s)..."
% (str(datetime.now()), WINDOW_LENGTH, WINDOW_SHIFT))
k0 = 6. # wavenumber of Morlet wavelet used in analysis
y = 365.25 # year in days
fourier_factor = (4 * np.pi) / (k0 + np.sqrt(2 + np.power(k0, 2)))
period = PERIOD * y # frequency of interest
s0 = period / fourier_factor # get scale
# wavelet - data
wave, _, _, _ = wavelet_analysis.continous_wavelet(g.data,
1,
False,
wavelet_analysis.morlet,
dj=0,
ev_start_year = 1861 if USE_SURR else 1802
for MIDDLE_YEAR in range(ev_start_year, 1988):
if USE_SURR:
result_temp_surr = np.zeros((NUM_SURR, 8, 2))
for surr in range(NUM_SURR):
sg.construct_surrogates_with_residuals()
sg.add_seasonality(mean[:-1], var[:-1], trend[:-1]) # so SAT data
g.data = sg.surr_data.copy()
tg_sat = g.copy_data()
g.time = g.time[:-1]
g.anomalise()
g_temp = DataField()
tg_temp = tg_sat.copy()
sy = int(MIDDLE_YEAR - (WINDOW_LENGTH / year) / 2)
g_temp.data = g.data.copy()
g_temp.time = g.time.copy()
start = g_temp.find_date_ndx(date(sy - 4, sm, sd))
end = start + 16384 if WINDOW_LENGTH < 16000 else start + 32768
g_temp.data = g_temp.data[start:end]
g_temp.time = g_temp.time[start:end]
tg_temp = tg_temp[start:end]
k0 = 6. # wavenumber of Morlet wavelet used in analysis
fourier_factor = (4 * np.pi) / (k0 + np.sqrt(2 + np.power(k0, 2)))
period = PERIOD * year # frequency of interest
s0 = period / fourier_factor # get scale
GRID_POINTS = [[50, 15], [50, 12.5], [52.5, 12.5], [52.5, 15]]
for lat, lon in GRID_POINTS:
g = load_NCEP_data_monthly(
"../data/air.mon.mean.levels.nc",
"air",
date(1948, 1, 1),
date(2014, 1, 1),
[lat - 1, lat + 1],
[lon - 1, lon + 1],
level=LEVEL,
anom=False,
)
print g.data.shape
lat_arg = np.argmin(np.abs(LAT - g.lats))
lon_arg = np.argmin(np.abs(LON - g.lons))
ts = g.data[:, lat_arg, lon_arg].copy()
time = g.time.copy()
loc = "GRID | lat: %.1f, lon: %.1f" % (g.lats[lat_arg], g.lons[lon_arg])
g_grid = DataField(data=ts, time=time)
g_grid.location = loc
with open("%s_time_series_%.1fN_%.1fE.bin" % ("NCEP30hPa", lat, lon), "wb") as f:
cPickle.dump({"g": g_grid}, f, protocol=cPickle.HIGHEST_PROTOCOL)
print ("[%s] Dumped time-series from %.1f N and %.1f E." % (str(datetime.now()), g.lats[lat_arg], g.lons[lon_arg]))
NUM_WORKERS = 20
net = ScaleSpecificNetwork('%sair.mon.mean.levels.nc' % path_to_data, 'air',
date(1948,1,1), date(2016,1,1), None, None, 0, dataset = "NCEP", sampling = 'monthly', anom = False)
pool = Pool(NUM_WORKERS)
net.wavelet(1, 'y', pool = pool, cut = 1)
net.get_continuous_phase(pool = pool)
net.get_phase_fluctuations(rewrite = True, pool = pool)
pool.close()
pool.join()
nao = DataField()
raw = np.loadtxt("%sNAO.station.monthly.1865-2016.txt" % (path_to_data))
raw = raw[:, 1:]
nao.data = raw.reshape(-1)
nao.create_time_array(date_from = date(1865, 1, 1), sampling = 'm')
nao.select_date(date(1949, 1, 1), date(2015, 1, 1))
nao.anomalise()
jfm_index = nao.select_months([1,2,3], apply_to_data = False)
jfm_nao = nao.data[jfm_index]
_, _, y = nao.extract_day_month_year()
y = y[jfm_index]
ann_nao = []
for year in np.unique(y):
ann_nao.append(np.mean(jfm_nao[np.where(year == y)[0]]))
def _corrs_surrs_ind(args):
index_surr = DataField()
index_surr.data = get_single_FT_surrogate(index_data.data)
index_correlations_surrs = get_corrs(net, index_surr)
return index_correlations_surrs
elif MOMENT == 'std':
func = np.std
axplot = [2.5, 5.5]
elif MOMENT == 'skewness':
func = sts.skew
axplot = [-0.5, 1]
elif MOMENT == 'kurtosis':
func = sts.kurtosis
axplot = [0, 5]
# load data - at least 32k of data because of surrogates
# 00047 - Hamburg, 00054 - Potsdam
g = load_station_data('TG_STAID000027.txt', date(1834,4,28), date(2013,10,1), ANOMALISE)
if AMPLITUDE:
g_amp = load_station_data('TG_STAID000027.txt', date(1834,4,28), date(2013, 10, 1), False)
g_data = DataField()
print("[%s] Wavelet analysis in progress with %d year window shifted by %d year(s)..." % (str(datetime.now()), WINDOW_LENGTH, WINDOW_SHIFT))
k0 = 6. # wavenumber of Morlet wavelet used in analysis
y = 365.25 # year in days
fourier_factor = (4 * np.pi) / (k0 + np.sqrt(2 + np.power(k0,2)))
period = PERIOD * y # frequency of interest
s0 = period / fourier_factor # get scale
# wavelet - data
wave, _, _, _ = wavelet_analysis.continous_wavelet(g.data, 1, False, wavelet_analysis.morlet, dj = 0, s0 = s0, j1 = 0, k0 = k0) # perform wavelet
phase = np.arctan2(np.imag(wave), np.real(wave)) # get phases from oscillatory modes
if AMPLITUDE:
period = 1 * 365.25 # frequency of interest
s0_amp = period / fourier_factor # get scale
