def load_neutron_NESDIS_data(fname, start_date, end_date, anom=True):
raw = np.loadtxt(fname, skiprows=2)
data = []
time = []
for year in range(raw.shape[0]):
for month in range(1, 13):
dat = float(raw[year, month])
if dat == 9999.:
dat = (float(raw[year, month - 2]) + float(
raw[year, month - 1]) + float(raw[year, month + 1]) +
float(raw[year, month + 2])) / 4.
data.append(dat)
time.append(date(int(raw[year, 0]), month, 1).toordinal())
g = DataField(data=np.array(data), time=np.array(time))
g.location = ('%s cosmic data' % (fname[32].upper() + fname[33:-4]))
g.select_date(start_date, end_date)
if anom:
g.anomalise()
if NUM_SURR != 0:
g_surr = SurrogateField()
seasonality = g.get_seasonality()
g_surr.copy_field(g)
g.return_seasonality(seasonality[0], seasonality[1], None)
else:
g_surr, seasonality = None, None
return g, g_surr, seasonality
def load_CR_climax_daily_data(fname, start_date, end_date, anom = False):
from dateutil.relativedelta import relativedelta
raw = np.loadtxt(fname)
time = []
datenow = date(1994, 1, 1)
delta = timedelta(days = 1)
for t in range(raw.shape[0]):
time.append(datenow.toordinal())
datenow += delta
print raw.shape
print len(time)
g = DataField(data = np.array(raw), time = np.array(time))
g.location = 'Climax, CO cosmic data'
g.select_date(start_date, end_date)
if anom:
g.anomalise()
if NUM_SURR != 0:
g_surr = SurrogateField()
seasonality = g.get_seasonality(True)
g_surr.copy_field(g)
g.return_seasonality(seasonality[0], seasonality[1], seasonality[2])
else:
g_surr, seasonality = None, None
return g, g_surr, seasonality
def load_CR_climax_daily_data(fname, start_date, end_date, anom=False):
from dateutil.relativedelta import relativedelta
raw = np.loadtxt(fname)
time = []
datenow = date(1994, 1, 1)
delta = timedelta(days=1)
for t in range(raw.shape[0]):
time.append(datenow.toordinal())
datenow += delta
print raw.shape
print len(time)
g = DataField(data=np.array(raw), time=np.array(time))
g.location = 'Climax, CO cosmic data'
g.select_date(start_date, end_date)
if anom:
g.anomalise()
if NUM_SURR != 0:
g_surr = SurrogateField()
seasonality = g.get_seasonality(True)
g_surr.copy_field(g)
g.return_seasonality(seasonality[0], seasonality[1], seasonality[2])
else:
g_surr, seasonality = None, None
return g, g_surr, seasonality
def load_neutron_NESDIS_data(fname, start_date, end_date, anom = True):
raw = np.loadtxt(fname, skiprows = 2)
data = []
time = []
for year in range(raw.shape[0]):
for month in range(1,13):
dat = float(raw[year, month])
if dat == 9999.:
dat = (float(raw[year, month-2]) + float(raw[year, month-1]) + float(raw[year, month+1]) + float(raw[year, month+2])) / 4.
data.append(dat)
time.append(date(int(raw[year,0]), month, 1).toordinal())
g = DataField(data = np.array(data), time = np.array(time))
g.location = ('%s cosmic data' % (fname[32].upper() + fname[33:-4]))
g.select_date(start_date, end_date)
if anom:
g.anomalise()
if NUM_SURR != 0:
g_surr = SurrogateField()
seasonality = g.get_seasonality()
g_surr.copy_field(g)
g.return_seasonality(seasonality[0], seasonality[1], None)
else:
g_surr, seasonality = None, None
return g, g_surr, seasonality
def load_cosmic_data(fname,
start_date,
end_date,
anom=True,
daily=False,
corrected=True):
# corrected stands for if use corrected data or not
from dateutil.relativedelta import relativedelta
raw = open(fname).read()
lines = raw.split('\n')
data = []
time = []
d = date(int(lines[0][:4]), int(lines[0][5:7]), 1)
if not daily:
delta = relativedelta(months=+1)
elif daily:
delta = timedelta(days=1)
for line in lines:
row = line.split(' ')
if len(row) < 6:
continue
time.append(d.toordinal())
if corrected:
data.append(float(row[4]))
else:
data.append(float(row[5]))
d += delta
g = DataField(data=np.array(data), time=np.array(time))
g.location = 'Oulu cosmic data'
g.select_date(start_date, end_date)
if anom:
g.anomalise()
g.data = X[:, 0].copy()
if NUM_SURR != 0:
g_surr = SurrogateField()
seasonality = g.get_seasonality(True)
g_surr.copy_field(g)
g.return_seasonality(seasonality[0], seasonality[1], seasonality[2])
else:
g_surr, seasonality = None, None
return g, g_surr, seasonality
def load_cosmic_data(fname, start_date, end_date, anom = True, daily = False, corrected = True):
# corrected stands for if use corrected data or not
from dateutil.relativedelta import relativedelta
raw = open(fname).read()
lines = raw.split('\n')
data = []
time = []
d = date(int(lines[0][:4]), int(lines[0][5:7]), 1)
if not daily:
delta = relativedelta(months = +1)
elif daily:
delta = timedelta(days = 1)
for line in lines:
row = line.split(' ')
if len(row) < 6:
continue
time.append(d.toordinal())
if corrected:
data.append(float(row[4]))
else:
data.append(float(row[5]))
d += delta
g = DataField(data = np.array(data), time = np.array(time))
g.location = 'Oulu cosmic data'
g.select_date(start_date, end_date)
if anom:
g.anomalise()
g.data = X[:, 0].copy()
if NUM_SURR != 0:
g_surr = SurrogateField()
seasonality = g.get_seasonality(True)
g_surr.copy_field(g)
g.return_seasonality(seasonality[0], seasonality[1], seasonality[2])
else:
g_surr, seasonality = None, None
return g, g_surr, seasonality
date(2009, 2, 8),
anom=False,
daily=DAILY)
# temp, temp_surr, temp_seas = load_CR_climax_daily_data('../data/CR-climax-daily-1-1-94--30-11-06.txt', date(1994,1,1), date(2006,11,30), False)
# temp.get_data_of_precise_length('16k', start_date = date(1964, 1, 1), COPY = True)
elif idx1 == 'sunspot':
temp = load_sunspot_data("../data/sunspot_daily.txt",
date(1964, 4, 1),
date(2009, 2, 8),
anom=False,
daily=DAILY)
# temp.get_data_of_precise_length(length = 1024, end_date = date(2007, 1, 1), COPY = True)
# temp.get_data_of_precise_length('16k', start_date = date(1964, 1, 1), COPY = True)
temp_surr = SurrogateField()
temp_seas = temp.get_seasonality(True)
temp_surr.copy_field(temp)
temp.return_seasonality(temp_seas[0], temp_seas[1], temp_seas[2])
elif idx1 == 'AAindex':
temp = load_AAgeomag_data("../data/aa_day.raw",
date(1964, 4, 1),
date(2009, 2, 8),
anom=False,
daily=DAILY)
# temp.get_data_of_precise_length('16k', start_date = date(1964, 1, 1), COPY = True)
temp_surr = SurrogateField()
temp_seas = temp.get_seasonality(True)
temp_surr.copy_field(temp)
temp.return_seasonality(temp_seas[0], temp_seas[1], temp_seas[2])
if idx2 == 'OuluCR':
aa, aa_surr, aa_seas = load_cosmic_data(
total_surrogates_condition = []
if np.all(np.isnan(g_surrs.data) == False):
# construct the job queue
jobQ = Queue()
resQ = Queue()
if CONDITION:
for i in range(3 * NUM_SURR):
jobQ.put(1)
else:
for i in range(NUM_SURR):
jobQ.put(1)
for i in range(WORKERS):
jobQ.put(None)
a = g_surrs.get_seasonality(DETREND=True)
sg = SurrogateField()
sg.copy_field(g_surrs)
if AMPLITUDE:
a_amp = g_surrs_amp.get_seasonality(True)
sg_amp = SurrogateField()
sg_amp.copy_field(g_surrs_amp)
else:
sg_amp = None
a_amp = None
if SURR_TYPE == "AR":
sg.prepare_AR_surrogates()
if AMPLITUDE:
sg_amp.prepare_AR_surrogates()
workers = [
Process(
target=_cond_difference_surrogates,
args=(sg, sg_amp, g_surrs, a, a_amp, start_cut, jobQ, resQ, ndx_season),
# positive extremes
g_e = np.greater_equal(tg_sat_temp,
np.mean(tg_avg_sat, axis=0) + 2 * sigma)
avg_bins[i, 0] = np.sum(g_e)
# negative extremes
l_e = np.less_equal(tg_sat_temp,
np.mean(tg_avg_sat, axis=0) - 2 * sigma)
avg_bins[i, 1] = np.sum(l_e)
else:
sg = SurrogateField()
# g_for_avg are SAT data
mean, var, trend = g_for_avg.get_seasonality(detrend=True)
sg.copy_field(g_for_avg)
sg.prepare_AR_surrogates()
year = 365.25
k0 = 6. # wavenumber of Morlet wavelet used in analysis
fourier_factor = (4 * np.pi) / (k0 + np.sqrt(2 + np.power(k0, 2)))
period = PERIOD * year # frequency of interest
s0 = period / fourier_factor # get scale
avg_bins_surr = np.zeros(
(NUM_SURR, 8,
2)) # num surr x bin no. x result no. (hot / cold extremes)
phase_bins = get_equidistant_bins()
for surr in range(NUM_SURR):
sg.construct_surrogates_with_residuals()
net = ScaleSpecificNetwork(
fname,
"air",
date(1948, 1, 1),
date(2016, 1, 1),
[-60, 0],
[40, 100],
level=0,
dataset="NCEP",
sampling="monthly",
anom=False,
)
surrs = SurrogateField()
a = net.get_seasonality(detrend=True)
surrs.copy_field(net)
net.return_seasonality(a[0], a[1], a[2])
pool = Pool(20)
net.wavelet(8, "y", cut=1, pool=pool)
net.get_adjacency_matrix(net.phase, method="MIEQQ", num_workers=0, pool=pool, use_queue=False)
pool.close()
pool.join()
data_adj_matrix = net.adjacency_matrix.copy()
surrs_adj_matrices = []
for i in range(NUM_SURR):
print("surr %d/%d computing..." % (i + 1, NUM_SURR))
pool = Pool(20)
effect_windows = np.zeros((n_windows))
mean_amp_windows = np.zeros((n_windows))
mean_ampAAC_windows = np.zeros((n_windows))
amp_windows_surrs = np.zeros((NUM_SURRS, n_windows))
effect_windows_surrs = np.zeros((NUM_SURRS, n_windows))
mean_amp_windows_surrs = np.zeros((NUM_SURRS, n_windows))
mean_ampAAC_windows_surrs = np.zeros((NUM_SURRS, n_windows))
for i, ndx in zip(range(len(ndxs)), ndxs):
# copy part of data
prg_temp = prg.copy(temporal_ndx = ndx)
# get ready for surrs
mean, var, trend = prg_temp.get_seasonality(detrend = True)
prg_surr = SurrogateField()
prg_surr.copy_field(prg_temp)
prg_temp.return_seasonality(mean, var, trend)
## COMPUTE FOR DATA
prg_temp.wavelet(1, 'y', cut = 1, cut_time = False, cut_data = False, regress_amp_to_data = True)
annual_amp = prg_temp.amplitude.copy()
annual_phase = prg_temp.phase.copy()
prg_temp.anomalise()
prg_temp.wavelet(8, 'y', cut = 1, cut_time = False, cut_data = False, regress_amp_to_data = True, continuous_phase = False)
amplitude = prg_temp.amplitude.copy()
prg_temp.wavelet(8, 'y', cut = 1, cut_time = True, cut_data = True, regress_amp_to_data = False, continuous_phase = False)
amplitudeAACreg = prg_temp.amplitude.copy()
m, c, r, p, std_err = sts.linregress(amplitudeAACreg*np.cos(prg_temp.phase), annual_amp*np.cos(annual_phase))
amplitudeAACreg = m*amplitudeAACreg + c
tg_sat_temp = tg_avg_sat[ndx].copy()
# positive extremes
g_e = np.greater_equal(tg_sat_temp, np.mean(tg_avg_sat, axis = 0) + 2 * sigma)
avg_bins[i, 0] = np.sum(g_e)
# negative extremes
l_e = np.less_equal(tg_sat_temp, np.mean(tg_avg_sat, axis = 0) - 2 * sigma)
avg_bins[i, 1] = np.sum(l_e)
else:
sg = SurrogateField()
# g_for_avg are SAT data
mean, var, trend = g_for_avg.get_seasonality(detrend = True)
sg.copy_field(g_for_avg)
sg.prepare_AR_surrogates()
year = 365.25
k0 = 6. # wavenumber of Morlet wavelet used in analysis
fourier_factor = (4 * np.pi) / (k0 + np.sqrt(2 + np.power(k0,2)))
period = PERIOD * year # frequency of interest
s0 = period / fourier_factor # get scale
avg_bins_surr = np.zeros((NUM_SURR, 8, 2)) # num surr x bin no. x result no. (hot / cold extremes)
phase_bins = get_equidistant_bins()
for surr in range(NUM_SURR):
sg.construct_surrogates_with_residuals()
sg.add_seasonality(mean[:-1], var[:-1], trend[:-1]) # so SAT data
g_for_avg.data = sg.surr_data.copy()
# fname = '/home/nikola/Work/phd/data/air.mon.mean.sig995.nc'
fname = "/Users/nikola/work-ui/data/air.mon.mean.sig995.nc"
surrs = SurrogateField()
net = ScaleSpecificNetwork(fname,
'air',
date(1950, 1, 1),
date(2016, 1, 1),
None,
None,
None,
'monthly',
anom=False)
a = net.get_seasonality(detrend=True)
surrs.copy_field(net)
# surrs.construct_fourier_surrogates()
# surrs.add_seasonality(a[0], a[1], a[2])
for num in range(NUM_SURRS):
pool = Pool(20)
surrs.construct_fourier_surrogates(pool=pool)
surrs.add_seasonality(a[0], a[1], a[2])
net.data = surrs.get_surr()
net.wavelet(1, 'y', pool=pool, cut=1)
net.get_continuous_phase(pool=pool)
print "wavelet done"
net.get_phase_fluctuations(rewrite=True, pool=pool)
print "fluctuations done"
pool.close()
parts=3,
logger_function=log)
idx = g.get_data_of_precise_length(
'16k', START_DATE, None, True) # get 2^n data because of MF surrogates
if AMPLITUDE:
g_amp = g_amp[idx[0]:idx[1], ...]
END_DATE = g.get_date_from_ndx(-1)
if SURR_TYPE is not None:
log("Creating surrogate fields...")
sg = SurrogateField() # for MF and FT surrs
log("De-seasonalising the data and copying to surrogate field...")
mean, var, trend = g.get_seasonality(
True) # subtract mean, divide by std and subtract trend from data
sg.copy_field(g) # copy standartised data to SurrogateField
g.return_seasonality(mean, var, trend) # return seasonality to data
log("Surrogate fields created.")
if ANOMALISE:
g.anomalise()
## wavelet data
log("Running wavelet analysis on data using %d workers..." % (WORKERS))
k0 = 6. # wavenumber of Morlet wavelet used in analysis
y = 365.25 # year in days
fourier_factor = (4 * np.pi) / (k0 + np.sqrt(2 + np.power(k0, 2)))
period = PERIOD * y # frequency of interest
s0 = period / fourier_factor # get scale
phase_data = np.zeros_like(g.data)
data_diff = cond_means[:, 0, 0].max() - cond_means[:, 0, 0].min()
if SURR:
surr_diff = np.zeros((NUM_SURR,))
amp_surr = np.zeros((NUM_SURR,))
surr_completed = 0
jobQ = Queue()
resQ = Queue()
for i in range(NUM_SURR):
jobQ.put(1)
for i in range(WORKERS):
jobQ.put(None)
a = g_amp.get_seasonality(True)
sg = SurrogateField()
sg.copy_field(g_amp)
phase_bins = get_equidistant_bins(BINS)
workers = [Process(target = _reconstruction_surrs, args = (sg, a, jobQ, resQ, idx)) for iota in range(WORKERS)]
for w in workers:
w.start()
while surr_completed < NUM_SURR:
surr_means = resQ.get()
surr_diff[surr_completed] = surr_means[0]
amp_surr[surr_completed] = surr_means[1]
surr_completed += 1
if (surr_completed % 100) == 0:
print("%d. surrogate done..." % surr_completed)
names = [['sunspot', 'AAindex'], ['sunspot', 'OuluCR'], ['OuluCR', 'AAindex']]
for [idx1, idx2] in names:
if idx1 == 'OuluCR':
temp, temp_surr, temp_seas = load_cosmic_data("../data/oulu_cosmic_daily.dat", date(1964, 4, 1), date(2009, 2, 8), anom = False, daily = DAILY)
# temp, temp_surr, temp_seas = load_CR_climax_daily_data('../data/CR-climax-daily-1-1-94--30-11-06.txt', date(1994,1,1), date(2006,11,30), False)
# temp.get_data_of_precise_length('16k', start_date = date(1964, 1, 1), COPY = True)
elif idx1 == 'sunspot':
temp = load_sunspot_data("../data/sunspot_daily.txt", date(1964, 4, 1), date(2009, 2, 8), anom = False, daily = DAILY)
# temp.get_data_of_precise_length(length = 1024, end_date = date(2007, 1, 1), COPY = True)
# temp.get_data_of_precise_length('16k', start_date = date(1964, 1, 1), COPY = True)
temp_surr = SurrogateField()
temp_seas = temp.get_seasonality(True)
temp_surr.copy_field(temp)
temp.return_seasonality(temp_seas[0], temp_seas[1], temp_seas[2])
elif idx1 == 'AAindex':
temp = load_AAgeomag_data("../data/aa_day.raw", date(1964, 4, 1), date(2009, 2, 8), anom = False, daily = DAILY)
# temp.get_data_of_precise_length('16k', start_date = date(1964, 1, 1), COPY = True)
temp_surr = SurrogateField()
temp_seas = temp.get_seasonality(True)
temp_surr.copy_field(temp)
temp.return_seasonality(temp_seas[0], temp_seas[1], temp_seas[2])
if idx2 == 'OuluCR':
aa, aa_surr, aa_seas = load_cosmic_data("../data/oulu_cosmic_daily.dat", date(1964, 4, 1), date(2009, 2, 8), anom = False, daily = DAILY)
# aa, aa_surr, aa_seas = load_CR_climax_daily_data('../data/CR-climax-daily-1-1-94--30-11-06.txt', date(1994,1,1), date(2006,11,30), False)
elif idx2 == 'sunspot':
aa = load_sunspot_data("../data/sunspot_daily.txt", date(1964, 4, 1), date(2009, 2, 8), anom = False, daily = DAILY)
aa_surr = SurrogateField()
total_surrogates_condition = []
if np.all(np.isnan(g_surrs.data) == False):
# construct the job queue
jobQ = Queue()
resQ = Queue()
if CONDITION:
for i in range(3 * NUM_SURR):
jobQ.put(1)
else:
for i in range(NUM_SURR):
jobQ.put(1)
for i in range(WORKERS):
jobQ.put(None)
a = g_surrs.get_seasonality(DETREND=True)
sg = SurrogateField()
sg.copy_field(g_surrs)
if AMPLITUDE:
a_amp = g_surrs_amp.get_seasonality(True)
sg_amp = SurrogateField()
sg_amp.copy_field(g_surrs_amp)
else:
sg_amp = None
a_amp = None
if SURR_TYPE == 'AR':
sg.prepare_AR_surrogates()
if AMPLITUDE:
sg_amp.prepare_AR_surrogates()
workers = [
Process(target=_cond_difference_surrogates,
args=(sg, sg_amp, g_surrs, a, a_amp, start_cut, jobQ,
resQ, ndx_season)) for iota in range(WORKERS)
# ERA-40 + ERA-Interim
else:
g = load_ERA_data_daily('ERA40_EU', 't2m', date(1958,1,1), date(2004,1,1), LATS, LONS, False,
parts = 3, logger_function = log)
idx = g.get_data_of_precise_length('16k', START_DATE, None, True) # get 2^n data because of MF surrogates
if AMPLITUDE:
g_amp = g_amp[idx[0] : idx[1], ...]
END_DATE = g.get_date_from_ndx(-1)
if SURR_TYPE is not None:
log("Creating surrogate fields...")
sg = SurrogateField() # for MF and FT surrs
log("De-seasonalising the data and copying to surrogate field...")
mean, var, trend = g.get_seasonality(True) # subtract mean, divide by std and subtract trend from data
sg.copy_field(g) # copy standartised data to SurrogateField
g.return_seasonality(mean, var, trend) # return seasonality to data
log("Surrogate fields created.")
if ANOMALISE:
g.anomalise()
## wavelet data
log("Running wavelet analysis on data using %d workers..." % (WORKERS))
k0 = 6. # wavenumber of Morlet wavelet used in analysis
y = 365.25 # year in days
fourier_factor = (4 * np.pi) / (k0 + np.sqrt(2 + np.power(k0,2)))
period = PERIOD * y # frequency of interest
s0 = period / fourier_factor # get scale