def load_neutron_NESDIS_data(fname, start_date, end_date, anom=True): raw = np.loadtxt(fname, skiprows=2) data = [] time = [] for year in range(raw.shape[0]): for month in range(1, 13): dat = float(raw[year, month]) if dat == 9999.: dat = (float(raw[year, month - 2]) + float( raw[year, month - 1]) + float(raw[year, month + 1]) + float(raw[year, month + 2])) / 4. data.append(dat) time.append(date(int(raw[year, 0]), month, 1).toordinal()) g = DataField(data=np.array(data), time=np.array(time)) g.location = ('%s cosmic data' % (fname[32].upper() + fname[33:-4])) g.select_date(start_date, end_date) if anom: g.anomalise() if NUM_SURR != 0: g_surr = SurrogateField() seasonality = g.get_seasonality() g_surr.copy_field(g) g.return_seasonality(seasonality[0], seasonality[1], None) else: g_surr, seasonality = None, None return g, g_surr, seasonality
def load_CR_climax_daily_data(fname, start_date, end_date, anom = False): from dateutil.relativedelta import relativedelta raw = np.loadtxt(fname) time = [] datenow = date(1994, 1, 1) delta = timedelta(days = 1) for t in range(raw.shape[0]): time.append(datenow.toordinal()) datenow += delta print raw.shape print len(time) g = DataField(data = np.array(raw), time = np.array(time)) g.location = 'Climax, CO cosmic data' g.select_date(start_date, end_date) if anom: g.anomalise() if NUM_SURR != 0: g_surr = SurrogateField() seasonality = g.get_seasonality(True) g_surr.copy_field(g) g.return_seasonality(seasonality[0], seasonality[1], seasonality[2]) else: g_surr, seasonality = None, None return g, g_surr, seasonality
def load_CR_climax_daily_data(fname, start_date, end_date, anom=False): from dateutil.relativedelta import relativedelta raw = np.loadtxt(fname) time = [] datenow = date(1994, 1, 1) delta = timedelta(days=1) for t in range(raw.shape[0]): time.append(datenow.toordinal()) datenow += delta print raw.shape print len(time) g = DataField(data=np.array(raw), time=np.array(time)) g.location = 'Climax, CO cosmic data' g.select_date(start_date, end_date) if anom: g.anomalise() if NUM_SURR != 0: g_surr = SurrogateField() seasonality = g.get_seasonality(True) g_surr.copy_field(g) g.return_seasonality(seasonality[0], seasonality[1], seasonality[2]) else: g_surr, seasonality = None, None return g, g_surr, seasonality
def load_neutron_NESDIS_data(fname, start_date, end_date, anom = True): raw = np.loadtxt(fname, skiprows = 2) data = [] time = [] for year in range(raw.shape[0]): for month in range(1,13): dat = float(raw[year, month]) if dat == 9999.: dat = (float(raw[year, month-2]) + float(raw[year, month-1]) + float(raw[year, month+1]) + float(raw[year, month+2])) / 4. data.append(dat) time.append(date(int(raw[year,0]), month, 1).toordinal()) g = DataField(data = np.array(data), time = np.array(time)) g.location = ('%s cosmic data' % (fname[32].upper() + fname[33:-4])) g.select_date(start_date, end_date) if anom: g.anomalise() if NUM_SURR != 0: g_surr = SurrogateField() seasonality = g.get_seasonality() g_surr.copy_field(g) g.return_seasonality(seasonality[0], seasonality[1], None) else: g_surr, seasonality = None, None return g, g_surr, seasonality
def load_cosmic_data(fname, start_date, end_date, anom=True, daily=False, corrected=True): # corrected stands for if use corrected data or not from dateutil.relativedelta import relativedelta raw = open(fname).read() lines = raw.split('\n') data = [] time = [] d = date(int(lines[0][:4]), int(lines[0][5:7]), 1) if not daily: delta = relativedelta(months=+1) elif daily: delta = timedelta(days=1) for line in lines: row = line.split(' ') if len(row) < 6: continue time.append(d.toordinal()) if corrected: data.append(float(row[4])) else: data.append(float(row[5])) d += delta g = DataField(data=np.array(data), time=np.array(time)) g.location = 'Oulu cosmic data' g.select_date(start_date, end_date) if anom: g.anomalise() g.data = X[:, 0].copy() if NUM_SURR != 0: g_surr = SurrogateField() seasonality = g.get_seasonality(True) g_surr.copy_field(g) g.return_seasonality(seasonality[0], seasonality[1], seasonality[2]) else: g_surr, seasonality = None, None return g, g_surr, seasonality
def load_cosmic_data(fname, start_date, end_date, anom = True, daily = False, corrected = True): # corrected stands for if use corrected data or not from dateutil.relativedelta import relativedelta raw = open(fname).read() lines = raw.split('\n') data = [] time = [] d = date(int(lines[0][:4]), int(lines[0][5:7]), 1) if not daily: delta = relativedelta(months = +1) elif daily: delta = timedelta(days = 1) for line in lines: row = line.split(' ') if len(row) < 6: continue time.append(d.toordinal()) if corrected: data.append(float(row[4])) else: data.append(float(row[5])) d += delta g = DataField(data = np.array(data), time = np.array(time)) g.location = 'Oulu cosmic data' g.select_date(start_date, end_date) if anom: g.anomalise() g.data = X[:, 0].copy() if NUM_SURR != 0: g_surr = SurrogateField() seasonality = g.get_seasonality(True) g_surr.copy_field(g) g.return_seasonality(seasonality[0], seasonality[1], seasonality[2]) else: g_surr, seasonality = None, None return g, g_surr, seasonality
"../data/oulu_cosmic_daily.dat", date(1964, 4, 1), date(2009, 2, 8), anom=False, daily=DAILY) # temp, temp_surr, temp_seas = load_CR_climax_daily_data('../data/CR-climax-daily-1-1-94--30-11-06.txt', date(1994,1,1), date(2006,11,30), False) # temp.get_data_of_precise_length('16k', start_date = date(1964, 1, 1), COPY = True) elif idx1 == 'sunspot': temp = load_sunspot_data("../data/sunspot_daily.txt", date(1964, 4, 1), date(2009, 2, 8), anom=False, daily=DAILY) # temp.get_data_of_precise_length(length = 1024, end_date = date(2007, 1, 1), COPY = True) # temp.get_data_of_precise_length('16k', start_date = date(1964, 1, 1), COPY = True) temp_surr = SurrogateField() temp_seas = temp.get_seasonality(True) temp_surr.copy_field(temp) temp.return_seasonality(temp_seas[0], temp_seas[1], temp_seas[2]) elif idx1 == 'AAindex': temp = load_AAgeomag_data("../data/aa_day.raw", date(1964, 4, 1), date(2009, 2, 8), anom=False, daily=DAILY) # temp.get_data_of_precise_length('16k', start_date = date(1964, 1, 1), COPY = True) temp_surr = SurrogateField() temp_seas = temp.get_seasonality(True) temp_surr.copy_field(temp) temp.return_seasonality(temp_seas[0], temp_seas[1], temp_seas[2])
if CONDITION: total_surrogates_condition = [] if np.all(np.isnan(g_surrs.data) == False): # construct the job queue jobQ = Queue() resQ = Queue() if CONDITION: for i in range(3 * NUM_SURR): jobQ.put(1) else: for i in range(NUM_SURR): jobQ.put(1) for i in range(WORKERS): jobQ.put(None) a = g_surrs.get_seasonality(DETREND=True) sg = SurrogateField() sg.copy_field(g_surrs) if AMPLITUDE: a_amp = g_surrs_amp.get_seasonality(True) sg_amp = SurrogateField() sg_amp.copy_field(g_surrs_amp) else: sg_amp = None a_amp = None if SURR_TYPE == "AR": sg.prepare_AR_surrogates() if AMPLITUDE: sg_amp.prepare_AR_surrogates() workers = [ Process( target=_cond_difference_surrogates,
data_temp = g_for_avg.data[ndx].copy() time_temp = g_for_avg.time[ndx].copy() tg_sat_temp = tg_avg_sat[ndx].copy() # positive extremes g_e = np.greater_equal(tg_sat_temp, np.mean(tg_avg_sat, axis=0) + 2 * sigma) avg_bins[i, 0] = np.sum(g_e) # negative extremes l_e = np.less_equal(tg_sat_temp, np.mean(tg_avg_sat, axis=0) - 2 * sigma) avg_bins[i, 1] = np.sum(l_e) else: sg = SurrogateField() # g_for_avg are SAT data mean, var, trend = g_for_avg.get_seasonality(detrend=True) sg.copy_field(g_for_avg) sg.prepare_AR_surrogates() year = 365.25 k0 = 6. # wavenumber of Morlet wavelet used in analysis fourier_factor = (4 * np.pi) / (k0 + np.sqrt(2 + np.power(k0, 2))) period = PERIOD * year # frequency of interest s0 = period / fourier_factor # get scale avg_bins_surr = np.zeros( (NUM_SURR, 8, 2)) # num surr x bin no. x result no. (hot / cold extremes) phase_bins = get_equidistant_bins()
# fname = "/Users/nikola/work-ui/data/NCEP/air.mon.mean.levels.nc" fname = "/home/nikola/Work/phd/data/air.mon.mean.levels.nc" net = ScaleSpecificNetwork( fname, "air", date(1948, 1, 1), date(2016, 1, 1), [-60, 0], [40, 100], level=0, dataset="NCEP", sampling="monthly", anom=False, ) surrs = SurrogateField() a = net.get_seasonality(detrend=True) surrs.copy_field(net) net.return_seasonality(a[0], a[1], a[2]) pool = Pool(20) net.wavelet(8, "y", cut=1, pool=pool) net.get_adjacency_matrix(net.phase, method="MIEQQ", num_workers=0, pool=pool, use_queue=False) pool.close() pool.join() data_adj_matrix = net.adjacency_matrix.copy() surrs_adj_matrices = [] for i in range(NUM_SURR):
amp_windows = np.zeros((n_windows)) effect_windows = np.zeros((n_windows)) mean_amp_windows = np.zeros((n_windows)) mean_ampAAC_windows = np.zeros((n_windows)) amp_windows_surrs = np.zeros((NUM_SURRS, n_windows)) effect_windows_surrs = np.zeros((NUM_SURRS, n_windows)) mean_amp_windows_surrs = np.zeros((NUM_SURRS, n_windows)) mean_ampAAC_windows_surrs = np.zeros((NUM_SURRS, n_windows)) for i, ndx in zip(range(len(ndxs)), ndxs): # copy part of data prg_temp = prg.copy(temporal_ndx = ndx) # get ready for surrs mean, var, trend = prg_temp.get_seasonality(detrend = True) prg_surr = SurrogateField() prg_surr.copy_field(prg_temp) prg_temp.return_seasonality(mean, var, trend) ## COMPUTE FOR DATA prg_temp.wavelet(1, 'y', cut = 1, cut_time = False, cut_data = False, regress_amp_to_data = True) annual_amp = prg_temp.amplitude.copy() annual_phase = prg_temp.phase.copy() prg_temp.anomalise() prg_temp.wavelet(8, 'y', cut = 1, cut_time = False, cut_data = False, regress_amp_to_data = True, continuous_phase = False) amplitude = prg_temp.amplitude.copy() prg_temp.wavelet(8, 'y', cut = 1, cut_time = True, cut_data = True, regress_amp_to_data = False, continuous_phase = False) amplitudeAACreg = prg_temp.amplitude.copy() m, c, r, p, std_err = sts.linregress(amplitudeAACreg*np.cos(prg_temp.phase), annual_amp*np.cos(annual_phase))
ndx = ((phase >= phase_bins[i]) & (phase <= phase_bins[i+1])) data_temp = g_for_avg.data[ndx].copy() time_temp = g_for_avg.time[ndx].copy() tg_sat_temp = tg_avg_sat[ndx].copy() # positive extremes g_e = np.greater_equal(tg_sat_temp, np.mean(tg_avg_sat, axis = 0) + 2 * sigma) avg_bins[i, 0] = np.sum(g_e) # negative extremes l_e = np.less_equal(tg_sat_temp, np.mean(tg_avg_sat, axis = 0) - 2 * sigma) avg_bins[i, 1] = np.sum(l_e) else: sg = SurrogateField() # g_for_avg are SAT data mean, var, trend = g_for_avg.get_seasonality(detrend = True) sg.copy_field(g_for_avg) sg.prepare_AR_surrogates() year = 365.25 k0 = 6. # wavenumber of Morlet wavelet used in analysis fourier_factor = (4 * np.pi) / (k0 + np.sqrt(2 + np.power(k0,2))) period = PERIOD * year # frequency of interest s0 = period / fourier_factor # get scale avg_bins_surr = np.zeros((NUM_SURR, 8, 2)) # num surr x bin no. x result no. (hot / cold extremes) phase_bins = get_equidistant_bins() for surr in range(NUM_SURR):
from scale_network import ScaleSpecificNetwork from datetime import date from pathos.multiprocessing import Pool import numpy as np from src.data_class import DataField import csv import matplotlib.pyplot as plt import src.wavelet_analysis as wvlt from src.surrogates import SurrogateField NUM_SURRS = 1 # fname = '/home/nikola/Work/phd/data/air.mon.mean.sig995.nc' fname = "/Users/nikola/work-ui/data/air.mon.mean.sig995.nc" surrs = SurrogateField() net = ScaleSpecificNetwork(fname, 'air', date(1950, 1, 1), date(2016, 1, 1), None, None, None, 'monthly', anom=False) a = net.get_seasonality(detrend=True) surrs.copy_field(net) # surrs.construct_fourier_surrogates() # surrs.add_seasonality(a[0], a[1], a[2])
date(2004, 1, 1), LATS, LONS, False, parts=3, logger_function=log) idx = g.get_data_of_precise_length( '16k', START_DATE, None, True) # get 2^n data because of MF surrogates if AMPLITUDE: g_amp = g_amp[idx[0]:idx[1], ...] END_DATE = g.get_date_from_ndx(-1) if SURR_TYPE is not None: log("Creating surrogate fields...") sg = SurrogateField() # for MF and FT surrs log("De-seasonalising the data and copying to surrogate field...") mean, var, trend = g.get_seasonality( True) # subtract mean, divide by std and subtract trend from data sg.copy_field(g) # copy standartised data to SurrogateField g.return_seasonality(mean, var, trend) # return seasonality to data log("Surrogate fields created.") if ANOMALISE: g.anomalise() ## wavelet data log("Running wavelet analysis on data using %d workers..." % (WORKERS)) k0 = 6. # wavenumber of Morlet wavelet used in analysis y = 365.25 # year in days fourier_factor = (4 * np.pi) / (k0 + np.sqrt(2 + np.power(k0, 2)))
data_diff = cond_means[:, 0, 0].max() - cond_means[:, 0, 0].min() if SURR: surr_diff = np.zeros((NUM_SURR,)) amp_surr = np.zeros((NUM_SURR,)) surr_completed = 0 jobQ = Queue() resQ = Queue() for i in range(NUM_SURR): jobQ.put(1) for i in range(WORKERS): jobQ.put(None) a = g_amp.get_seasonality(True) sg = SurrogateField() sg.copy_field(g_amp) phase_bins = get_equidistant_bins(BINS) workers = [Process(target = _reconstruction_surrs, args = (sg, a, jobQ, resQ, idx)) for iota in range(WORKERS)] for w in workers: w.start() while surr_completed < NUM_SURR: surr_means = resQ.get() surr_diff[surr_completed] = surr_means[0] amp_surr[surr_completed] = surr_means[1] surr_completed += 1 if (surr_completed % 100) == 0: print("%d. surrogate done..." % surr_completed)
# aa_surr.copy_field(aa) # aa.return_seasonality(aa_seas[0], aa_seas[1], None) names = [['sunspot', 'AAindex'], ['sunspot', 'OuluCR'], ['OuluCR', 'AAindex']] for [idx1, idx2] in names: if idx1 == 'OuluCR': temp, temp_surr, temp_seas = load_cosmic_data("../data/oulu_cosmic_daily.dat", date(1964, 4, 1), date(2009, 2, 8), anom = False, daily = DAILY) # temp, temp_surr, temp_seas = load_CR_climax_daily_data('../data/CR-climax-daily-1-1-94--30-11-06.txt', date(1994,1,1), date(2006,11,30), False) # temp.get_data_of_precise_length('16k', start_date = date(1964, 1, 1), COPY = True) elif idx1 == 'sunspot': temp = load_sunspot_data("../data/sunspot_daily.txt", date(1964, 4, 1), date(2009, 2, 8), anom = False, daily = DAILY) # temp.get_data_of_precise_length(length = 1024, end_date = date(2007, 1, 1), COPY = True) # temp.get_data_of_precise_length('16k', start_date = date(1964, 1, 1), COPY = True) temp_surr = SurrogateField() temp_seas = temp.get_seasonality(True) temp_surr.copy_field(temp) temp.return_seasonality(temp_seas[0], temp_seas[1], temp_seas[2]) elif idx1 == 'AAindex': temp = load_AAgeomag_data("../data/aa_day.raw", date(1964, 4, 1), date(2009, 2, 8), anom = False, daily = DAILY) # temp.get_data_of_precise_length('16k', start_date = date(1964, 1, 1), COPY = True) temp_surr = SurrogateField() temp_seas = temp.get_seasonality(True) temp_surr.copy_field(temp) temp.return_seasonality(temp_seas[0], temp_seas[1], temp_seas[2]) if idx2 == 'OuluCR': aa, aa_surr, aa_seas = load_cosmic_data("../data/oulu_cosmic_daily.dat", date(1964, 4, 1), date(2009, 2, 8), anom = False, daily = DAILY) # aa, aa_surr, aa_seas = load_CR_climax_daily_data('../data/CR-climax-daily-1-1-94--30-11-06.txt', date(1994,1,1), date(2006,11,30), False) elif idx2 == 'sunspot':
if CONDITION: total_surrogates_condition = [] if np.all(np.isnan(g_surrs.data) == False): # construct the job queue jobQ = Queue() resQ = Queue() if CONDITION: for i in range(3 * NUM_SURR): jobQ.put(1) else: for i in range(NUM_SURR): jobQ.put(1) for i in range(WORKERS): jobQ.put(None) a = g_surrs.get_seasonality(DETREND=True) sg = SurrogateField() sg.copy_field(g_surrs) if AMPLITUDE: a_amp = g_surrs_amp.get_seasonality(True) sg_amp = SurrogateField() sg_amp.copy_field(g_surrs_amp) else: sg_amp = None a_amp = None if SURR_TYPE == 'AR': sg.prepare_AR_surrogates() if AMPLITUDE: sg_amp.prepare_AR_surrogates() workers = [ Process(target=_cond_difference_surrogates, args=(sg, sg_amp, g_surrs, a, a_amp, start_cut, jobQ,
g_amp = g.copy_data() log("SAT data copied...") # ERA-40 + ERA-Interim else: g = load_ERA_data_daily('ERA40_EU', 't2m', date(1958,1,1), date(2004,1,1), LATS, LONS, False, parts = 3, logger_function = log) idx = g.get_data_of_precise_length('16k', START_DATE, None, True) # get 2^n data because of MF surrogates if AMPLITUDE: g_amp = g_amp[idx[0] : idx[1], ...] END_DATE = g.get_date_from_ndx(-1) if SURR_TYPE is not None: log("Creating surrogate fields...") sg = SurrogateField() # for MF and FT surrs log("De-seasonalising the data and copying to surrogate field...") mean, var, trend = g.get_seasonality(True) # subtract mean, divide by std and subtract trend from data sg.copy_field(g) # copy standartised data to SurrogateField g.return_seasonality(mean, var, trend) # return seasonality to data log("Surrogate fields created.") if ANOMALISE: g.anomalise() ## wavelet data log("Running wavelet analysis on data using %d workers..." % (WORKERS)) k0 = 6. # wavenumber of Morlet wavelet used in analysis y = 365.25 # year in days