def load_cosmic_data(fname, start_date, end_date, anom=True, daily=False, corrected=True): # corrected stands for if use corrected data or not from dateutil.relativedelta import relativedelta raw = open(fname).read() lines = raw.split('\n') data = [] time = [] d = date(int(lines[0][:4]), int(lines[0][5:7]), 1) if not daily: delta = relativedelta(months=+1) elif daily: delta = timedelta(days=1) for line in lines: row = line.split(' ') if len(row) < 6: continue time.append(d.toordinal()) if corrected: data.append(float(row[4])) else: data.append(float(row[5])) d += delta g = DataField(data=np.array(data), time=np.array(time)) g.location = 'Oulu cosmic data' g.select_date(start_date, end_date) if anom: g.anomalise() g.data = X[:, 0].copy() if NUM_SURR != 0: g_surr = SurrogateField() seasonality = g.get_seasonality(True) g_surr.copy_field(g) g.return_seasonality(seasonality[0], seasonality[1], seasonality[2]) else: g_surr, seasonality = None, None return g, g_surr, seasonality
def load_cosmic_data(fname, start_date, end_date, anom = True, daily = False, corrected = True): # corrected stands for if use corrected data or not from dateutil.relativedelta import relativedelta raw = open(fname).read() lines = raw.split('\n') data = [] time = [] d = date(int(lines[0][:4]), int(lines[0][5:7]), 1) if not daily: delta = relativedelta(months = +1) elif daily: delta = timedelta(days = 1) for line in lines: row = line.split(' ') if len(row) < 6: continue time.append(d.toordinal()) if corrected: data.append(float(row[4])) else: data.append(float(row[5])) d += delta g = DataField(data = np.array(data), time = np.array(time)) g.location = 'Oulu cosmic data' g.select_date(start_date, end_date) if anom: g.anomalise() g.data = X[:, 0].copy() if NUM_SURR != 0: g_surr = SurrogateField() seasonality = g.get_seasonality(True) g_surr.copy_field(g) g.return_seasonality(seasonality[0], seasonality[1], seasonality[2]) else: g_surr, seasonality = None, None return g, g_surr, seasonality
start_year = date.fromordinal(g.time[0]).year + 4 sm = date.fromordinal(g.time[0]).month sd = date.fromordinal(g.time[0]).day start_idx = 0 end_idx = to_wavelet _, _, idx = g.get_data_of_precise_length(WINDOW_LENGTH, date.fromordinal(g.time[4 * y]), None, False) first_mid_year = date.fromordinal(g.time[idx[0] + WINDOW_LENGTH / 2]).year while end_idx < g.data.shape[0]: # data g_working.data = g.data[start_idx:end_idx].copy() g_working.time = g.time[start_idx:end_idx].copy() if np.all(np.isnan(g_working.data) == False): wave, _, _, _ = wavelet_analysis.continous_wavelet( g_working.data, 1, False, wavelet_analysis.morlet, dj=0, s0=s0, j1=0, k0=k0) # perform wavelet phase = np.arctan2(np.imag(wave), np.real(wave)) # get phases from oscillatory modes start_cut = date(start_year + cnt * WINDOW_SHIFT, sm, sd) idx = g_working.get_data_of_precise_length(WINDOW_LENGTH, start_cut,
fit_x = np.vstack([reconstruction, np.ones(reconstruction.shape[0])]).T m, c = np.linalg.lstsq(fit_x, g_amp.data)[0] amplitude = m * amplitude + c print( "Oscillatory series fitted to SAT data with coeff. %.3f and intercept %.3f" % (m, c)) cond_means = np.zeros((BINS, )) def get_equidistant_bins(num): return np.array(np.linspace(-np.pi, np.pi, num + 1)) start_cut = date(1958, 1, 1) g_data.data, g_data.time, idx = g.get_data_of_precise_length( '16k', start_cut, None, False) phase = phase[0, idx[0]:idx[1]] if AMPLITUDE: amplitude = amplitude[idx[0]:idx[1]] # subselect season if EVAL_SEASON: ndx_season = g_data.select_months(season) phase = phase[ndx_season] if AMPLITUDE: amplitude = amplitude[ndx_season] phase_bins = get_equidistant_bins(BINS) # equidistant bins if AMPLITUDE: data_mom = func(amplitude) else:
start_idx = 0 end_idx = to_wavelet _, _, idx = g.get_data_of_precise_length(WINDOW_LENGTH, date.fromordinal(g.time[4 * y]), None, False) first_mid_year = date.fromordinal(g.time[idx[0] + WINDOW_LENGTH / 2]).year last_mid_year = first_mid_year if PLOT_PHASE: phase_total = [] if PLOT_PHASE and not BEGIN: last_day = g.get_date_from_ndx(4 * y) while end_idx < g.data.shape[0]: # data g_working.data = g.data[start_idx:end_idx].copy() g_working.time = g.time[start_idx:end_idx].copy() if AMPLITUDE: g_working_amp.data = g_amp.data[start_idx:end_idx].copy() if np.all(np.isnan(g_working.data) == False): wave, _, _, _ = wavelet_analysis.continous_wavelet( g_working.data, 1, False, wavelet_analysis.morlet, dj=0, s0=s0, j1=0, k0=k0 ) # perform wavelet phase = np.arctan2(np.imag(wave), np.real(wave)) # get phases from oscillatory modes if AMPLITUDE: wave, _, _, _ = wavelet_analysis.continous_wavelet( g_working_amp.data, 1, False, wavelet_analysis.morlet, dj=0, s0=s0_amp, j1=0, k0=k0 ) # perform wavelet amplitude = np.sqrt(np.power(np.real(wave), 2) + np.power(np.imag(wave), 2)) amplitude = amplitude[0, :]
net = ScaleSpecificNetwork('%sair.mon.mean.levels.nc' % path_to_data, 'air', date(1948,1,1), date(2016,1,1), None, None, 0, dataset = "NCEP", sampling = 'monthly', anom = False) pool = Pool(NUM_WORKERS) net.wavelet(1, 'y', pool = pool, cut = 1) net.get_continuous_phase(pool = pool) net.get_phase_fluctuations(rewrite = True, pool = pool) pool.close() pool.join() nao = DataField() raw = np.loadtxt("%sNAO.station.monthly.1865-2016.txt" % (path_to_data)) raw = raw[:, 1:] nao.data = raw.reshape(-1) nao.create_time_array(date_from = date(1865, 1, 1), sampling = 'm') nao.select_date(date(1949, 1, 1), date(2015, 1, 1)) nao.anomalise() jfm_index = nao.select_months([1,2,3], apply_to_data = False) jfm_nao = nao.data[jfm_index] _, _, y = nao.extract_day_month_year() y = y[jfm_index] ann_nao = [] for year in np.unique(y): ann_nao.append(np.mean(jfm_nao[np.where(year == y)[0]])) ann_nao = np.array(ann_nao) ann_phase_fluc = np.zeros([ann_nao.shape[0]] + list(net.get_spatial_dims()))
if USE_SURR: result_temp_surr = np.zeros((NUM_SURR, 8, 2)) for surr in range(NUM_SURR): sg.construct_surrogates_with_residuals() sg.add_seasonality(mean[:-1], var[:-1], trend[:-1]) # so SAT data g.data = sg.surr_data.copy() tg_sat = g.copy_data() g.time = g.time[:-1] g.anomalise() g_temp = DataField() tg_temp = tg_sat.copy() sy = int(MIDDLE_YEAR - (WINDOW_LENGTH / year) / 2) g_temp.data = g.data.copy() g_temp.time = g.time.copy() start = g_temp.find_date_ndx(date(sy - 4, sm, sd)) end = start + 16384 if WINDOW_LENGTH < 16000 else start + 32768 g_temp.data = g_temp.data[start:end] g_temp.time = g_temp.time[start:end] tg_temp = tg_temp[start:end] k0 = 6. # wavenumber of Morlet wavelet used in analysis fourier_factor = (4 * np.pi) / (k0 + np.sqrt(2 + np.power(k0, 2))) period = PERIOD * year # frequency of interest s0 = period / fourier_factor # get scale wave, _, _, _ = wvlt.continous_wavelet(g_temp.data, 1, False,
resq.put([cond_temp, amp_diff, data_diff, data_diff]) # plt.figure(figsize=(20,10)) # plt.plot(amplitude, color = '#867628', linewidth = 2) # plt.plot(g_amp.data, color = '#004739', linewidth = 1) # plt.show() cond_means = np.zeros((BINS, 2)) start_cut = date(1962,1,1) # 1958, 1, 1 # l = int(16384 - 8*y) l = 17532 g_data.data, g_data.time, idx = g.get_data_of_precise_length(l, start_cut, None, False) # 16k phase = phase[0, idx[0] : idx[1]] amplitude = amplitude[idx[0] : idx[1]] # amp_to_plot = amp_to_plot[idx[0] : idx[1]] # amp_to_plot = g_data.copy_data() phase_bins = get_equidistant_bins(BINS) for i in range(cond_means.shape[0]): ndx = ((phase >= phase_bins[i]) & (phase <= phase_bins[i+1])) cond_means[i, 0] = np.mean(amplitude[ndx]) cond_means[i, 1] = np.mean(g_data.data[ndx]) # cond_means[i] = np.mean(g_data.data[ndx]) # if SURR: # cond_means[i, 1] = np.mean(amplitude2[ndx])
if STATIONS == None: if SEASON == None: cond_means = np.zeros((BINS, 2, 1)) else: cond_means = np.zeros((BINS, 2, 2)) else: cond_means = np.zeros((BINS, 2, 2)) def get_equidistant_bins(num): return np.array(np.linspace(-np.pi, np.pi, num+1)) # start_cut = date(1958,1,1) start_cut = date(1962,1,1) l = 17532 if STATIONS == None: g_data.data, g_data.time, idx = g.get_data_of_precise_length(l, start_cut, None, False) print g_data.get_date_from_ndx(0), g_data.get_date_from_ndx(-1) phase = phase[0, idx[0] : idx[1]] if AMPLITUDE: amplitude = amplitude[idx[0] : idx[1]] else: for i in range(len(STATIONS)): locals()['g_data' + str(i)].data, locals()['g_data' + str(i)].time, idx = locals()['g' + str(i)].get_data_of_precise_length('16k', start_cut, None, False) locals()['phase' + str(i)] = locals()['phase' + str(i)][0, idx[0] : idx[1]] if AMPLITUDE: locals()['amplitude' + str(i)] = locals()['amplitude' + str(i)][idx[0] : idx[1]] phase_bins = get_equidistant_bins(BINS) mons = {0: 'J', 1: 'F', 2: 'M', 3: 'A', 4: 'M', 5: 'J', 6: 'J', 7: 'A', 8: 'S', 9: 'O', 10: 'N', 11: 'D'} if SEASON != None: idx = 0
if USE_SURR: result_temp_surr = np.zeros((NUM_SURR, 8,2)) for surr in range(NUM_SURR): sg.construct_surrogates_with_residuals() sg.add_seasonality(mean[:-1], var[:-1], trend[:-1]) # so SAT data g.data = sg.surr_data.copy() tg_sat = g.copy_data() g.time = g.time[:-1] g.anomalise() g_temp = DataField() tg_temp = tg_sat.copy() sy = int(MIDDLE_YEAR - (WINDOW_LENGTH/year)/2) g_temp.data = g.data.copy() g_temp.time = g.time.copy() start = g_temp.find_date_ndx(date(sy - 4, sm, sd)) end = start + 16384 if WINDOW_LENGTH < 16000 else start + 32768 g_temp.data = g_temp.data[start : end] g_temp.time = g_temp.time[start : end] tg_temp = tg_temp[start : end] k0 = 6. # wavenumber of Morlet wavelet used in analysis fourier_factor = (4 * np.pi) / (k0 + np.sqrt(2 + np.power(k0,2))) period = PERIOD * year # frequency of interest s0 = period / fourier_factor # get scale wave, _, _, _ = wvlt.continous_wavelet(g_temp.data, 1, False, wvlt.morlet, dj = 0, s0 = s0, j1 = 0, k0 = k0) # perform wavelet phase = np.arctan2(np.imag(wave), np.real(wave)) # get phases from oscillatory modes
0, dataset="NCEP", sampling='monthly', anom=False) pool = Pool(NUM_WORKERS) net.wavelet(1, 'y', pool=pool, cut=1) net.get_continuous_phase(pool=pool) net.get_phase_fluctuations(rewrite=True, pool=pool) pool.close() pool.join() nao = DataField() raw = np.loadtxt("%sWeMO.monthly.1821-2013.txt" % (path_to_data)) raw = raw[:, 1:] nao.data = raw.reshape(-1) nao.create_time_array(date_from=date(1821, 1, 1), sampling='m') nao.select_date(date(1949, 1, 1), date(2014, 1, 1)) nao.anomalise() jfm_index = nao.select_months([1, 2, 3], apply_to_data=False) jfm_nao = nao.data[jfm_index] _, _, y = nao.extract_day_month_year() y = y[jfm_index] ann_nao = [] for year in np.unique(y): ann_nao.append(np.mean(jfm_nao[np.where(year == y)[0]])) ann_nao = np.array(ann_nao) ann_phase_fluc = np.zeros([ann_nao.shape[0]] + list(net.get_spatial_dims()))
import calendar ts = OscillatoryTimeSeries('TG_STAID000027.txt', date(1834, 7, 28), date(2014, 1, 1), False) sg = SurrogateField() g = DataField() daily_var = np.zeros((365, 3)) mean, var_data, trend = ts.g.get_seasonality(True) sg.copy_field(ts.g) #MF sg.construct_multifractal_surrogates() sg.add_seasonality(mean, var_data, trend) g.data = sg.surr_data.copy() g.time = sg.time.copy() _, var_surr_MF, _ = g.get_seasonality(True) #FT sg.construct_fourier_surrogates_spatial() sg.add_seasonality(mean, var_data, trend) g.data = sg.surr_data.copy() g.time = sg.time.copy() _, var_surr_FT, _ = g.get_seasonality(True) delta = timedelta(days=1) d = date(1895, 1, 1)
def _corrs_surrs_ind(args): index_surr = DataField() index_surr.data = get_single_FT_surrogate(index_data.data) index_correlations_surrs = get_corrs(net, index_surr) return index_correlations_surrs
pool.close() pool.join() # index_correlations = {} # index_datas = {} # # SURROGATES # for index, ndx_type, start_date, end_year in zip(INDICES, DATE_TYPE, START_DATES, END_YEARS): # load index # print index # if index != 'NINO3.4': index_data = DataField() raw = np.loadtxt("%sNAO.station.monthly.1865-2016.txt" % (path_to_data)) raw = raw[:, 1:] index_data.data = raw.reshape(-1) index_data.create_time_array(date_from=date(1865, 1, 1), sampling='m') index_data.select_date(date(1951, 1, 1), date(2014, 1, 1)) index_data.anomalise() index_correlations = get_corrs(net, index_data) # with open("20CRtemp-phase-fluct-corr-with-%sindex-1950-2014.bin" % index, "wb") as f: # cPickle.dump({('%scorrs' % index) : index_correlations[index].reshape(np.prod(index_correlations[index].shape))}, f) # # plotting # tit = ("ECA&D annual phase SSA RC fluctuations x %s correlations" % index) # fname = ("../scale-nets/ECAD-SAT-annual-phase-fluc-SSA-RC-%scorrs.png" % index) # net.quick_render(field_to_plot = index_correlations[index], tit = tit, symm = True, whole_world = False, fname = fname) # def _corrs_surrs(args): # index_correlations_surrs = {}
pool.close() pool.join() # index_correlations = {} # index_datas = {} # # SURROGATES # for index, ndx_type, start_date, end_year in zip(INDICES, DATE_TYPE, START_DATES, END_YEARS): # load index # print index # if index != 'NINO3.4': index_data = DataField() raw = np.loadtxt("%sNAO.station.monthly.1865-2016.txt" % (path_to_data)) raw = raw[:, 1:] index_data.data = raw.reshape(-1) index_data.create_time_array(date_from = date(1865, 1, 1), sampling = 'm') index_data.select_date(date(1951, 1, 1), date(2014, 1, 1)) index_data.anomalise() index_correlations = get_corrs(net, index_data) # with open("20CRtemp-phase-fluct-corr-with-%sindex-1950-2014.bin" % index, "wb") as f: # cPickle.dump({('%scorrs' % index) : index_correlations[index].reshape(np.prod(index_correlations[index].shape))}, f) # # plotting # tit = ("ECA&D annual phase SSA RC fluctuations x %s correlations" % index) # fname = ("../scale-nets/ECAD-SAT-annual-phase-fluc-SSA-RC-%scorrs.png" % index) # net.quick_render(field_to_plot = index_correlations[index], tit = tit, symm = True, whole_world = False, fname = fname) # def _corrs_surrs(args):
def _corrs_surrs_ind(args): index_surr = DataField() index_surr.data = get_single_FT_surrogate(index_data.data) index_correlations_surrs = get_corrs(net, index_surr) return index_correlations_surrs
ts = OscillatoryTimeSeries('TG_STAID000027.txt', date(1834,7,28), date(2014,1,1), False) sg = SurrogateField() g = DataField() daily_var = np.zeros((365,3)) mean, var_data, trend = ts.g.get_seasonality(True) sg.copy_field(ts.g) #MF sg.construct_multifractal_surrogates() sg.add_seasonality(mean, var_data, trend) g.data = sg.surr_data.copy() g.time = sg.time.copy() _, var_surr_MF, _ = g.get_seasonality(True) #FT sg.construct_fourier_surrogates_spatial() sg.add_seasonality(mean, var_data, trend) g.data = sg.surr_data.copy() g.time = sg.time.copy() _, var_surr_FT, _ = g.get_seasonality(True) delta = timedelta(days = 1) d = date(1895,1,1)