for PERIOD in periods: for AVG in avg_to: print("computing for %d year period and averaging up to %d" % (PERIOD, 12 * AVG * PERIOD)) net = ScaleSpecificNetwork( '/home/nikola/Work/phd/data/air.mon.mean.levels.nc', 'air', date(1948, 1, 1), date(2014, 1, 1), None, None, 0, 'monthly', anom=True) pool = Pool(WORKERS) net.wavelet(PERIOD, get_amplitude=True, pool=pool) print "wavelet on data done" net.get_filtered_data(pool=pool) print "filtered data acquired" autocoherence = np.zeros(net.get_spatial_dims()) job_args = [(i, j, int(AVG * 12 * PERIOD), net.filtered_data[:, i, j]) for i in range(net.lats.shape[0]) for j in range(net.lons.shape[0])] job_result = pool.map(_get_autocoherence, job_args) del job_args pool.close() for i, j, res in job_result: autocoherence[i, j] = res del job_result
print("computing phase conditioned on NAO") nao_phase = load_NAOindex_wavelet_phase(date(1950, 1, 1), date(2014, 1, 1), period, False) net = ScaleSpecificNetwork( '/home/nikola/Work/phd/data/air.mon.mean.levels.nc', 'air', date(1950, 1, 1), date(2014, 1, 1), None, None, level=0, dataset="NCEP", sampling='monthly', anom=False) pool = Pool(WORKERS) net.wavelet(period, period_unit="y", pool=pool, cut=2) print("wavelet on data done") pool.close() pool.join() net.get_adjacency_matrix_conditioned(nao_phase, use_queue=True, num_workers=WORKERS) print("estimating adjacency matrix done") net.save_net('networks/NCEP-SAT%dy-phase-adjmatCMIEQQcondNAOphase.bin' % (period), only_matrix=True) print("computing phase conditioned on NINO") nino_phase = load_nino34_wavelet_phase(date(1950, 1, 1), date(2014, 1, 1), period, False) net.get_adjacency_matrix_conditioned(nino_phase,
fname = '/home/nikola/Work/phd/data/air.mon.mean.sig995.nc' # fname = "/Users/nikola/work-ui/data/air.mon.mean.sig995.nc" ## PHASE FLUCTUATIONS NETWORK L2 dist. print "Computing L2 distance..." net = ScaleSpecificNetwork(fname, 'air', date(1950, 1, 1), date(2016, 1, 1), None, None, None, 'monthly', anom=False) pool = Pool(20) net.wavelet(1, 'y', pool=pool, cut=1) net.get_continuous_phase(pool=pool) print "wavelet done" net.get_phase_fluctuations(rewrite=True, pool=pool) print "fluctuations done" pool.close() pool.join() net.phase_fluctuations -= np.nanmean(net.phase_fluctuations, axis=0) net.get_adjacency_matrix(net.phase_fluctuations, method="L2", pool=None, use_queue=True, num_workers=20) net.save_net('networks/NCEP-SATannual-phase-fluctuations-adjmatL2.bin', only_matrix=True) print "L2 done"
date(2016, 1, 1), [-60, 0], [40, 100], level=0, dataset="NCEP", sampling="monthly", anom=False, ) surrs = SurrogateField() a = net.get_seasonality(detrend=True) surrs.copy_field(net) net.return_seasonality(a[0], a[1], a[2]) pool = Pool(20) net.wavelet(8, "y", cut=1, pool=pool) net.get_adjacency_matrix(net.phase, method="MIEQQ", num_workers=0, pool=pool, use_queue=False) pool.close() pool.join() data_adj_matrix = net.adjacency_matrix.copy() surrs_adj_matrices = [] for i in range(NUM_SURR): print("surr %d/%d computing..." % (i + 1, NUM_SURR)) pool = Pool(20) surrs.construct_fourier_surrogates(pool=pool) surrs.add_seasonality(a[0], a[1], a[2]) net.data = surrs.get_surr()
for method in METHODS: for scale in SCALES: print("Computing networks using %s method..." % (method)) # phase if method in ['MIEQQ', 'MIGAU', 'MPC']: # net = ScaleSpecificNetwork(fname, 'air', date(1948,1,1), date(2016,1,1), None, None, level = 0, dataset = "NCEP", # sampling = 'monthly', anom = False) net = ScaleSpecificNetwork(fname, 't2m', date(1958,1,1), date(2014,1,1), None, None, level=None, pickled=True, sampling='monthly', anom=False) pool = Pool(NUM_WORKERS) # net.get_hilbert_phase_amp(period = 90, width = 12, pool = pool, cut = 1) net.wavelet(scale, period_unit='m', cut=2, pool=pool) pool.close() pool.join() net.get_adjacency_matrix(net.phase, method = method, pool = None, use_queue = True, num_workers = NUM_WORKERS) net.save_net('networks/ERA-SATsurface-scale%dmonths-phase-adjmat%s.bin' % (scale, method), only_matrix = True) # amplitude if method in ['MIEQQ', 'MIGAU', 'CORR']: # net = ScaleSpecificNetwork(fname, 'air', date(1948,1,1), date(2016,1,1), None, None, level = 0, dataset = "NCEP", # sampling = 'monthly', anom = False) net = ScaleSpecificNetwork(fname, 't2m', date(1958,1,1), date(2014,1,1), None, None, level=None, pickled=True, sampling='monthly', anom=False) pool = Pool(NUM_WORKERS) # net.get_hilbert_phase_amp(period = 90, width = 12, pool = pool, cut = 1) net.wavelet(scale, period_unit='m', cut=2, pool=pool) pool.close()
import matplotlib.pyplot as plt import numpy as np WORKERS = 10 print "computing SAT wavelet coherence..." to_do = [['WCOH', 4], ['WCOH', 6], ['WCOH', 8], ['WCOH', 11], ['WCOH', 15]] for do in to_do: METHOD = do[0] PERIOD = do[1] print("computing for %d period using %s method" % (PERIOD, METHOD)) net = ScaleSpecificNetwork('/home/nikola/Work/phd/data/air.mon.mean.levels.nc', 'air', date(1948,1,1), date(2014,1,1), None, None, 0, 'monthly', anom = False) pool = Pool(WORKERS) net.wavelet(PERIOD, get_amplitude = False, save_wavelet = True, pool = pool) print "wavelet on data done" pool.close() net.get_adjacency_matrix(net.wave, method = METHOD, pool = None, use_queue = True, num_workers = WORKERS) print "estimating adjacency matrix done" net.save_net('networks/NCEP-SATsurface-wave-adjmat%s-scale%dyears.bin' % (METHOD, PERIOD), only_matrix = True) print "computing SATA wavelet coherence..." to_do = [['WCOH', 4], ['WCOH', 6], ['WCOH', 8], ['WCOH', 11], ['WCOH', 15]] for do in to_do: METHOD = do[0] PERIOD = do[1] print("computing for %d period using %s method" % (PERIOD, METHOD)) net = ScaleSpecificNetwork('/home/nikola/Work/phd/data/air.mon.mean.levels.nc', 'air',
# del job_result # with open("networks/NCEP-SATAsurface-autocoherence-phase-scale%dyears-avg-to-%.1f.bin" % (PERIOD, AVG), "wb") as f: # cPickle.dump({'autocoherence' : autocoherence, 'lats' : net.lats, 'lons' : net.lons}, f, protocol = cPickle.HIGHEST_PROTOCOL) if not PLOT: ## autocoherence filtered data - SATA print "computing autocoherence for SATA filtered data" for PERIOD in periods: for AVG in avg_to: print("computing for %d year period and averaging up to %d" % (PERIOD, 12*AVG*PERIOD)) net = ScaleSpecificNetwork('/home/nikola/Work/phd/data/air.mon.mean.levels.nc', 'air', date(1948,1,1), date(2014,1,1), None, None, 0, 'monthly', anom = True) pool = Pool(WORKERS) net.wavelet(PERIOD, get_amplitude = True, pool = pool) print "wavelet on data done" net.get_filtered_data(pool = pool) print "filtered data acquired" autocoherence = np.zeros(net.get_spatial_dims()) job_args = [ (i, j, int(AVG*12*PERIOD), net.filtered_data[:, i, j]) for i in range(net.lats.shape[0]) for j in range(net.lons.shape[0]) ] job_result = pool.map(_get_autocoherence, job_args) del job_args pool.close() for i, j, res in job_result: autocoherence[i, j] = res del job_result with open("networks/NCEP-SATAsurface-autocoherence-filtered-scale%dyears-avg-to-%.1f.bin" % (PERIOD, AVG), "wb") as f: cPickle.dump({'autocoherence' : autocoherence, 'lats' : net.lats, 'lons' : net.lons}, f, protocol = cPickle.HIGHEST_PROTOCOL)
for do in to_do: METHOD = do[0] PERIOD = do[1] print("computing for %d period using %s method" % (PERIOD, METHOD)) net = ScaleSpecificNetwork( '/home/nikola/Work/phd/data/air.mon.mean.levels.nc', 'air', date(1948, 1, 1), date(2014, 1, 1), None, None, 0, 'monthly', anom=False) pool = Pool(WORKERS) net.wavelet(PERIOD, get_amplitude=False, save_wavelet=True, pool=pool) print "wavelet on data done" pool.close() net.get_adjacency_matrix(net.wave, method=METHOD, pool=None, use_queue=True, num_workers=WORKERS) print "estimating adjacency matrix done" net.save_net('networks/NCEP-SATsurface-wave-adjmat%s-scale%dyears.bin' % (METHOD, PERIOD), only_matrix=True) print "computing SATA wavelet coherence..." to_do = [['WCOH', 4], ['WCOH', 6], ['WCOH', 8], ['WCOH', 11], ['WCOH', 15]] for do in to_do:
date(1948, 1, 1), date(2016, 1, 1), None, None, level=0, dataset="NCEP", sampling='monthly', anom=False) surrs = SurrogateField() a = net.get_seasonality(detrend=True) surrs.copy_field(net) net.return_seasonality(a[0], a[1], a[2]) pool = Pool(20) net.wavelet(8, 'y', cut=1, pool=pool) net.get_adjacency_matrix(net.phase, method="MIEQQ", num_workers=20, pool=None, use_queue=True) pool.close() pool.join() data_adj_matrix = net.adjacency_matrix.copy() surrs_adj_matrices = [] for i in range(NUM_SURR): print("surr %d/%d computing..." % (i + 1, NUM_SURR)) pool = Pool(20)
g.wavelet(period, period_unit="y", cut=2) return g.phase.copy() WORKERS = 20 to_do_periods = [4, 6, 8, 11, 15] for period in to_do_periods: print("computing phase conditioned on NAO") nao_phase = load_NAOindex_wavelet_phase(date(1950,1,1), date(2014,1,1), period, False) net = ScaleSpecificNetwork('/home/nikola/Work/phd/data/air.mon.mean.levels.nc', 'air', date(1950,1,1), date(2014,1,1), None, None, level = 0, dataset="NCEP", sampling='monthly', anom=False) pool = Pool(WORKERS) net.wavelet(period, period_unit="y", pool=pool, cut=2) print("wavelet on data done") pool.close() pool.join() net.get_adjacency_matrix_conditioned(nao_phase, use_queue=True, num_workers=WORKERS) print("estimating adjacency matrix done") net.save_net('networks/NCEP-SAT%dy-phase-adjmatCMIEQQcondNAOphase.bin' % (period), only_matrix=True) print("computing phase conditioned on NINO") nino_phase = load_nino34_wavelet_phase(date(1950,1,1), date(2014,1,1), period, False) net.get_adjacency_matrix_conditioned(nino_phase, use_queue=True, num_workers=WORKERS) print("estimating adjacency matrix done") net.save_net('networks/NCEP-SAT%dy-phase-adjmatCMIEQQcondNINOphase.bin' % (period), only_matrix=True) print("computing phase conditioned on sunspots") sunspot_phase = load_sunspot_number_phase(date(1950,1,1), date(2014,1,1), period, False)
None, 0, "monthly", anom=True, ) if var: net.get_seasonality(det) if not var and det: continue if c: net.data *= net.latitude_cos_weights() pool = Pool(3) net.wavelet(per, get_amplitude=True, pool=pool) net.get_filtered_data(pool=pool) pool.close() fname = "filt-data/SATA-1000hPa-filtered%dperiod-%svarnorm-%sdetrend-%scosweighting.bin" % ( per, "" if var else "NO", "" if det else "NO", "" if c else "NO", ) with open(fname, "wb") as f: cPickle.dump( {"filt. data": net.filtered_data, "lats": net.lats, "lons": net.lons, "time": net.time}, f, protocol=cPickle.HIGHEST_PROTOCOL,
_, nao_ph, sg_nao, a_nao = load_NAOindex_wavelet_phase(date(1950, 1, 1), date(2014, 1, 1), period, anom=False) _, nino_ph, sg_nino, a_nino = load_nino34_wavelet_phase(date(1950, 1, 1), date(2014, 1, 1), period, anom=False) _, sunspots_ph, sg_sunspots, a_sunspots = load_sunspot_number_phase( date(1950, 1, 1), date(2014, 1, 1), period, anom=False) _, pdo_ph, sg_pdo, a_pdo = load_pdo_phase(date(1950, 1, 1), date(2014, 1, 1), period, anom=False) pool = Pool(WORKERS) net.wavelet(period, period_unit='y', cut=2, pool=pool) args = [(net.phase[:, i, j], i, j, nao_ph, nino_ph, sunspots_ph, pdo_ph) for i in range(net.lats.shape[0]) for j in range(net.lons.shape[0])] result = pool.map(_compute_MI_synch, args) synchs = np.zeros((4, net.lats.shape[0], net.lons.shape[0])) synchs_surrs = np.zeros( (NUM_SURRS, 4, net.lats.shape[0], net.lons.shape[0])) for i, j, naos, ninos, suns, pdos in result: synchs[0, i, j] = naos synchs[1, i, j] = ninos synchs[2, i, j] = suns synchs[3, i, j] = pdos for surr in range(NUM_SURRS): sg_nao.construct_fourier_surrogates(algorithm='FT') sg_nao.add_seasonality(a_nao[0], a_nao[1], a_nao[2])
WORKERS = 5 NUM_SURRS = 100 to_do_periods = np.arange(2,15.5,0.5) net = ScaleSpecificNetwork('/Users/nikola/work-ui/data/NCEP/air.mon.mean.levels.nc', 'air', date(1950,1,1), date(2014,1,1), None, None, level = 0, dataset="NCEP", sampling='monthly', anom=False) synchronization = {} for period in to_do_periods: print("running for %.1f period..." % (period)) _, nao_ph, sg_nao, a_nao = load_NAOindex_wavelet_phase(date(1950,1,1), date(2014,1,1), period, anom=False) _, nino_ph, sg_nino, a_nino = load_nino34_wavelet_phase(date(1950,1,1), date(2014,1,1), period, anom=False) _, sunspots_ph, sg_sunspots, a_sunspots = load_sunspot_number_phase(date(1950,1,1), date(2014,1,1), period, anom=False) _, pdo_ph, sg_pdo, a_pdo = load_pdo_phase(date(1950,1,1), date(2014,1,1), period, anom=False) pool = Pool(WORKERS) net.wavelet(period, period_unit='y', cut=2, pool=pool) args = [(net.phase[:, i, j], i, j, nao_ph, nino_ph, sunspots_ph, pdo_ph) for i in range(net.lats.shape[0]) for j in range(net.lons.shape[0])] result = pool.map(_compute_MI_synch, args) synchs = np.zeros((4, net.lats.shape[0], net.lons.shape[0])) synchs_surrs = np.zeros((NUM_SURRS, 4, net.lats.shape[0], net.lons.shape[0])) for i, j, naos, ninos, suns, pdos in result: synchs[0, i, j] = naos synchs[1, i, j] = ninos synchs[2, i, j] = suns synchs[3, i, j] = pdos for surr in range(NUM_SURRS): sg_nao.construct_fourier_surrogates(algorithm='FT') sg_nao.add_seasonality(a_nao[0], a_nao[1], a_nao[2]) sg_nao.wavelet(period, period_unit="y", cut=2) sg_nino.construct_fourier_surrogates(algorithm='FT') sg_nino.add_seasonality(a_nino[0], a_nino[1], a_nino[2])
None, None, 0, 'monthly', anom=True) if var: net.get_seasonality(det) if not var and det: continue if c: net.data *= net.latitude_cos_weights() pool = Pool(3) net.wavelet(per, get_amplitude=True, pool=pool) net.get_filtered_data(pool=pool) pool.close() fname = ( "filt-data/SATA-1000hPa-filtered%dperiod-%svarnorm-%sdetrend-%scosweighting.bin" % (per, '' if var else 'NO', '' if det else 'NO', '' if c else 'NO')) with open(fname, 'wb') as f: cPickle.dump( { 'filt. data': net.filtered_data, 'lats': net.lats, 'lons': net.lons, 'time': net.time
# cPickle.dump({'mean_phase_diff' : np.mean(phase_diffs, axis = 0).flatten(), # 'std_phase_diff' : np.std(phase_diffs, axis = 0, ddof = 1).flatten(), # 'var_phase_diff' : np.var(phase_diffs, axis = 0, ddof = 1).flatten()}, f, protocol = cPickle.HIGHEST_PROTOCOL) to_do = [['MIGAU', 8], ['MIGAU', 6]] for do in to_do: METHOD = do[0] PERIOD = do[1] print("computing for %d period using %s method" % (PERIOD, METHOD)) net = ScaleSpecificNetwork('/home/nikola/Work/phd/data/air.mon.mean.levels.nc', 'air', date(1958,1,1), date(2014,1,1), None, None, 0, 'monthly', anom = True) pool = Pool(WORKERS) net.wavelet(PERIOD, get_amplitude = False, pool = pool) print "wavelet on data done" pool.close() net.get_adjacency_matrix(net.phase, method = METHOD, pool = None, use_queue = True, num_workers = WORKERS) print "estimating adjacency matrix done" net.save_net('networks/NCEP-SATAsurface-phase-span-as-ERA-adjmat%s-scale%dyears.bin' % (METHOD, PERIOD), only_matrix = True) # print phase_diffs.shape # net.get_adjacency_matrix(phase_diffs, method = METHOD, pool = None, use_queue = True, num_workers = WORKERS) # net.save_net('networks/NCEP-ERA-phase-diff-adjmat%s-scale%dyears.bin' % (METHOD, PERIOD), only_matrix = True) # net.get_adjacency_matrix(net.phase, method = METHOD, pool = None, use_queue = True, num_workers = WORKERS) # print "estimating adjacency matrix done" # net.save_net('networks/ERA-SATAsurface-phase-adjmat%s-scale%dyears.bin' % (METHOD, PERIOD), only_matrix = True)
# net.get_adjacency_matrix_conditioned(nino34_phase, use_queue = True, num_workers = WORKERS) # print "estimating adjacency matrix done" # net.save_net('networks/NCEP-SAT%dy-phase-adjmatCMIEQQcondNINOphase.bin' % (PERIOD), only_matrix = True) print "computing phase conditioned on NAO" to_do = [4, 6, 11, 15] for PERIOD in to_do: nao_phase = load_NAOindex_wavelet_phase(date(1948, 1, 1), date(2014, 1, 1), True) net = ScaleSpecificNetwork( '/home/nikola/Work/phd/data/air.mon.mean.levels.nc', 'air', date(1950, 1, 1), date(2014, 1, 1), None, None, 0, 'monthly', anom=False) pool = Pool(WORKERS) net.wavelet(PERIOD, get_amplitude=False, pool=pool) print "wavelet on data done" pool.close() net.get_adjacency_matrix_conditioned(nao_phase, use_queue=True, num_workers=WORKERS) print "estimating adjacency matrix done" net.save_net('networks/NCEP-SAT%dy-phase-adjmatCMIEQQcondNAOphase.bin' % (PERIOD), only_matrix=True)
def _hilbert_ssa(args): i, j, data = args if not np.any(np.isnan(data)): ssa = ssa_class(data, M = 12) _, _, _, rc = ssa.run_ssa() real_part = rc[:, 0] + rc[:, 1] imag_part = np.imag(hilbert(real_part)) phase_hilb_rc = np.arctan2(imag_part[12:-12], real_part[12:-12]) else: phase_hilb_rc = np.nan return i, j, phase_hilb_rc pool = Pool(NUM_WORKERS) net.wavelet(1, 'y', pool = pool, cut = 1) # Hilbert on RC SSA fluctuations # args = [ (i, j, net.data[:, i, j]) for i in range(net.lats.shape[0]) for j in range(net.lons.shape[0]) ] # results = pool.map(_hilbert_ssa, args) # for i, j, res in results: # net.phase[:, i, j] = res net.get_continuous_phase(pool = pool) net.get_phase_fluctuations(rewrite = True, pool = pool) pool.close() pool.join() # index_correlations = {} # index_datas = {}
SCALES = np.arange(24, 186, 6) # 2 - 15yrs, 0.5yr step, in months METHODS = ['MIEQQ', 'MIKNN'] for method in METHODS: for scale in SCALES: print("Computing networks for %d month scale using %s method..." % (scale, method)) # phase net = ScaleSpecificNetwork(fname, 'air', date(1948,1,1), date(2016,1,1), None, None, level = 0, dataset = "NCEP", sampling = 'monthly', anom = False) pool = Pool(NUM_WORKERS) net.wavelet(scale, 'm', pool = pool, cut = 1) pool.close() pool.join() net.get_adjacency_matrix(net.phase, method = method, pool = None, use_queue = True, num_workers = NUM_WORKERS) net.save_net('networks/NCEP-SATsurface-scale%dmonths-phase-adjmat%s.bin' % (scale, method), only_matrix = True) # amplitude net = ScaleSpecificNetwork(fname, 'air', date(1948,1,1), date(2016,1,1), None, None, level = 0, dataset = "NCEP", sampling = 'monthly', anom = False) pool = Pool(NUM_WORKERS) net.wavelet(scale, 'm', pool = pool, cut = 1) pool.close() pool.join() net.get_adjacency_matrix(net.amplitude, method = method, pool = None, use_queue = True, num_workers = NUM_WORKERS) net.save_net('networks/NCEP-SATsurface-scale%dmonths-amplitude-adjmat%s.bin' % (scale, method), only_matrix = True)