def run_LSP(mod_data, x):
lat_i = lat_indices[x]
lon_i = lon_indices[x]
print lat_i, lon_i
current_lat = lat_c[lat_i]
current_lon = lon_c[lon_i]
waveform = mod_data
waveform_ave = np.average(waveform)
model_date_val = np.copy(model_date)
model_time_val = np.copy(model_time)
time = modules.date_process(model_date_val, model_time_val, start_year)
if (species.lower() !=
'gmao_temp') and (species.lower() != 'gmao_psfc') and (
species.lower() != 'wind_speed') and (species.lower() !=
'wind_direction'):
waveform = waveform * 1e9
#check model vals are valid
#valid = vals >= 0
#vals = vals[valid]
#model_time_val = model_time[valid]
#model_date_val = model_date[valid]
#take 8 hour average
divisor = 8
total_len = len(waveform) / divisor
start = 0
end = divisor
ave_waveform = []
ave_time = []
for i in range(total_len):
ave = np.ma.average(waveform[start:end])
ave_time = np.append(ave_time, time[start])
ave_waveform = np.append(ave_waveform, ave)
start += divisor
end += divisor
time = np.copy(ave_time)
waveform = np.copy(ave_waveform)
#take lsp unwindowed of waveform
ua_periods, ua_mag, ua_ph, ua_fr, ua_fi = modules.take_lomb_unwindowed(
time, waveform, ofac, 1. / 24)
#take out known periodic components 1,182.625, and 365.25 a priori for more accurate red noise fit.
closest_daily_index = min(range(len(ua_periods)),
key=lambda i: abs(ua_periods[i] - 1.))
closest_ha_index = min(range(len(ua_periods)),
key=lambda i: abs(ua_periods[i] - 182.625))
closest_annual_index = min(range(len(ua_periods)),
key=lambda i: abs(ua_periods[i] - 365.25))
rm_indices = [closest_daily_index, closest_ha_index, closest_annual_index]
ua_mag_c, ua_fr, ua_fi = redfit.sidelobe_percent_remove(
np.copy(ua_mag), ua_fr, ua_fi, rm_indices, 5., ua_periods)
#-------------------------------------------------------------------------------
#Do IFFT of altered spectra - with significant periods removed and gaps left in real and imag components linearly interpolated.
#altered spectra provides red noise estimation baseline
##use ifft to get time series back from adjusted spectra
#complex Fourier spectrum which corresponds to the Lomb-Scargle periodogram:
F = [0] * ((len(ua_fr) * 2) + 1)
#set first real value to average
F[0] = complex(waveform_ave * len(waveform), 0)
#Get reverse real and imaginary values
rev_ua_fr = np.copy(ua_fr[::-1])
rev_ua_fi = np.copy(ua_fi[::-1])
rev_ua_fr[0] = 0
rev_ua_fi[0] = 0
f_index = 1
#Fill Fourier Spectrum real and imaginary values
for i in range(len(ua_fr)):
F[f_index] = complex(ua_fr[i], ua_fi[i])
f_index += 1
for i in range(len(ua_fr)):
F[f_index] = complex(rev_ua_fr[i], -rev_ua_fi[i])
f_index += 1
F = np.array(F)
#Take ifft and just take real values
ifft_ua_ts = numpy.fft.ifft(F)
ifft_ua_ts = ifft_ua_ts.astype('float64')
ifft_ua_ts_len = (len(ifft_ua_ts) / ofac) + np.mod(len(ifft_ua_ts), ofac)
ifft_time = time[-ifft_ua_ts_len:]
ifft_ua_ts = ifft_ua_ts[-len(waveform):]
ifft_time = ifft_time - ifft_time[0]
a_periods, a_mag, corr_a_mag, a_fr, a_fi, a_red_periods, a_red_mag, a_gredth, a_fac95, a_fac99, a_fac99_9, a_faccrit, a_fac_grid, a_sig_levels, a_tau, a_corr = redfit.red_background(
nsim, mctest, ifft_time, ifft_ua_ts, ofac)
#apply lsp correction from altered spectrum to unaltered spectrum
corr_ua_mag = ua_mag / a_corr
#check confidence of each point on spectrum
sigs = np.zeros(len(corr_ua_mag))
last_ind = len(a_sig_levels) - 1
for i in range(len(a_sig_levels) - 1):
conf_low = a_gredth * a_fac_grid[i]
conf_up = a_gredth * a_fac_grid[i + 1]
current_last_ind = i + 1
for j in range(len(corr_ua_mag)):
if sigs[j] == 0:
if (corr_ua_mag[j] >= conf_low[j]) and (corr_ua_mag[j] <
conf_up[j]):
sigs[j] = a_sig_levels[i]
elif current_last_ind == last_ind:
if corr_ua_mag[j] > conf_up[j]:
sigs[j] = a_sig_levels[i + 1]
#get critical significance for all points on spectrum
crit_sig = a_gredth * a_faccrit
#get 95,99 and 99.9 % chi squared significance bands for all points on spectrum
sig_95 = a_gredth * a_fac95
sig_99 = a_gredth * a_fac99
sig_99_9 = a_gredth * a_fac99_9
return (x, sigs, sig_95, sig_99, sig_99_9, crit_sig, a_gredth, corr_ua_mag,
ua_periods, a_tau)
def run_LSP(mod_data,x):
lat_i = lat_indices[x]
lon_i = lon_indices[x]
print lat_i,lon_i
current_lat = lat_c[lat_i]
current_lon = lon_c[lon_i]
waveform = mod_data
waveform_ave = np.average(waveform)
model_date_val = np.copy(model_date)
model_time_val = np.copy(model_time)
time = modules.date_process(model_date_val,model_time_val,start_year)
if (species.lower() != 'gmao_temp') and (species.lower() != 'gmao_psfc') and (species.lower() != 'wind_speed') and (species.lower() != 'wind_direction'):
waveform = waveform*1e9
#check model vals are valid
#valid = vals >= 0
#vals = vals[valid]
#model_time_val = model_time[valid]
#model_date_val = model_date[valid]
#take 8 hour average
divisor = 8
total_len = len(waveform)/divisor
start = 0
end = divisor
ave_waveform = []
ave_time = []
for i in range(total_len):
ave = np.ma.average(waveform[start:end])
ave_time=np.append(ave_time,time[start])
ave_waveform=np.append(ave_waveform,ave)
start+=divisor
end+=divisor
time=np.copy(ave_time)
waveform=np.copy(ave_waveform)
#take lsp unwindowed of waveform
ua_periods,ua_mag,ua_ph,ua_fr,ua_fi = modules.take_lomb_unwindowed(time,waveform,ofac,1./24)
#take out known periodic components 1,182.625, and 365.25 a priori for more accurate red noise fit.
closest_daily_index = min(range(len(ua_periods)), key=lambda i: abs(ua_periods[i]-1.))
closest_ha_index = min(range(len(ua_periods)), key=lambda i: abs(ua_periods[i]-182.625))
closest_annual_index = min(range(len(ua_periods)), key=lambda i: abs(ua_periods[i]-365.25))
rm_indices = [closest_daily_index,closest_ha_index,closest_annual_index]
ua_mag_c,ua_fr,ua_fi = redfit.sidelobe_percent_remove(np.copy(ua_mag),ua_fr,ua_fi,rm_indices,5.,ua_periods)
#-------------------------------------------------------------------------------
#Do IFFT of altered spectra - with significant periods removed and gaps left in real and imag components linearly interpolated.
#altered spectra provides red noise estimation baseline
##use ifft to get time series back from adjusted spectra
#complex Fourier spectrum which corresponds to the Lomb-Scargle periodogram:
F = [0]*((len(ua_fr)*2)+1)
#set first real value to average
F[0] = complex(waveform_ave*len(waveform),0)
#Get reverse real and imaginary values
rev_ua_fr=np.copy(ua_fr[::-1])
rev_ua_fi=np.copy(ua_fi[::-1])
rev_ua_fr[0] = 0
rev_ua_fi[0] = 0
f_index = 1
#Fill Fourier Spectrum real and imaginary values
for i in range(len(ua_fr)):
F[f_index] = complex(ua_fr[i],ua_fi[i])
f_index+=1
for i in range(len(ua_fr)):
F[f_index] = complex(rev_ua_fr[i],-rev_ua_fi[i])
f_index+=1
F = np.array(F)
#Take ifft and just take real values
ifft_ua_ts = numpy.fft.ifft(F)
ifft_ua_ts = ifft_ua_ts.astype('float64')
ifft_ua_ts_len = (len(ifft_ua_ts)/ofac) + np.mod(len(ifft_ua_ts),ofac)
ifft_time = time[-ifft_ua_ts_len:]
ifft_ua_ts = ifft_ua_ts[-len(waveform):]
ifft_time = ifft_time-ifft_time[0]
a_periods,a_mag,corr_a_mag,a_fr,a_fi,a_red_periods,a_red_mag,a_gredth,a_fac95,a_fac99,a_fac99_9,a_faccrit,a_fac_grid,a_sig_levels,a_tau,a_corr = redfit.red_background(nsim,mctest,ifft_time,ifft_ua_ts,ofac)
#apply lsp correction from altered spectrum to unaltered spectrum
corr_ua_mag = ua_mag/a_corr
#check confidence of each point on spectrum
sigs = np.zeros(len(corr_ua_mag))
last_ind = len(a_sig_levels)-1
for i in range(len(a_sig_levels)-1):
conf_low = a_gredth*a_fac_grid[i]
conf_up = a_gredth*a_fac_grid[i+1]
current_last_ind = i+1
for j in range(len(corr_ua_mag)):
if sigs[j] == 0:
if (corr_ua_mag[j] >= conf_low[j]) and (corr_ua_mag[j] < conf_up[j]):
sigs[j] = a_sig_levels[i]
elif current_last_ind == last_ind:
if corr_ua_mag[j] > conf_up[j]:
sigs[j] = a_sig_levels[i+1]
#get critical significance for all points on spectrum
crit_sig = a_gredth*a_faccrit
#get 95,99 and 99.9 % chi squared significance bands for all points on spectrum
sig_95 = a_gredth*a_fac95
sig_99 = a_gredth*a_fac99
sig_99_9 = a_gredth*a_fac99_9
return (x,sigs,sig_95,sig_99,sig_99_9,crit_sig,a_gredth,corr_ua_mag,ua_periods,a_tau)
def red_background(nsim,mctest,t,x,ofac,all_t,all_x ):
#average dt of entire time series
diffs = [t[i+1]-t[i] for i in range(len(t)-1)]
avgdt = np.average(diffs)
ave = np.mean(x)
#subtract mean from data
x = x - ave
#GET TAU
tau, rhoavg, ierr = gettau(t,x,1,avgdt)
nout = int(0.5*int(ofac)*1*len(t))
if tau == 'invalid':
model_periods,model_mag,corr_model_mag,model_fr,model_fi,red_periods,red_mag_avg,gredth,fac95,fac99,fac99_9,faccrit,fac_grid,sig_levels,tau,corr = np.zeros(nout),np.zeros(nout),np.zeros(nout),np.zeros(nout),np.zeros(nout),np.zeros(nout),np.zeros(nout),np.zeros(nout),0,0,0,0,np.zeros(nout),np.zeros(nout),0,1
return model_periods,model_mag,corr_model_mag,model_fr,model_fi,red_periods,red_mag_avg,gredth,fac95,fac99,fac99_9,faccrit,fac_grid,sig_levels,tau,corr
#make sure that tau is non-negative
if (tau < 0.0):
print 'Negative tau is forced to zero.'
tau = 0.0
x = x + ave
#determine lag-1 autocorrelation coefficient
rho = np.exp(-avgdt / tau) # avg. autocorrelation coefficient
rhosq = rho * rho
#t = np.copy(all_t)
#x = np.copy(all_x)
xdif = np.max(t)-np.min(t)
#Calculate model spectrum
model_periods,model_mag,model_ph,model_fr,model_fi = modules.take_lomb_unwindowed(t,x,ofac,avgdt)
fft_periods_all,fft_mag_all,fft_fr,fft_fi,fft_array = modules.take_fft_unwindowed(t,x,avgdt)
fft_freq_all = 1./fft_periods_all
full_n = len(model_periods)
model_freq = 1./model_periods
# estimate data variance from data spectrum
# ----------------------------------------
varx = (model_freq[0]) * np.sum(model_mag) # NB: freq[1] = df
varx_fft = (fft_freq_all[0]) * np.sum(fft_mag_all)
red_mag = np.zeros((nsim,len(model_periods)))
red_mag_sum = np.zeros(len(model_periods))
fft_mag = np.zeros((nsim,len(fft_periods_all)))
fft_mag_sum = np.zeros(len(fft_periods_all))
#create AR1 spectrum nsim times
for i in range(nsim):
print 'Nsim = ', i+1
red = makear1(t,len(x),tau)
if mctest == True:
#red_periods,red_mag[i,:],red_ph,red_fr,red_fi = modules.take_lomb_unwindowed(t,red,ofac,avgdt)
#red_freq = 1./red_periods
fft_periods,fft_mag[i,:],fft_fr,fft_fi,fft_array = modules.take_fft_unwindowed(t,red,avgdt)
fft_freq = 1./fft_periods
else:
#red_periods,red_mag[0,:],red_ph,red_fr,red_fi = modules.take_lomb_unwindowed(t,red,ofac,avgdt)
#red_freq = 1./red_periods
fft_periods,fft_mag[0,:],fft_fr,fft_fi,fft_array = modules.take_fft_unwindowed(t,red,avgdt)
fft_freq = 1./fft_periods
#plt.loglog(red_periods,red_mag[0,:],color='black')
#plt.loglog(fft_periods,fft_mag[0,:],color='red')
#plt.show()
#red_periods = red_periods[cp:]
#red_mag = red_mag[0,cp:]
#scale and sum red-noise spectra
#-------------------------------
if mctest == True:
#varr = (red_freq[0]) * np.sum(red_mag[i,:]) # NB: freq[1] = df
#fac = varx / varr
#red_mag[i,:] = fac * red_mag[i,:]
#red_mag_sum = red_mag_sum + red_mag[i,:]
varr = (fft_freq[0]) * np.sum(fft_mag[i,:]) # NB: freq[1] = df
fac = varx_fft / varr
fft_mag[i,:] = fac * fft_mag[i,:]
fft_mag_sum = fft_mag_sum + fft_mag[i,:]
else:
#varr = (red_freq[0]) * np.sum(red_mag[0,:]) # NB: freq[1] = df
#fac = varx / varr
#red_mag_sum = red_mag_sum + fac * red_mag[0,:]
varr = (fft_freq[0]) * np.sum(fft_mag[0,:]) # NB: freq[1] = df
fac = varx_fft / varr
fft_mag_sum = fft_mag_sum + fac * fft_mag[0,:]
#determine average red-noise spectrum; scale average again to
#make sure that roundoff errors do not affect the scaling
#------------------------------------------------------------
#red_mag_avg = red_mag_sum / float(nsim)
#varr = (red_freq[0]) * np.sum(red_mag_avg)
#fac = varx / varr
#red_mag_avg = fac * red_mag_avg
fft_mag_avg = fft_mag_sum / float(nsim)
varr = (fft_freq[0]) * np.sum(fft_mag_avg)
fac = varx_fft / varr
fft_mag_avg = fac * fft_mag_avg
# set theoretical spectrum (e.g., Mann and Lees, 1996, Eq. 4)
# make area equal to that of the input time series
# -----------------------------------------------------------
#print red_freq[-1]
#fnyq = red_freq[-1]
#gredth = (1.0-rhosq) / (1.0-2.0*rho*np.cos(np.pi*red_freq/fnyq)+rhosq)
#varr = red_freq[0] * np.sum(gredth)
#fac = varx / varr
#gredth = fac * gredth
print fft_freq[-1]
fnyq = fft_freq[-1]
gredth_fft = (1.0-rhosq) / (1.0-2.0*rho*np.cos(np.pi*fft_freq/fnyq)+rhosq)
varr = fft_freq[0] * np.sum(gredth_fft)
fac = varx_fft / varr
gredth_fft = fac * gredth_fft
#ratio = float(len(act_periods))/float(len(model_periods))
#print 'ratio = ', ratio
#model_mag = model_mag/ratio
#red_mag_avg = red_mag_avg/ratio
#gredth = gredth/ratio
#determine correction factor
#---------------------------
#corr = red_mag_avg / gredth
#correct for bias in autospectrum
#--------------------------------
#corr_model_mag = model_mag / corr
#corr_model_mag = model_mag
#gredth = gredth*corr
#print 'model freq 0 = ', model_freq[1]
print 'varx = ', varx
print 'ofac = ', ofac
print 'avgdt = ', avgdt
print 'Avg. autocorr. coeff., rho = ', rho
print 'Avg. tau = ', tau
npoints = len(t) # of data points
nseg = int(2 * npoints / (1 + 1)) # points per segment
#avgdt = (t[-1] - t[0]) / (npoints-1.0) # average sampling interval
tp = avgdt * nseg # average period of a segment
#df = 1.0 / (ofac * tp) # freq. spacing
df = model_freq[0]
wz = 2.0 * np.pi * df # omega = 2*pi*f
fnyq = 1.0 * 1.0 / (2.0 * avgdt) # average Nyquist freq.
nfreq = fnyq / df + 1 # f(1) = f0; f(nfreq) = fNyq
lfreq = nfreq * 2
nout = nfreq
scal = 2.0 / (1.0 * nseg * df * ofac)
print 'df = ', df
print 'Nseg = ', nseg
print 'Scal = ', scal
print model_freq[0:20]
print model_freq[-10:]
#fft_mag_all = fft_mag_all*scal
#fft_mag_all = fft_mag_all/4.
#winbw = (model_freq[1]-model_freq[0]) * ofac * 1.21
#model_mag = model_mag*len(t)
#model_mag = 20*np.log10(model_mag)
#gredth = 20*np.log10(gredth)
#plt.loglog(red_periods,red_mag_avg)
#plt.loglog(fft_periods,fft_mag_avg)
#plt.loglog(fft_periods_all,fft_mag_avg)
#plt.loglog(fft_periods,gredth_fft)
#plt.plot(model_freq,model_mag)
#plt.plot(red_freq,red_mag_avg)
plt.loglog(fft_periods_all,fft_mag_all)
plt.loglog(fft_periods,fft_mag_avg)
#plt.plot(red_freq,gredth)
#plt.loglog(fft_periods,gredth_fft)
#plt.plot(model_freq,corr_model_mag)
#plt.show()
corr = [1.]*len(fft_freq)
fac95,fac99,fac99_9,faccrit,fac_grid,sig_levels,ci99 = significance_tests(x,fft_periods,fft_mag,nsim,1,corr,mctest)
plt.loglog(fft_periods,fft_mag_avg*fac95)
plt.loglog(fft_periods,fft_mag_avg*fac99)
plt.loglog(fft_periods,fft_mag_avg*faccrit)
plt.show()
return model_periods,model_mag,corr_model_mag,model_fr,model_fi,red_periods,red_mag_avg,gredth,fac95,fac99,fac99_9,faccrit,fac_grid,sig_levels,tau,corr
