log_window_size.append(f_sral)
ssha_m = astro_conv(ssha, np.ones((window_size))/float(window_size), boundary='extend',
nan_treatment='interpolate', preserve_nan=True)
# INSERT ssha with MASK in the dictionary data_matrix
# Basically, the ~idx AFTER the smoothing represents NaNs, because
# ~idx positions have been replaced by NaNs
data_matrix['SSHA_'+ str(window_size)] = np.ma.array(ssha_m, mask=~idx)
print 'SSHA: OK'
# ================================ OLCI
fullpath = os.path.join(os.path.join(paths['OLCI'], f_olci), fname_olci)
# Read netcdf
try:
lonol, latol, varValues, flagol = ncman.olci1D_read_nc(fullpath, lst_olci)
except:
bad_olci.append(f_olci)
continue
# transform coordinates
xol, yol = s3ct.slstr_olci_coordtran(lonol.data, latol.data, inEPSG, outEPSG)
del lonol, latol
# subset dataset
varValues = ncman.slstr_olci_subset_nc(xol, yol, varValues, bound)
# Apply flags/masks
# Extract bits of the wqsf flag
flag_out = S3postproc.extract_bits(flagol.data, 64)
# Clear
del flagol
# Extract dictionary with flag meanings and values
def my_fun():
filter_method = {
'MEDIAN': False,
'AVERAGE': False,
'BUTTER': False,
'ASTROPY': True
}
# Find common dates
# paths = {'SRAL': r'D:\vlachos\Documents\KV MSc thesis\Data\Satellite\Actual_data\SRAL'.replace('\\', '\\'),
# 'OLCI': r'D:\vlachos\Documents\KV MSc thesis\Data\Satellite\Actual_data\OLCI'.replace('\\', '\\'),
# 'SLSTR': r'D:\vlachos\Documents\KV MSc thesis\Data\Satellite\Actual_data\SLSTR'.replace('\\','\\')
# }
# Gulf Stream Test
paths = {
'SRAL':
r'D:\vlachos\Documents\KV MSc thesis\Data\Satellite\Gulf Stream_1\SRAL'
.replace('\\', '\\'),
'OLCI':
r'D:\vlachos\Documents\KV MSc thesis\Data\Satellite\Gulf Stream_1\OLCI'
.replace('\\', '\\'),
'SLSTR':
r'D:\vlachos\Documents\KV MSc thesis\Data\Satellite\Gulf Stream_1\SLSTR'
.replace('\\', '\\')
}
# Folder names with the common dates
common_date = s3utilities.find_common_dates(paths)
# Define constants
inEPSG = '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs '
outEPSG = '+proj=utm +zone=23 +ellps=GRS80 +datum=NAD83 +units=m +no_defs ' # Gulf Stream 1
# outEPSG = '+proj=laea +lat_0=52 +lon_0=10 +x_0=4321000 +y_0=3210000 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs ' # North Sea
# bound = [3500000, 4300000, 3100000, 4000000] # North Sea
bound = [-3000000, -1000000, 3625000, 4875000] # Gulf Stream
fname_sral = 'sub_enhanced_measurement.nc'
lst_sral = ['ssha_20_ku', 'flags']
fname_olci = 'sub_OLCI.nc'
lst_olci = ['ADG443_NN', 'WQSF']
bad_sral = []
bad_olci = []
log_window_size = []
log_window_size2 = []
counter = 0
n = len(paths['SRAL']) * len(paths['OLCI'])
# Plot common dates
for f_sral in common_date['SRAL']:
for f_olci in common_date['OLCI']:
if f_olci[16:24] == f_sral[16:24]:
# if (dt.datetime.strptime(f_sral[16:31], '%Y%m%dT%H%M%S') < dt.datetime(2018, 6, 28)) or (dt.datetime.strptime(f_sral[16:31], '%Y%m%dT%H%M%S') > dt.datetime(2018, 8, 3)):
# pass
# else:
# n = n - 1
# continue
# === Progress
sys.stdout.write("\rProgress... {0:.2f}%".format(
(float(counter) / n) * 100))
sys.stdout.flush()
#================================= SRAL
fullpath = os.path.join(os.path.join(paths['SRAL'], f_sral),
fname_sral)
# Read netcdf
try:
lonsr, latsr, ssha, flagsr = ncman.sral_read_nc(
fullpath, lst_sral)
except:
bad_sral.append(f_sral)
continue
# transform coordinates
xsr, ysr = s3ct.sral_coordtran(lonsr, latsr, inEPSG, outEPSG)
# subset dataset
xsr, ysr, ssha, flagsr = ncman.sral_subset_nc(
xsr, ysr, ssha, flagsr, bound)
ssha = ssha['ssha_20_ku']
# Apply flags/masks
ssha_m, outmask = S3postproc.apply_masks_sral(
ssha, 'ssha_20_ku', flagsr)
# Apply outmask
xsr = xsr[outmask]
ysr = ysr[outmask]
# Clear workspace
del lonsr, latsr, flagsr, outmask, ssha
fdate_sral = dt.datetime.strptime(f_sral[16:31],
'%Y%m%dT%H%M%S')
fdate_sral = fdate_sral.strftime('%Y-%m-%d %H_%M_%S')
#================================ OLCI
fullpath = os.path.join(os.path.join(paths['OLCI'], f_olci),
fname_olci)
# Read netcdf
try:
lonol, latol, varValues, flagol = ncman.olci1D_read_nc(
fullpath, lst_olci)
except:
bad_olci.append(f_olci)
continue
# transform coordinates
xol, yol = s3ct.slstr_olci_coordtran(lonol.data, latol.data,
inEPSG, outEPSG)
del lonol, latol
# subset dataset
varValues = ncman.slstr_olci_subset_nc(xol, yol, varValues,
bound)
# Apply flags/masks
# Extract bits of the wqsf flag
flag_out = S3postproc.extract_bits(flagol.data, 64)
# Clear
del flagol
# Extract dictionary with flag meanings and values
bitval = S3postproc.extract_maskmeanings(fullpath)
# Create masks
masks = S3postproc.extract_mask(bitval, flag_out, 64)
# clean
del flag_out, bitval
# Apply masks to given variables
# Define variables separately
chl_oc4me = varValues['ADG443_NN']
del varValues
chl_oc4me, outmasks = S3postproc.apply_masks_olci(
chl_oc4me, 'ADG443_NN', masks)
# Clean
del masks
# Apply flag masks
xol = xol[outmasks]
yol = yol[outmasks]
del outmasks
# Apply chl_nn mask
xol = xol[chl_oc4me.mask]
yol = yol[chl_oc4me.mask]
chl_oc4me = chl_oc4me.data[chl_oc4me.mask]
if check_npempty(chl_oc4me):
continue
fdate_olci = dt.datetime.strptime(f_olci[16:31],
'%Y%m%dT%H%M%S')
fdate_olci = fdate_olci.strftime('%Y-%m-%d %H_%M_%S')
dicinp = {
'plttitle':
'SRAL ' + f_sral[:3] + ' ' + fdate_sral + '\n' + 'OLCI ' +
f_olci[:3] + ' ' + fdate_olci,
'filename':
fdate_sral + '__' + fdate_olci
}
# Interpolate IDW
olci_interp = S3postproc.ckdnn_traject_idw(
xsr, ysr, xol, yol, chl_oc4me, {
'k': 12,
'distance_upper_bound': 330 * np.sqrt(2)
})
# Check if empty
if check_npempty(olci_interp):
continue
# Low pass moving average filter
olci_movAvlow = S3postproc.twoDirregularFilter(
xsr, ysr, olci_interp, xol, yol, chl_oc4me, {'r': 50000})
# "Trend" moving average filter
olci_movAv = S3postproc.twoDirregularFilter(
xsr, ysr, olci_interp, xol, yol, chl_oc4me, {'r': 150000})
# Spatial detrend (sst_est = sst - sst_movAv)
olci_est = olci_movAvlow - olci_movAv
# If interpolation fails for ALL points, then go to next date
if np.all(np.isnan(olci_est)) == True:
continue
else:
pass
# Choose inside percentiles
idx = (ssha_m > np.percentile(
ssha_m, 1)) & (ssha_m < np.percentile(ssha_m, 99))
# Keep ssha_m
ssha_m_keep = np.ones_like(ssha_m) * ssha_m
# Compute distances between SRAL points
# dst_sr = S3postproc.sral_dist(xsr[idx], ysr[idx])
dst_sr = S3postproc.sral_dist(xsr, ysr)
dst_ol = S3postproc.sral_dist(xsr, ysr)
# # ================ Insert NaNs ===============
# Choose filtering method
if filter_method['ASTROPY'] == True:
ssha_m[~idx] = np.nan
window_size = 303
# Check window size
if ssha_m.size < window_size:
window_size = ssha_m.size
# Check if window size is odd or even (needs to be odd)
if window_size % 2 == 0:
window_size = window_size + 1
# Log which files do not use the default window size
log_window_size.append(f_sral)
ssha_m = astro_conv(ssha_m,
np.ones((window_size)) /
float(window_size),
boundary='extend',
nan_treatment='interpolate',
preserve_nan=True)
# ====== 2nd filter (larger window size)
ssha_m_keep[~idx] = np.nan
window_size2 = 901
# Check window size
if ssha_m_keep.size < window_size2:
window_size2 = ssha_m_keep.size
# Check if window size is odd or even (needs to be odd)
if window_size2 % 2 == 0:
window_size2 = window_size2 + 1
# Log which files do not use the default window size
log_window_size2.append(f_sral)
ssha_m_keep = astro_conv(ssha_m_keep,
np.ones((window_size2)) /
float(window_size2),
boundary='extend',
nan_treatment='interpolate',
preserve_nan=True)
# Subtract large trend
ssha_m = ssha_m - ssha_m_keep
# Choosle filtering method
if filter_method['MEDIAN'] == True:
# Insert nans
dst_sr_nan, ssha_m_nan, idx_nan = S3postproc.sral_dist_nans(
dst_sr, ssha_m, threshold_sr)
# Filter SSHA
ssha_m_nan = scsign.medfilt(ssha_m_nan, 31)
elif filter_method['AVERAGE'] == True:
# Moving Average filter SSHA
ssha_m = np.convolve(ssha_m,
np.ones((29)) / ssha_m.size,
mode='same')
sst_est = np.convolve(olci_est,
np.ones((29)) / olci_est.size,
mode='same')
# Insert nans
dst_sr_nan, ssha_m_nan, idx_nan = S3postproc.sral_dist_nans(
dst_sr, ssha_m, threshold_sr)
elif filter_method['BUTTER'] == True:
def butter_lowpass(cutoff, fs, order=5):
nyq = 0.5 * fs
normal_cutoff = cutoff / nyq
b, a = butter(order,
normal_cutoff,
btype='low',
analog=False)
return b, a
def butter_lowpass_filter(data, cutoff, fs, order=5):
b, a = butter_lowpass(cutoff, fs, order=order)
y = lfilter(b, a, data)
return y
# Filter requirements.
order = 3
fs = 8 # sample rate, Hz
cutoff = 0.25 #2.667 # desired cutoff frequency of the filter, Hz
# Get the filter coefficients so we can check its frequency response.
b, a = butter_lowpass(cutoff, fs, order)
# Plot the frequency response.
# w, h = freqz(b, a, worN=8000)
# plt.subplot(2, 1, 1)
# plt.plot(0.5*fs*w/np.pi, np.abs(h), 'b')
# plt.plot(cutoff, 0.5*np.sqrt(2), 'ko')
# plt.axvline(cutoff, color='k')
# plt.xlim(0, 0.5*fs)
# plt.title("Lowpass Filter Frequency Response")
# plt.xlabel('Frequency [Hz]')
# plt.grid()
# Filter the data, and plot both the original and filtered signals.
ssha_m = butter_lowpass_filter(ssha_m, cutoff, fs, order)
# Insert nans
# dst_sr_nan, ssha_m_nan, idx_nan = S3postproc.sral_dist_nans(dst_sr, ssha_m, threshold_sr)
# dst_sl_nan, sst_est_nan, idx_sl_nan = S3postproc.sral_dist_nans(dst_sl, sst_est, threshold_sl)
# Normalize variables between upper and lower bounds
ub = 1 # upper bound
lb = -1 # lower bound
# Rescale
ssha_m_nan = S3postproc.rescale_between(ssha_m, ub, lb)
olci_est = S3postproc.rescale_between(olci_est, ub, lb)
# Variables in dictionary
variables = {'SRAL': ssha_m_nan, 'SLSTR': [], 'OLCI': olci_est}
distance = {'SRAL': dst_sr, 'SLSTR': [], 'OLCI': dst_ol}
plotpath = r'D:\vlachos\Documents\KV MSc thesis\Data\Satellite\Outputs\Gulf_Stream_1\SRAL_OLCI\Trajectories'.replace(
'\\', '\\') # Gulf stream
# plotpath = r'D:\vlachos\Documents\KV MSc thesis\Data\Satellite\Outputs\North_Sea\SRAL_OLCI\Trajectories'.replace('\\','\\') # North Sea
# Plot
S3plots.sral_cross_sections_olci(variables, distance, dicinp,
plotpath)
counter = counter + 1
del olci_est, olci_interp, olci_movAv, olci_movAvlow, ssha_m, ssha_m_nan, ssha_m_keep,
return bad_sral, bad_olci