def rotatingTHR(filelist,
varin,
kernel,
outputdir,
oroNV=None,
selector=None,
high_terrain=600,
verbose=True):
'''Compute time filtering on data in different files.
Args:
filelist (list): list of abs paths to data files. User is responsible to
make sure files in list have correct chronological order.
Note that time axis in data files should be the 1st axis.
varin (str): variable id in files.
kernel (list or tuple): list/tuple of integers specifying the shape of
the kernel/structuring element used in the gray erosion process.
selector (utils.funcs.Selector): selector obj to select subset of data.
outputdir (str): path to folder to save outputs.
Keyword Args:
oroNV (NCVAR): 2D array, surface orographic data in meters.
This additional surface height info is used to perform a separate
reconstruction computation for areas with high elevations, and
the results can be used to enhance the continent-penetration
ability of landfalling ARs. Sensitivity in landfalling ARs is
enhanced, other areas are not affected. Needs to have compatible
shape as <ivt>.
If None, omit this process and treat areas with different heights
all equally.
New in v2.0.
high_terrain (float): minimum orographic height to define as high
terrain area, within which a separate reconstruction is performed.
Only used if <oroNV> is not None.
New in v2.0.
verbose (bool): print some messages or not.
Designed to perform temporal filtering on data that are too large to fit
into memory, e.g. high-resolution data across multiple decades.
Function will read in 2 files at once, call the filtering function on the
concatenated data, and shift 1 step to the next 2 files. If at the begining,
pad 0s to the left end. If in the mid, pad filtered data in the mid of
the concatenated data in the previous step. If at the end, pad 0s to
the right end.
The filtering function <func> is assumed to apply a filtering window with
odd length n, and truncates (n-1)/2 points from both ends. If the function
doesn't truncate data, will raise an exception.
'''
#----------------Check input files----------------
funcs.checkFiles(filelist)
for ii, fii in enumerate(filelist[:-1]):
if ii == 0:
var1 = funcs.readNC(fii, varin)
var1 = var1(selector)
else:
var1 = var2
del var2
fii2 = filelist[ii + 1]
var2 = funcs.readNC(fii2, varin)
var2 = var2(selector)
timeidx = funcs.interpretAxis('time', var1)
if timeidx != 0:
raise Exception("Time axis in variable is not at axis 0.")
timeidx = funcs.interpretAxis('time', var2)
if timeidx != 0:
raise Exception("Time axis in variable is not at axis 0.")
n1 = var1.shape[0]
vartmp = funcs.cat(var1, var2, axis=0)
vartmp, vartmp_rec, vartmp_ano = THR(vartmp,
kernel,
oroNV=oroNV,
high_terrain=high_terrain)
# crop end points
dt = kernel[0]
vartmp_rec = vartmp_rec.sliceIndex(dt, -dt)
vartmp_ano = vartmp_ano.sliceIndex(dt, -dt)
if dt <= 0:
raise Exception("dt<=0 not supported yet")
if verbose:
print('\n# <rotatingTHR>: Concatenated var shape:', vartmp.shape)
print('# <rotatingTHR>: Filtered var shape:', vartmp_rec.shape)
print('# <rotatingTHR>: Length difference:', dt)
if ii == 0:
#----------------------Pad 0s----------------------
left_rec = var1.sliceIndex(0, dt)
left_rec.data = left_rec.data * 0
left_rec.data.mask = True
left_ano = var1.sliceIndex(0, dt)
left_ano.data = left_ano.data * 0
left_ano.data.mask = True
else:
left_rec = mid_left_rec
left_ano = mid_left_ano
rec_pad = funcs.cat(left_rec, vartmp_rec, axis=0)
rec1 = rec_pad.sliceIndex(0, n1)
ano_pad = funcs.cat(left_ano, vartmp_ano, axis=0)
ano1 = ano_pad.sliceIndex(0, n1)
var1 = vartmp.sliceIndex(0, n1)
if verbose:
print('\n# <rotatingTHR>: Shape of left section after padding:',
rec1.shape)
rec1.id = vartmp_rec.id
attdict = getAttrDict(var1, 'reconstruction')
attdict['name'] = 'ivt_rec'
rec1.attributes = attdict
ano1.id = vartmp_ano.id
attdict = getAttrDict(var1, 'anomaly')
attdict['name'] = 'ivt_ano'
ano1.attributes = attdict
# left to pad in next iteration
mid_left_rec = vartmp_rec.sliceIndex(n1 - dt, n1)
mid_left_ano = vartmp_ano.sliceIndex(n1 - dt, n1)
#-----------------------Save-----------------------
fname = os.path.split(fii)[1]
file_out_name='%s-THR-kernel-t%d-s%d.nc'\
%(os.path.splitext(fname)[0], kernel[0], kernel[1])
abpath_out = os.path.join(outputdir, file_out_name)
print('\n### <testrotatingfilter>: Saving output to:\n', abpath_out)
funcs.saveNC(abpath_out, var1, 'w')
funcs.saveNC(abpath_out, rec1, 'a')
funcs.saveNC(abpath_out, ano1, 'a')
# save the right section for the last file
if ii == len(filelist) - 2:
right = var2.sliceIndex(-dt, None)
right.data = right.data * 0
right.data.mask = True
rec2 = rec_pad.sliceIndex(n1, None)
rec2 = funcs.cat(rec2, right, axis=0)
ano2 = ano_pad.sliceIndex(n1, None)
ano2 = funcs.cat(ano2, right, axis=0)
var2 = vartmp.sliceIndex(n1, None)
if verbose:
print(
'\n# <rotatingTHR>: Shape of last section after padding:',
ano2.shape)
rec2.id = vartmp_rec.id
attdict = getAttrDict(var2, 'reconstruction')
attdict['name'] = 'ivt_rec'
rec2.attributes = attdict
ano2.id = vartmp_ano.id
attdict = getAttrDict(var2, 'anomaly')
attdict['name'] = 'ivt_ano'
ano2.attributes = attdict
#-----------------------Save-----------------------
fname = os.path.split(fii2)[1]
file_out_name='%s-THR-kernel-t%d-s%d.nc'\
%(os.path.splitext(fname)[0], kernel[0], kernel[1])
abpath_out = os.path.join(outputdir, file_out_name)
print('\n### <testrotatingfilter>: Saving output to:\n',
abpath_out)
funcs.saveNC(abpath_out, var2, 'w')
funcs.saveNC(abpath_out, rec2, 'a')
funcs.saveNC(abpath_out, ano2, 'a')
return
os.makedirs(OUTPUTDIR)
#-----------Read in data----------------------
print('\n### <compute_thr_singlefile>: Read in file:\n', IVT_FILE)
var = funcs.readNC(IVT_FILE, VARIN)
var = var.sliceIndex(0, 200, axis=0)
#--------------------Read in orographic data--------------------
oroNV = None
#-----------------Shift longitude-----------------
var = var.sliceData(LAT1, LAT2, 2)
var = var.shiftLon(SHIFT_LON)
#----------------------Do THR----------------------
ivt, ivtrec, ivtano = thr.THR(var,
KERNEL,
oroNV=oroNV,
high_terrain=HIGH_TERRAIN)
#--------Save------------------------------------
fname = os.path.split(IVT_FILE)[1]
file_out_name='%s-THR-kernel-t%d-s%d.nc'\
%(os.path.splitext(fname)[0], KERNEL[0], KERNEL[1])
abpath_out = os.path.join(OUTPUTDIR, file_out_name)
print('\n### <testrotatingfilter>: Saving output to:\n', abpath_out)
funcs.saveNC(abpath_out, ivt, 'w')
funcs.saveNC(abpath_out, ivtrec, 'a')
funcs.saveNC(abpath_out, ivtano, 'a')
#######################################################################
# Save all records #
#######################################################################
if not os.path.exists(OUTPUTDIR):
os.makedirs(OUTPUTDIR)
if PLOT:
plot_dir = os.path.join(OUTPUTDIR, 'plots')
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
#-----------------Save label file-----------------
abpath_out = os.path.join(OUTPUTDIR, LABEL_FILE_OUT_NAME)
print('\n### <detect_ARs>: Saving output to:\n', abpath_out)
funcs.saveNC(abpath_out, labels, 'w')
funcs.saveNC(abpath_out, angles, 'a')
funcs.saveNC(abpath_out, crossfluxes, 'a')
#-----------------Save record file-----------------
abpath_out = os.path.join(OUTPUTDIR, RECORD_FILE_OUT_NAME)
print('\n### <detect_ARs>: Saving output to:\n', abpath_out)
# Necessary: to remove ... in csv file
if sys.version_info.major == 2:
np.set_printoptions(threshold=np.inf)
elif sys.version_info.major == 3:
np.set_printoptions(threshold=sys.maxsize)
result_df.to_csv(abpath_out, index=False)
#-------------------Create plots and save------------------------
if PLOT:
ufluxNV = funcs.readNC(UFLUX_FILE, UFLUX_VARID)
vfluxNV = funcs.readNC(VFLUX_FILE, VFLUX_VARID)
ivtdata = np.ma.sqrt(ufluxNV.data**2 + vfluxNV.data**2)
ivtNV = funcs.NCVAR(
ivtdata, 'ivt', ufluxNV.axislist, {
'name': 'ivt',
'long_name': 'integrated vapor transport (IVT)',
'standard_name': 'integrated vapor transport (IVT)',
'title': 'integrated vapor transport (IVT)',
'units': getattr(ufluxNV, 'units', '')
})
#--------Save------------------------------------
print('\n### <compute_ivt>: Saving output to:\n', OUTPUTFILE)
funcs.saveNC(OUTPUTFILE, ivtNV, 'w')
#-------------------Plot------------------------
import matplotlib.pyplot as plt
figure = plt.figure(figsize=(7, 10), dpi=100)
idx = 40
time_str = str(ufluxNV.getTime()[idx])
plot_vars = [ufluxNV.data[idx], vfluxNV.data[idx], ivtNV.data[idx]]
titles = ['U', 'V', 'IVT']
for ii, vii in enumerate(plot_vars):
axii = figure.add_subplot(3, 1, ii + 1)
iso = plot.Isofill(vii, 10, 1, 2)
plot.plot2(vii,
iso,