def setUp(self):
thisdir = os.path.dirname(__file__)
fixture_dir = os.path.join(thisdir, 'fixtures')
abpath_in = os.path.join(fixture_dir, 'uflux_vflux_ivt_test.nc')
uflux = funcs.readNC(abpath_in, 'uflux')
vflux = funcs.readNC(abpath_in, 'vflux')
ivt = funcs.readNC(abpath_in, 'ivt')
self.uflux = uflux
self.vflux = vflux
self.ivt = ivt
def setUp(self):
thisdir=os.path.dirname(__file__)
fixture_dir=os.path.join(thisdir, 'fixtures')
abpath_in=os.path.join(fixture_dir,'uflux_vflux_ivt_test.nc')
uflux=funcs.readNC(abpath_in, 'uflux')
vflux=funcs.readNC(abpath_in, 'vflux')
ivt=funcs.readNC(abpath_in, 'ivt')
_,ivtrec,ivtano=THR(ivt, [3,3,3], verbose=False)
self.ivt=ivt.squeeze()
self.ivtrec=ivtrec.squeeze()
self.ivtano=ivtano.squeeze()
self.uflux=uflux.squeeze()
self.vflux=vflux.squeeze()
self.latax=ivt.getLatitude()
self.lonax=ivt.getLongitude()
timeax=ivt.getTime()
self.timeax=timeax
self.param_dict={
# kg/m/s, define AR candidates as regions >= than this anomalous ivt.
'thres_low' : 1,
# km^2, drop AR candidates smaller than this area.
'min_area': 50*1e4,
# km^2, drop AR candidates larger than this area.
'max_area': 1800*1e4,
# float, isoperimetric quotient. ARs larger than this (more circular in shape) is treated as relaxed.
#'max_isoq': 0.6,
# float, isoperimetric quotient. ARs larger than this is discarded.
#'max_isoq_hard': 0.7,
'min_LW': 2,
# degree, exclude systems whose centroids are lower than this latitude.
'min_lat': 20,
# degree, exclude systems whose centroids are higher than this latitude.
'max_lat': 80,
# km, ARs shorter than this length is treated as relaxed.
'min_length': 2000,
# km, ARs shorter than this length is discarded.
'min_length_hard': 1500,
# degree lat/lon, error when simplifying axis using rdp algorithm.
'rdp_thres': 2,
# grids. Remove small holes in AR contour.
'fill_radius': max(1,int(4*0.75/0.75)),
# max prominence/height ratio of a local peak. Only used when SINGLE_DOME=True
'single_dome': False,
'max_ph_ratio': 0.4,
# minimal proportion of flux component in a direction to total flux to
# allow edge building in that direction
'edge_eps': 0.4,
'zonal_cyclic': False,
}
#--------Import modules-------------------------
import os
import sys
import numpy as np
import matplotlib.pyplot as plt
from netCDF4 import Dataset
from ipart.utils import funcs
from ipart.utils import plot
from ipart.AR_detector import plotAR, findARsGen
#-------------Main---------------------------------
if __name__ == '__main__':
#-----------Read in flux data----------------------
quNV = funcs.readNC(UQ_FILE_NAME, UQ_VAR)
qvNV = funcs.readNC(VQ_FILE_NAME, VQ_VAR)
#-----------------Shift longitude-----------------
quNV = quNV.shiftLon(SHIFT_LON)
qvNV = qvNV.shiftLon(SHIFT_LON)
#-------------------Read in ivt and THR results-------------------
ivtNV = funcs.readNC(IVT_FILE_NAME, 'ivt')
ivtrecNV = funcs.readNC(IVT_FILE_NAME, 'ivt_rec')
ivtanoNV = funcs.readNC(IVT_FILE_NAME, 'ivt_ano')
#--------------------Slice data--------------------
quNV = quNV.sliceData(TIME_START, TIME_END).sliceData(LAT1, LAT2,
axis=2).squeeze()
qvNV = qvNV.sliceData(TIME_START, TIME_END).sliceData(LAT1, LAT2,
OUTPUTDIR = '/home/guangzhi/datasets/quicksave2/THR/'
#--------Import modules-------------------------
import os
from ipart.utils import funcs
from ipart import thr
#-------------Main---------------------------------
if __name__ == '__main__':
filelist = []
if not os.path.exists(OUTPUTDIR):
os.makedirs(OUTPUTDIR)
#--------------------Read in orographic data--------------------
oroNV = funcs.readNC(ORO_FILE, 'z')
oroNV.data = oroNV.data / 9.8
oroNV = oroNV.sliceData(LAT1, LAT2, axis=1)
oroNV = oroNV.shiftLon(SHIFT_LON).squeeze()
for year in YEARS:
#-----------Read in data----------------------
file_in_name = FILE1_BASE % (year)
abpath_in = os.path.join(SOURCEDIR1, file_in_name)
filelist.append(abpath_in)
if len(filelist) < 2:
raise Exception(
"Need to give at least 2 files. For single file, use compute_thr_singlefile.py"
)
def rotatingTHR(filelist,
varin,
kernel,
outputdir,
oroNV=None,
selector=None,
high_terrain=600,
verbose=True):
'''Compute time filtering on data in different files.
Args:
filelist (list): list of abs paths to data files. User is responsible to
make sure files in list have correct chronological order.
Note that time axis in data files should be the 1st axis.
varin (str): variable id in files.
kernel (list or tuple): list/tuple of integers specifying the shape of
the kernel/structuring element used in the gray erosion process.
selector (utils.funcs.Selector): selector obj to select subset of data.
outputdir (str): path to folder to save outputs.
Keyword Args:
oroNV (NCVAR): 2D array, surface orographic data in meters.
This additional surface height info is used to perform a separate
reconstruction computation for areas with high elevations, and
the results can be used to enhance the continent-penetration
ability of landfalling ARs. Sensitivity in landfalling ARs is
enhanced, other areas are not affected. Needs to have compatible
shape as <ivt>.
If None, omit this process and treat areas with different heights
all equally.
New in v2.0.
high_terrain (float): minimum orographic height to define as high
terrain area, within which a separate reconstruction is performed.
Only used if <oroNV> is not None.
New in v2.0.
verbose (bool): print some messages or not.
Designed to perform temporal filtering on data that are too large to fit
into memory, e.g. high-resolution data across multiple decades.
Function will read in 2 files at once, call the filtering function on the
concatenated data, and shift 1 step to the next 2 files. If at the begining,
pad 0s to the left end. If in the mid, pad filtered data in the mid of
the concatenated data in the previous step. If at the end, pad 0s to
the right end.
The filtering function <func> is assumed to apply a filtering window with
odd length n, and truncates (n-1)/2 points from both ends. If the function
doesn't truncate data, will raise an exception.
'''
#----------------Check input files----------------
funcs.checkFiles(filelist)
for ii, fii in enumerate(filelist[:-1]):
if ii == 0:
var1 = funcs.readNC(fii, varin)
var1 = var1(selector)
else:
var1 = var2
del var2
fii2 = filelist[ii + 1]
var2 = funcs.readNC(fii2, varin)
var2 = var2(selector)
timeidx = funcs.interpretAxis('time', var1)
if timeidx != 0:
raise Exception("Time axis in variable is not at axis 0.")
timeidx = funcs.interpretAxis('time', var2)
if timeidx != 0:
raise Exception("Time axis in variable is not at axis 0.")
n1 = var1.shape[0]
vartmp = funcs.cat(var1, var2, axis=0)
vartmp, vartmp_rec, vartmp_ano = THR(vartmp,
kernel,
oroNV=oroNV,
high_terrain=high_terrain)
# crop end points
dt = kernel[0]
vartmp_rec = vartmp_rec.sliceIndex(dt, -dt)
vartmp_ano = vartmp_ano.sliceIndex(dt, -dt)
if dt <= 0:
raise Exception("dt<=0 not supported yet")
if verbose:
print('\n# <rotatingTHR>: Concatenated var shape:', vartmp.shape)
print('# <rotatingTHR>: Filtered var shape:', vartmp_rec.shape)
print('# <rotatingTHR>: Length difference:', dt)
if ii == 0:
#----------------------Pad 0s----------------------
left_rec = var1.sliceIndex(0, dt)
left_rec.data = left_rec.data * 0
left_rec.data.mask = True
left_ano = var1.sliceIndex(0, dt)
left_ano.data = left_ano.data * 0
left_ano.data.mask = True
else:
left_rec = mid_left_rec
left_ano = mid_left_ano
rec_pad = funcs.cat(left_rec, vartmp_rec, axis=0)
rec1 = rec_pad.sliceIndex(0, n1)
ano_pad = funcs.cat(left_ano, vartmp_ano, axis=0)
ano1 = ano_pad.sliceIndex(0, n1)
var1 = vartmp.sliceIndex(0, n1)
if verbose:
print('\n# <rotatingTHR>: Shape of left section after padding:',
rec1.shape)
rec1.id = vartmp_rec.id
attdict = getAttrDict(var1, 'reconstruction')
attdict['name'] = 'ivt_rec'
rec1.attributes = attdict
ano1.id = vartmp_ano.id
attdict = getAttrDict(var1, 'anomaly')
attdict['name'] = 'ivt_ano'
ano1.attributes = attdict
# left to pad in next iteration
mid_left_rec = vartmp_rec.sliceIndex(n1 - dt, n1)
mid_left_ano = vartmp_ano.sliceIndex(n1 - dt, n1)
#-----------------------Save-----------------------
fname = os.path.split(fii)[1]
file_out_name='%s-THR-kernel-t%d-s%d.nc'\
%(os.path.splitext(fname)[0], kernel[0], kernel[1])
abpath_out = os.path.join(outputdir, file_out_name)
print('\n### <testrotatingfilter>: Saving output to:\n', abpath_out)
funcs.saveNC(abpath_out, var1, 'w')
funcs.saveNC(abpath_out, rec1, 'a')
funcs.saveNC(abpath_out, ano1, 'a')
# save the right section for the last file
if ii == len(filelist) - 2:
right = var2.sliceIndex(-dt, None)
right.data = right.data * 0
right.data.mask = True
rec2 = rec_pad.sliceIndex(n1, None)
rec2 = funcs.cat(rec2, right, axis=0)
ano2 = ano_pad.sliceIndex(n1, None)
ano2 = funcs.cat(ano2, right, axis=0)
var2 = vartmp.sliceIndex(n1, None)
if verbose:
print(
'\n# <rotatingTHR>: Shape of last section after padding:',
ano2.shape)
rec2.id = vartmp_rec.id
attdict = getAttrDict(var2, 'reconstruction')
attdict['name'] = 'ivt_rec'
rec2.attributes = attdict
ano2.id = vartmp_ano.id
attdict = getAttrDict(var2, 'anomaly')
attdict['name'] = 'ivt_ano'
ano2.attributes = attdict
#-----------------------Save-----------------------
fname = os.path.split(fii2)[1]
file_out_name='%s-THR-kernel-t%d-s%d.nc'\
%(os.path.splitext(fname)[0], kernel[0], kernel[1])
abpath_out = os.path.join(outputdir, file_out_name)
print('\n### <testrotatingfilter>: Saving output to:\n',
abpath_out)
funcs.saveNC(abpath_out, var2, 'w')
funcs.saveNC(abpath_out, rec2, 'a')
funcs.saveNC(abpath_out, ano2, 'a')
return
OUTPUTDIR = '/home/guangzhi/datasets/quicksave2/THR'
#--------Import modules-------------------------
import os
from ipart.utils import funcs
from ipart import thr
#-------------Main---------------------------------
if __name__ == '__main__':
if not os.path.exists(OUTPUTDIR):
os.makedirs(OUTPUTDIR)
#-----------Read in data----------------------
print('\n### <compute_thr_singlefile>: Read in file:\n', IVT_FILE)
var = funcs.readNC(IVT_FILE, VARIN)
var = var.sliceIndex(0, 200, axis=0)
#--------------------Read in orographic data--------------------
oroNV = None
#-----------------Shift longitude-----------------
var = var.sliceData(LAT1, LAT2, 2)
var = var.shiftLon(SHIFT_LON)
#----------------------Do THR----------------------
ivt, ivtrec, ivtano = thr.THR(var,
KERNEL,
oroNV=oroNV,
high_terrain=HIGH_TERRAIN)
#-----------vflux----------------------
VFLUX_FILE = '/home/guangzhi/datasets/erai_qflux/vflux_m1-60_6_2007_cln-cea-proj.nc'
VFLUX_VARID = 'vflux'
OUTPUTFILE = '/home/guangzhi/datasets/quicksave2/THR/ivt_m1-60_6_2007_crop2.nc'
#--------Import modules-------------------------
import numpy as np
from ipart.utils import funcs
from ipart.utils import plot
#-------------Main---------------------------------
if __name__ == '__main__':
#-----------Read in data----------------------
ufluxNV = funcs.readNC(UFLUX_FILE, UFLUX_VARID)
vfluxNV = funcs.readNC(VFLUX_FILE, VFLUX_VARID)
ivtdata = np.ma.sqrt(ufluxNV.data**2 + vfluxNV.data**2)
ivtNV = funcs.NCVAR(
ivtdata, 'ivt', ufluxNV.axislist, {
'name': 'ivt',
'long_name': 'integrated vapor transport (IVT)',
'standard_name': 'integrated vapor transport (IVT)',
'title': 'integrated vapor transport (IVT)',
'units': getattr(ufluxNV, 'units', '')
})
#--------Save------------------------------------
print('\n### <compute_ivt>: Saving output to:\n', OUTPUTFILE)
funcs.saveNC(OUTPUTFILE, ivtNV, 'w')
if not os.path.exists(file_path):
raise Exception("Input file not exists.")
print('\n### <test_data>: Read in file:\n', file_path)
try:
fin = Dataset(file_path, 'r')
except Exception as e:
raise Exception("Failed to open file.\nException: %s" % str(e))
else:
fin.close()
try:
#var=fin.variables[sys.argv[2]]
var = funcs.readNC(file_path, sys.argv[2])
except:
raise Exception(
"Variable not found in file. Please verify the variable id is correct."
)
else:
# check rank
try:
if var.ndim not in [3, 4]:
raise Exception(
"Data should be in rank 3 or 4, i.e. (time, lat, lon) or (time, level, lat, lon)."
)
except:
raise Exception(
"Data should be in rank 3 or 4, i.e. (time, lat, lon) or (time, level, lat, lon)."