def extract_cfgrib1(self, fname):
nwps = dict()
data = cfgrib.open_dataset(fname)
dates = pd.to_datetime(
data.valid_time.data,
format='%Y-%m-%d %H:%M:%S').strftime('%d%m%y%H%M')
Uwind = data.u100.data
Vwind = data.v100.data
temp = data.t2m.data
cloud = data.tcc.data
flux = data.ssrd.data
lat = data.latitude.data
long = data.longitude.data
r2d = 45.0 / np.arctan(1.0)
wspeed = np.sqrt(np.square(Uwind) + np.square(Vwind))
wdir = np.arctan2(Uwind, Vwind) * r2d + 180
for i, dt in enumerate(dates):
nwps[dt] = dict()
lat1, long1 = self.find_latlong(lat, long)
nwps[dt]['lat'] = lat[lat1]
nwps[dt]['long'] = long[long1]
nwps[dt]['Uwind'] = Uwind[i][lat1, long1]
nwps[dt]['Vwind'] = Vwind[i][lat1, long1]
nwps[dt]['WS'] = wspeed[i][lat1, long1]
nwps[dt]['WD'] = wdir[i][lat1, long1]
nwps[dt]['Temperature'] = temp[i][lat1, long1]
nwps[dt]['Cloud'] = cloud[i][lat1, long1]
nwps[dt]['Flux'] = flux[i][lat1, long1]
return nwps
def load_grib_surface(f):
return cfgrib.open_dataset(f,
backend_kwargs={
'filter_by_keys': {
'typeOfLevel': 'surface'
}
}).metpy.parse_cf()
def extract_cfgrib2(self, t, fname):
path_extract = os.path.join(self.pathnwp,
'extract/' + t.strftime('%d%m%y'))
if not os.path.exists(path_extract):
os.makedirs(path_extract)
tar = tarfile.open(fname)
tar.extractall(path_extract)
tar.close()
dates = pd.date_range(start=t,
end=t + pd.DateOffset(hours=48),
freq='H')
nwps = dict()
for i, dt in enumerate(dates):
file = os.path.join(
path_extract, 'E_H6S' + t.strftime('%m%d') + '0000' +
dt.strftime('%m%d') + str(dt.hour).zfill(2) + '001')
if not os.path.exists(file):
file = os.path.join(
path_extract, 'E_H6S' + t.strftime('%m%d') + '0000' +
t.strftime('%m%d') + '00011')
if not os.path.exists(file):
continue
data = cfgrib.open_dataset(file)
Uwind = data.u100.data
Vwind = data.v100.data
temp = data.t2m.data
cloud = data.tcc.data
flux = data.ssrd.data
lat = data.latitude.data
long = data.longitude.data
r2d = 45.0 / np.arctan(1.0)
wspeed = np.sqrt(np.square(Uwind) + np.square(Vwind))
wdir = np.arctan2(Uwind, Vwind) * r2d + 180
lat1, long1 = self.find_latlong(lat, long)
nwp = dict()
nwp['lat'] = lat[lat1][0]
nwp['long'] = long[long1][0]
nwp['Uwind'] = Uwind[lat1, long1][0]
nwp['Vwind'] = Vwind[lat1, long1][0]
nwp['WS'] = wspeed[lat1, long1][0]
nwp['WD'] = wdir[lat1, long1][0]
nwp['Temperature'] = temp[lat1, long1][0]
nwp['Cloud'] = cloud[lat1, long1][0]
nwp['Flux'] = flux[lat1, long1][0]
nwps[dt.strftime('%d%m%y%H%M')] = nwp
return nwps
def adhoc_merge(datasets, filepattern):
"""File structures are heterogenious across time.
This function is to try to fix some of that to 2018-07 which is
the file structure I was working with first
"""
if 1 in datasets.keys():
# identify change by cfgrib extra splitting behaviour
# before file strucuture change [t, r, dpt, vis] are all at index 0
if list(datasets[1][0].keys()) == ['t'] and list(
datasets[1][1].keys()) == ['r', 'dpt', 'vis']:
datasets[1][0] = cfgrib.open_dataset(
filepattern.format(1),
backend_kwargs=dict(filter_by_keys={'level': 1}))
for i in range(1, len(datasets[1]) - 1):
datasets[1][i] = datasets[1][i + 1]
return datasets
def extract_cfgrib3(self, t, fname):
dates = pd.date_range(start=t,
end=t + pd.DateOffset(hours=48),
freq='H')
nwps = dict()
for i, dt in enumerate(dates):
file = os.path.join(
fname, 'H6S' + t.strftime('%m%d') + '0000' +
dt.strftime('%m%d') + str(dt.hour).zfill(2) + '001')
if not os.path.exists(file):
file = os.path.join(
fname, 'H6S' + t.strftime('%m%d') + '0000' +
t.strftime('%m%d') + '00011')
if not os.path.exists(file):
continue
data = cfgrib.open_dataset(file)
Uwind = data.u100.data
Vwind = data.v100.data
temp = data.t2m.data
cloud = data.tcc.data
flux = data.ssrd.data
lat = data.latitude.data
long = data.longitude.data
r2d = 45.0 / np.arctan(1.0)
wspeed = np.sqrt(np.square(Uwind) + np.square(Vwind))
wdir = np.arctan2(Uwind, Vwind) * r2d + 180
nwp = dict()
nwp['lat'] = lat
nwp['long'] = long
nwp['Uwind'] = Uwind
nwp['Vwind'] = Vwind
nwp['WS'] = wspeed
nwp['WD'] = wdir
nwp['Temperature'] = temp
nwp['Cloud'] = cloud
nwp['Flux'] = flux
nwps[dt.strftime('%d%m%y%H%M')] = nwp
return nwps
'''
******************************************************************************
******************************************************************************
******************************************************************************
WIND CORRELATION
******************************************************************************
******************************************************************************
******************************************************************************
'''
# reading wind file
fwind = '/home/aline/Documents/Dados/ERA5/wind_monthly_1979_2020.grib'
dwind = cfgrib.open_dataset(fwind)
# for wind
index_crop = indices[slice(
dwind.isel(time=0).time.values,
dwind.isel(time=-1).time.values)]
index_crop = index_crop.id.to_xarray()
# selecting winter season for index data
index_winter = index_crop.sel(time=index_crop['time.season'] == 'DJF')
correlacao = xr.merge([
xr.corr(dwind.si10.round(3), index_crop.round(3),
dim='time').to_dataset(name='wspd'),
xr.corr(dwind.u10.round(3), index_crop.round(3),
dim='time').to_dataset(name='U'),
inLat = 37.56083
inLon = 129.2172
inVars = ['shww', 'wvdir', 'pp1d', 'u', 'v']
# =================================================
# Main
# =================================================
try:
log.info('[START] {}'.format('Main'))
fileList = glob.glob('{}/{}'.format(globalVar['inpPath'], '*.gb2'))
for fileInfo in fileList:
log.info('[CHECK] fileInfo : {}'.format(fileInfo))
ds = cfgrib.open_dataset(fileInfo)
dtDateTime = pd.to_datetime(ds.time.values).strftime('%Y%m%d%H%M')
log.info('[CHECK] ds : {}'.format(ds))
for inVar in inVars:
log.info('[CHECK] inVar : {}'.format(inVar))
# 오랜 시간이 소요
# df = ds[inVar].to_dataframe()
# saveFile = '{}/{}_{}_{}_{}.csv'.format(globalVar['outPath'], serviceName, inVar, 'TOTAL_DATA', dtDateTime)
# df.to_csv(saveFile, index=None)
dataL2 = pd.DataFrame()
for i in ds[inVar]:
dtDateTimeFore = pd.to_datetime((ds.time + i.step).values).strftime('%Y%m%d%H%M')
nearVal = np.nan_to_num(i.sel(latitude = inLat, longitude = inLon, method='nearest'))