def linearInterpolation(grd0,grid):
if(grd0 is None): return None
grd1 = grd0.copy()
if(grd0.dlon * grd0.nlon >= 360):
grd1 = bd.grid_data(bd.grid(grd0.slon,grd0.dlon,grd0.elon + grd0.dlon, grd0.slat,grd0.dlat,grd0.elat))
grd1.dat[:,0:-1] = grd0.dat[:,:]
grd1.dat[:,grd0.nlon] = grd0.dat[:,0]
grd2 = bd.grid_data(grid)
if(grd2.elon > grd1.elon or grd2.slon < grd1.slon or grd2.elat > grd1.elat or grd2.slat < grd1.slat):
print("object grid is out range of original grid")
grd2.dat = bd.IV
return
x = ((np.arange(grd2.nlon) * grd2 .dlon + grd2.slon - grd1.slon) / grd1.dlon)
ig = x[:].astype(dtype = 'int16')
dx = x - ig
y = (np.arange(grd2.nlat) * grd2 .dlat + grd2.slat - grd1.slat) / grd1.dlat
jg = y[:].astype(dtype = 'int16')
dy = y[:] - jg
ii,jj = np.meshgrid(ig,jg)
ii1 = np.minimum(ii + 1, grd1.nlon-1)
jj1 = np.minimum(jj + 1, grd1.nlat-1)
ddx,ddy = np.meshgrid(dx,dy)
c00 = (1 - ddx) * (1 - ddy)
c01 = ddx * (1 - ddy)
c10 = (1 - ddx) * ddy
c11 = ddx * ddy
grd2.dat = (c00 * grd1.dat[jj,ii] + c10 * grd1.dat[jj1,ii] +c01 * grd1.dat[jj,ii1] + c11 * grd1.dat[jj1,ii1])
return grd2
def timelist_to_grid_nearHappen(grid, tL, R=0.2, start=None, end=None):
sta = timelist_to_sta(tL, start, end)
grd = bd.grid_data(grid)
yN = int(R / grid.dlat) + 1
minD = grid.dlat * grid.dlat / (R * R)
for k in range(sta.num):
sr = math.cos(sta.dat[k, 2] * math.pi / 180)
sr2 = sr * sr
xN = int(yN / sr)
jg = int(round((sta.dat[k, 2] - grd.slat) / grd.dlat))
ig = int(round((sta.dat[k, 1] - grd.slon) / grd.dlat))
maxp = min(ig + xN, grd.nlon) - ig
minp = max(ig - xN, 0) - ig
maxq = min(jg + yN, grd.nlat) - jg
minq = max(jg - yN, 0) - jg
for p in range(minp, maxp):
p2 = p * p * sr2
for q in range(minq, maxq):
dense = 1 / (p2 + q * q + 0.0001)
if (dense >= minD):
grd.dat[jg + q, ig + p] += dense
grd.dat[grd.dat >= 0.5] = 1
grd.dat[grd.dat < 1] = 0
return grd
def read_from_micaps4(filename):
try:
if not os.path.exists(filename):
print(filename + " is not exist")
return None
file = open(filename,'r')
str1 = file.read()
file.close()
strs = str1.split()
dlon = float(strs[9]);
dlat = float(strs[10]);
slon = float(strs[11]);
elon = float(strs[12]);
slat = float(strs[13]);
elat = float(strs[14]);
grd = bd.grid_data(bd.grid(slon,dlon,elon,slat,dlat,elat))
if len(strs) - 22 >= grd.nlon * grd.nlat :
k=22
grd.dat = (np.array(strs[k:])).astype(float).reshape((grd.nlat,grd.nlon))
grd.reset()
return grd
else:
return None
except:
print(filename + "'s format is wrong")
return None
def explain_awx_bytes(byteArray):
sat96 = struct.unpack("12s", byteArray[:12])[0]
levl = np.frombuffer(byteArray[12:30], dtype='int16').astype(dtype = "int32")
formatstr = struct.unpack("8s", byteArray[30:38])[0]
qualityflag = struct.unpack("h", byteArray[38:40])[0]
satellite = struct.unpack("8s", byteArray[40:48])[0]
lev2 = np.frombuffer(byteArray[48:104], dtype='int16').astype(dtype = "int32")
recordlen = levl[4]
headnum = levl[5]
datanum = levl[6]
timenum =lev2[0:5]
nlon = lev2[7]
nlat = lev2[8]
range =lev2[12:16].astype("float32")
slat = range[0]/100
elat = range[1]/100
slon = range[2]/100
elon = range[3]/100
#nintels=lev2[20:22].astype("float32")
dlon = (elon - slon) / (nlon-1)
dlat = (elat - slat) / (nlat-1)
grd = bd.grid_data(bd.grid(slon, dlon, elon, slat, dlat, elat))
colorlen = lev2[24]
caliblen = lev2[25]
geololen = lev2[26]
#print(levl)
#print(lev2)
head_lenght = headnum * recordlen
data_lenght = datanum * recordlen
#print(head_lenght + data_lenght)
#print( data_lenght)
#print(grd.nlon * grd.nlat)
#headrest = np.frombuffer(byteArray[:head_lenght], dtype='int8')
data_awx = np.frombuffer(byteArray[head_lenght:(head_lenght+data_lenght)], dtype='int8')
#print(headrest)
if colorlen<=0:
calib = np.frombuffer(byteArray[104:(104+2048)], dtype='int16').astype(dtype="float32")
else:
#color = np.frombuffer(byteArray[104:(104+colorlen*2)], dtype='int16')
calib = np.frombuffer(byteArray[(104+colorlen*2):(104+colorlen*2+ 2048)], dtype='int16').astype(dtype="float32")
realcalib = calib /100.0
realcalib[calib<0] = (calib[calib<0] + 65536) /100.0
awx_index = np.empty(len(data_awx),dtype = "int32")
awx_index[:] = data_awx[:]
awx_index[data_awx <0] = data_awx[data_awx <0] +256
awx_index *= 4
real_data_awx = realcalib[awx_index]
grd.dat = real_data_awx.reshape(grd.nlat,grd.nlon)
grd.reset()
return grd
def sta_to_grid_idw(sta, grid, background=None, effectR=1000, nearNum=16):
grd = bd.grid_data(grid)
xyz_sta = lon_lat_to_cartesian(sta.ix[:, 0], sta.ix[:, 1], R=bd.ER)
lon = np.arange(grd.nlon) * grd.dlon + grd.slon
lat = np.arange(grd.nlat) * grd.dlat + grd.slat
grid_lon, grid_lat = np.meshgrid(lon, lat)
xyz_grid = lon_lat_to_cartesian(grid_lon.flatten(),
grid_lat.flatten(),
R=bd.ER)
tree = cKDTree(xyz_sta)
d, inds = tree.query(xyz_grid, k=nearNum)
d += 1e-6
w = 1.0 / d**2
input_dat = sta.ix[:, 2]
dat = np.sum(w * input_dat[inds], axis=1) / np.sum(w, axis=1)
bg = bd.grid_data(grid)
if (background is not None):
bg = ggf.linearInterpolation(background, grid)
bg_dat = bg.dat.flatten()
dat = np.where(d[:, 0] > effectR, bg_dat, dat)
grd.dat = dat.reshape((grd.nlat, grd.nlon))
return grd
def sta_to_grid_linear(sta, grid, background=None):
grd = bd.grid_data(grid)
points = sta.ix[:, 0:2]
values = sta.ix[:, 2]
x = np.arange(grd.nlon) * grd.dlon + grd.slon
y = np.arange(grd.nlat) * grd.dlat + grd.slat
grid_x, grid_y = np.meshgrid(x, y)
dat_linear = griddata(points, values, (grid_x, grid_y), method='linear')
if (background is None):
dat_near = griddata(points, values, (grid_x, grid_y), method='nearest')
grd.dat = np.where(np.isnan(dat_linear), dat_near, dat_linear)
else:
bg = ggf.linearInterpolation(background, grid)
grd.dat = np.where(np.isnan(dat_linear), bg.dat, dat_linear)
return grd
def read_from_gds(filename,service = None,grid = None):
try:
if(service is None):service = GDS_data_service.service
directory,fileName = os.path.split(filename)
status, response = byteArrayResult = service.getData(directory, fileName)
ByteArrayResult = DataBlock_pb2.ByteArrayResult()
if status == 200:
ByteArrayResult.ParseFromString(response)
if ByteArrayResult is not None:
byteArray = ByteArrayResult.byteArray
file_type = os.path.splitext(filename)
if file_type[1] == '.AWX':
return explain_awx_bytes(byteArray)
else:
#print(len(byteArray))
discriminator = struct.unpack("4s", byteArray[:4])[0].decode("gb2312")
t = struct.unpack("h", byteArray[4:6])
mName = struct.unpack("20s", byteArray[6:26])[0].decode("gb2312")
eleName = struct.unpack("50s", byteArray[26:76])[0].decode("gb2312")
description = struct.unpack("30s", byteArray[76:106])[0].decode("gb2312")
level, y, m, d, h, timezone, period = struct.unpack("fiiiiii", byteArray[106:134])
startLon, endLon, lonInterval, lonGridCount = struct.unpack("fffi", byteArray[134:150])
startLat, endLat, latInterval, latGridCount = struct.unpack("fffi", byteArray[150:166])
isolineStartValue, isolineEndValue, isolineInterval = struct.unpack("fff", byteArray[166:178])
gridCount = lonGridCount * latGridCount
description = mName.rstrip('\x00') + '_' + eleName.rstrip('\x00') + "_" + str(
level) + '(' + description.rstrip('\x00') + ')' + ":" + str(period)
if (gridCount == (len(byteArray) - 278) / 4):
if(startLat > 90):startLat = 90.0
if(startLat < -90) : startLat = -90.0
if(endLat > 90) : endLat = 90.0
if(endLat < -90): endLat = -90.0
grd = bd.grid_data(bd.grid(startLon,lonInterval,endLon,startLat,latInterval,endLat))
grd.dat = np.frombuffer(byteArray[278:], dtype='float32').reshape(grd.nlat,grd.nlon)
grd.reset()
if (grid is None):
return grd
else:
return bf.ggf.linearInterpolation(grd,grid)
except Exception as e:
print(e)
return None
def transform(sta):
slon = np.min(sta.ix[:, 0])
elon = np.max(sta.ix[:, 0])
slat = np.min(sta.ix[:, 1])
elat = np.max(sta.ix[:, 1])
nsta = len(sta.index)
for i in range(nsta - 1):
dlon = sta.ix[i, 0] - sta.ix[i + 1, 0]
if dlon != 0:
dlon = math.fabs(dlon)
break
for i in range(nsta - 1):
dlat = sta.ix[i, 1] - sta.ix[i + 1, 1]
if dlat != 0:
dlat = math.fabs(dlat)
break
grd = bd.grid_data(bd.grid(slon, dlon, elon, slat, dlat, elat))
ig = ((sta.ix[:, 0] - grd.slon) // grd.dlon).astype(dtype='int16')
jg = ((sta.ix[:, 1] - grd.slat) // grd.dlat).astype(dtype='int16')
grd.dat[jg, ig] = sta.ix[:, 2]
return grd
def read_from_nc(filename,valueName = None):
if os.path.exists(filename):
f = Dataset(filename)
lons = None
lats = None
for key in f.variables.keys():
#print(key)
if 'lon' in key.lower():
lonName = str(key)
lons = f.variables[lonName][:]
elif 'lat' in key.lower():
latName = key
lats = f.variables[latName][:]
elif 'time' not in key.lower():
if(valueName is None):
valueName = key
dlon = (lons[-1] - lons[0]) / (len(lons) - 1)
dlat = (lats[-1] - lats[0]) / (len(lats) - 1)
grd = bd.grid_data(bd.grid(lons[0],dlon,lons[-1],lats[0],dlat,lats[-1]))
grd.dat = f.variables[valueName][:].T
return grd
def expand_to_contain_another_grid(grd0,grid):
grd0.reset()
grid.reset()
si = 0
sj = 0
ei = 0
ej = 0
if(grid.slon < grd0.grid.slon):
si = int(math.ceil((grd0.grid.slon - grid.slon)/grd0.grid.dlon))
if(grid.slat < grd0.grid.slat):
sj = int(math.ceil((grd0.grid.slat - grid.slat)/grd0.grid.dlat))
if(grid.elon > grd0.grid.elon):
ei = int(math.ceil((-grd0.grid.elon + grid.elon)/grd0.grid.dlon))
if(grid.elat > grd0.grid.elat):
ej = int(math.ceil((-grd0.grid.elat + grid.elat)/grd0.grid.dlat))
slon = grd0.grid.slon - si * grd0.grid.dlon
slat = grd0.grid.slat - sj * grd0.grid.dlat
elon = grd0.grid.elon + ei * grd0.grid.dlon
elat = grd0.grid.elat + ej * grd0.grid.dlat
grd1 = bd.grid_data(bd.grid(slon,grd0.grid.dlon,elon,slat,grd0.grid.dlat,elat))
grd1.dat[sj:(sj + grd0.grid.nlat), si:(si + grd0.grid.nlon)] = grd0.dat[:, :]
return grd1
def read_grd_array_by_ctl(filename,ctl = None):
if ctl is None:
if os.path.exists(filename):
ctl = read_ctl(filename)
return read_grd_array_by_ctl(ctl.data_path,ctl)
else:
if os.path.exists(filename):
data = np.fromfile(filename, dtype='f')
data = data.reshape(ctl.nvar, ctl.nensemble, ctl.ntime, ctl.nlevel, ctl.nlat, ctl.nlon)
#print(data.shape)
grid = bd.grid(ctl.slon, ctl.dlon, ctl.elon, ctl.slat, ctl.dlat, ctl.elat)
arr = [[[[] for t in range(ctl.ntime)] for n in range(ctl.nensemble)] for v in range(ctl.nvar)]
for v in range(ctl.nvar):
for n in range(ctl.nensemble):
for t in range(ctl.ntime):
for z in range(ctl.nlevel):
grd = bd.grid_data(grid)
grd.dat = data[v, n, t, z, :, :]
grd.reset()
arr[v][n][t].append(grd)
return arr
return None
def cubicInterpolation(grd1,grid):
if (grd1 is None): return None
grd2 = bd.grid_data(grid)
if(grd2.elon > grd1.elon or grd2.slon < grd1.slon or grd2.elat > grd1.elat or grd2.slat < grd1.slat):
grd2.dat = bd.IV
return
x = ((np.arange(grd2.nlon) * grd2 .dlon + grd2.slon - grd1.slon) / grd1.dlon)
ig = x[:].astype(dtype = 'int16')
dx = x - ig
y = (np.arange(grd2.nlat) * grd2 .dlat + grd2.slat - grd1.slat) / grd1.dlat
jg = y[:].astype(dtype = 'int16')
dy = y[:] - jg
ii, jj = np.meshgrid(ig, jg)
for p in range(-1,3,1):
iip = np.minimum(np.maximum(ii+p,0),grd1.nlon-1)
fdx = cubic_f(p, dx)
for q in range(-1,3,1):
jjq = np.minimum(np.maximum(jj+q,0),grd1.nlat-1)
fdy = cubic_f(q,dy)
fdxx,fdyy = np.meshgrid(fdx,fdy)
fdxy = fdxx * fdyy
grd2.dat += fdxy * grd1.dat[jjq,iip]
return grd2
def read_from_grib2(filename, valueName, typeOfLevel=None, level=None):
if os.path.exists(filename):
gribs = pygrib.open(filename)
if (level is None):
grib = gribs.select(name=valueName, typeOfLevel='surface')[0]
else:
grib = gribs.select(shortName=valueName,
typeOfLevel=typeOfLevel,
level=level)[0]
if (grib is not None):
latlons = grib.latlons()
lats = latlons[0][:, 0]
lons = latlons[1][0, :]
grd = bd.grid_data(
bd.grid(lons[0], lons[1] - lons[0], lons[-1], lats[0],
lats[1] - lats[0], lats[-1]))
grd.dat = grib.values
print(grd.nlon)
print(grd.nlat)
print(grd.dat.shape)
grd.reset()
return grd
def get_u(wind):
grd = bd.grid_data(wind.grid)
grd.dat[:, :] = wind.u[:, :]
return grd
def get_wind_speed(wind):
grd = bd.grid_data(wind.grid)
grd.dat = np.power((np.power(wind.v, 2) + np.power(wind.u, 2)), 0.5)
return grd
def sta_to_grid_oa2(sta0, background, sm=1, effect_R=1000, rate_of_model=0):
sta = ssf.remove_IV(sta0)
sta = ssf.get_sta_in_grid(sta, background.grid)
grd = background.copy()
ig = ((sta.ix[:, 0] - grd.slon) // grd.dlon).astype(dtype='int16')
jg = ((sta.ix[:, 1] - grd.slat) // grd.dlat).astype(dtype='int16')
dx = (sta.ix[:, 0] - grd.slon) / grd.dlon - ig
dy = (sta.ix[:, 1] - grd.slat) / grd.dlat - jg
c00 = (1 - dx) * (1 - dy)
c01 = dx * (1 - dy)
c10 = (1 - dx) * dy
c11 = dx * dy
lat = np.arange(grd.nlat) * grd.dlat + grd.slat
sr = 1 / np.power(np.cos(lat * math.pi / 180), 4)
def targe(x):
grdv = x.reshape(grd.nlat, grd.nlon)
dx = grdv[:, :-2] + grdv[:, 2:] - 2 * grdv[:, 1:-1]
cost1 = np.sum(dx * dx)
dy = grdv[:-2, :] + grdv[2:, :] - 2 * grdv[1:-1, :]
dy2 = dy * dy
sum_dy = np.sum(dy2, axis=1)
cost1 = cost1 + np.dot(sum_dy, sr[1:-1])
sta_g = c00 * grd.dat[jg, ig] + c01 * grd.dat[
jg, ig + 1] + c10 * grd.dat[jg + 1, ig] + c11 * grd.dat[jg + 1,
ig + 1]
error = sta.ix[:, 2] - sta_g
cost2 = np.sum(error * error)
cost = sm * cost1 + cost2
return cost
def grads(x):
grdv = x.reshape(grd.nlat, grd.nlon)
g1 = np.zeros(grdv.shape)
dx = 2 * (grdv[:, :-2] + grdv[:, 2:] - 2 * grdv[:, 1:-1])
g1[:, :-2] = dx
g1[:, 2:] += dx
g1[:, 1:-1] -= 2 * dx
sr_expend = np.tile(sr[1:-1], [grd.nlon, 1]).T
dy = 2 * (grdv[:-2, :] + grdv[2:, :] - 2 * grdv[1:-1, :])
dy_sr = dy * sr_expend
g1[:-2, :] += dy_sr
g1[2:, :] += dy_sr
g1[1:-1, :] -= 2 * dy_sr
g2 = np.zeros(grdv.shape)
sta_g = c00 * grd.dat[jg, ig] + c01 * grd.dat[
jg, ig + 1] + c10 * grd.dat[jg + 1, ig] + c11 * grd.dat[jg + 1,
ig + 1]
d = 2 * (sta_g - sta.ix[:, 2])
g2[jg, ig] += d * c00
g2[jg, ig + 1] += d * c01
g2[jg + 1, ig] += d * c10
g2[jg + 1, ig + 1] += d * c11
g = sm * g1 + g2
return g.reshape(-1)
x = grd.dat.reshape(-1)
x_oa = frprmn2(x, targe, grads)
grd = bd.grid_data(grd.grid)
grd.dat = x_oa.reshape(grd.nlat, grd.nlon)
return grd