def linearInterpolation(grd0, grid): if (grd0 is None): return None grd1 = grd0.copy() if (grd0.dlon * grd0.nlon >= 360): grd1 = bd.grid_data( bd.grid(grd0.slon, grd0.dlon, grd0.elon + grd0.dlon, grd0.slat, grd0.dlat, grd0.elat)) grd1.dat[:, 0:-1] = grd0.dat[:, :] grd1.dat[:, grd0.nlon] = grd0.dat[:, 0] grd2 = bd.grid_data(grid) if (grd2.elon > grd1.elon or grd2.slon < grd1.slon or grd2.elat > grd1.elat or grd2.slat < grd1.slat): print("object grid is out range of original grid") grd2.dat = bd.IV return x = ((np.arange(grd2.nlon) * grd2.dlon + grd2.slon - grd1.slon) / grd1.dlon) ig = x[:].astype(dtype='int16') dx = x - ig y = (np.arange(grd2.nlat) * grd2.dlat + grd2.slat - grd1.slat) / grd1.dlat jg = y[:].astype(dtype='int16') dy = y[:] - jg ii, jj = np.meshgrid(ig, jg) ii1 = np.minimum(ii + 1, grd1.nlon - 1) jj1 = np.minimum(jj + 1, grd1.nlat - 1) ddx, ddy = np.meshgrid(dx, dy) c00 = (1 - ddx) * (1 - ddy) c01 = ddx * (1 - ddy) c10 = (1 - ddx) * ddy c11 = ddx * ddy grd2.dat = (c00 * grd1.dat[jj, ii] + c10 * grd1.dat[jj1, ii] + c01 * grd1.dat[jj, ii1] + c11 * grd1.dat[jj1, ii1]) return grd2
def read_grd_array_by_ctl(filename, ctl=None, endian=None): if ctl is None: if os.path.exists(filename): ctl = read_ctl(filename) return read_grd_array_by_ctl(ctl.data_path, ctl) else: if os.path.exists(filename): if (endian is None): data = np.fromfile(filename, dtype='f') else: data = np.fromfile(filename, dtype='>f') data = data.reshape(ctl.nvar, ctl.nensemble, ctl.ntime, ctl.nlevel, ctl.nlat, ctl.nlon) #print(data.shape) grid = bd.grid(ctl.slon, ctl.dlon, ctl.elon, ctl.slat, ctl.dlat, ctl.elat) arr = [[[[] for t in range(ctl.ntime)] for n in range(ctl.nensemble)] for v in range(ctl.nvar)] for v in range(ctl.nvar): for n in range(ctl.nensemble): for t in range(ctl.ntime): for z in range(ctl.nlevel): grd = bd.grid_data(grid) grd.dat = data[v, n, t, z, :, :] grd.reset() arr[v][n][t].append(grd) return arr return None
def cubicInterpolation(grd1, grid): if (grd1 is None): return None grd2 = bd.grid_data(grid) if (grd2.elon > grd1.elon or grd2.slon < grd1.slon or grd2.elat > grd1.elat or grd2.slat < grd1.slat): grd2.dat = bd.IV return x = ((np.arange(grd2.nlon) * grd2.dlon + grd2.slon - grd1.slon) / grd1.dlon) ig = x[:].astype(dtype='int16') dx = x - ig y = (np.arange(grd2.nlat) * grd2.dlat + grd2.slat - grd1.slat) / grd1.dlat jg = y[:].astype(dtype='int16') dy = y[:] - jg ii, jj = np.meshgrid(ig, jg) for p in range(-1, 3, 1): iip = np.minimum(np.maximum(ii + p, 0), grd1.nlon - 1) fdx = cubic_f(p, dx) for q in range(-1, 3, 1): jjq = np.minimum(np.maximum(jj + q, 0), grd1.nlat - 1) fdy = cubic_f(q, dy) fdxx, fdyy = np.meshgrid(fdx, fdy) fdxy = fdxx * fdyy grd2.dat += fdxy * grd1.dat[jjq, iip] return grd2
def sta_to_grid_idw(sta, grid, background=None, effectR=1000, nearNum=16): grd = bd.grid_data(grid) xyz_sta = lon_lat_to_cartesian(sta.ix[:, 0], sta.ix[:, 1], R=bd.ER) lon = np.arange(grd.nlon) * grd.dlon + grd.slon lat = np.arange(grd.nlat) * grd.dlat + grd.slat grid_lon, grid_lat = np.meshgrid(lon, lat) xyz_grid = lon_lat_to_cartesian(grid_lon.flatten(), grid_lat.flatten(), R=bd.ER) tree = cKDTree(xyz_sta) d, inds = tree.query(xyz_grid, k=nearNum) d += 1e-6 w = 1.0 / d**2 input_dat = sta.values[:, 2] dat = np.sum(w * input_dat[inds], axis=1) / np.sum(w, axis=1) bg = bd.grid_data(grid) if (background is not None): bg = ggf.linearInterpolation(background, grid) bg_dat = bg.dat.flatten() dat = np.where(d[:, 0] > effectR, bg_dat, dat) grd.dat = dat.reshape((grd.nlat, grd.nlon)) return grd
def sta_to_grid_linear(sta, grid, background=None): grd = bd.grid_data(grid) points = sta.ix[:, 0:2] values = sta.ix[:, 2] x = np.arange(grd.nlon) * grd.dlon + grd.slon y = np.arange(grd.nlat) * grd.dlat + grd.slat grid_x, grid_y = np.meshgrid(x, y) dat_linear = griddata(points, values, (grid_x, grid_y), method='linear') if (background is None): dat_near = griddata(points, values, (grid_x, grid_y), method='nearest') grd.dat = np.where(np.isnan(dat_linear), dat_near, dat_linear) else: bg = ggf.linearInterpolation(background, grid) grd.dat = np.where(np.isnan(dat_linear), bg.dat, dat_linear) return grd
def read_from_gds_file(filename, grid=None): print("a") try: if not os.path.exists(filename): print(filename + " is not exist") return None file = open(filename, 'rb') byteArray = file.read() discriminator = struct.unpack("4s", byteArray[:4])[0].decode("gb2312") t = struct.unpack("h", byteArray[4:6]) mName = struct.unpack("20s", byteArray[6:26])[0].decode("gb2312") eleName = struct.unpack("50s", byteArray[26:76])[0].decode("gb2312") description = struct.unpack("30s", byteArray[76:106])[0].decode("gb2312") level, y, m, d, h, timezone, period = struct.unpack( "fiiiiii", byteArray[106:134]) startLon, endLon, lonInterval, lonGridCount = struct.unpack( "fffi", byteArray[134:150]) startLat, endLat, latInterval, latGridCount = struct.unpack( "fffi", byteArray[150:166]) isolineStartValue, isolineEndValue, isolineInterval = struct.unpack( "fff", byteArray[166:178]) gridCount = lonGridCount * latGridCount description = mName.rstrip('\x00') + '_' + eleName.rstrip( '\x00') + "_" + str(level) + '(' + description.rstrip( '\x00') + ')' + ":" + str(period) if (gridCount == (len(byteArray) - 278) / 4): if (startLat > 90): startLat = 90.0 if (startLat < -90): startLat = -90.0 if (endLat > 90): endLat = 90.0 if (endLat < -90): endLat = -90.0 grd = bd.grid_data( bd.grid(startLon, lonInterval, endLon, startLat, latInterval, endLat)) grd.dat = np.frombuffer(byteArray[278:], dtype='float32').reshape( grd.nlat, grd.nlon) grd.reset() if (grid is None): return grd else: return bt.ggf.linearInterpolation(grd, grid) except Exception as e: print(e) return None
def read_from_nc(filename, valueName=None, grid=None): if os.path.exists(filename): f = Dataset(filename) lons = None lats = None for key in f.variables.keys(): #print(key) ndim = f.variables[key].ndim if (ndim < 2): if 'lon' in key.lower(): lonName = str(key) lons = f.variables[lonName][:] if 'lat' in key.lower(): latName = key lats = f.variables[latName][:] else: if (valueName is None): valueName = key dlon = (lons[-1] - lons[0]) / (len(lons) - 1) dlat = (lats[-1] - lats[0]) / (len(lats) - 1) grd = bd.grid_data( bd.grid(lons[0], dlon, lons[-1], lats[0], dlat, lats[-1])) dat = np.squeeze(f.variables[valueName][:]) if (str(type(dat)) == "<class 'numpy.ma.core.MaskedArray'>"): dat[dat.mask == True] = 0 dat = dat.data if grd.nlon == grd.nlat: strings = str(f.variables[valueName]).split("\n")[1] lon_index = strings.find("lon") lat_index = strings.find("lat") if (lon_index < lat_index): dat = dat.T else: if grd.nlon == len(dat): dat = dat.T grd.dat = dat grd.reset() f.close() if (grid is None): return grd else: return bt.ggf.linearInterpolation(grd, grid)
def read_from_micaps4(filename, grid=None): try: if not os.path.exists(filename): print(filename + " is not exist") return None try: file = open(filename, 'r') str1 = file.read() file.close() except: file = open(filename, 'r', encoding='utf-8') str1 = file.read() file.close() strs = str1.split() dlon = float(strs[9]) dlat = float(strs[10]) slon = float(strs[11]) elon = float(strs[12]) slat = float(strs[13]) elat = float(strs[14]) nlon = int(strs[15]) nlat = int(strs[16]) elon = slon + dlon * (nlon - 1) elat = slat + dlat * (nlat - 1) grd = bd.grid_data(bd.grid(slon, dlon, elon, slat, dlat, elat)) if len(strs) - 22 >= grd.nlon * grd.nlat: k = 22 grd.dat = (np.array(strs[k:])).astype(float).reshape( (grd.nlat, grd.nlon)) grd.reset() if (grid is None): return grd else: return bt.ggf.linearInterpolation(grd, grid) else: return None except: print(filename + "'s format is wrong") return None
def transform(sta, dlon=None, dlat=None): slon = np.min(sta.ix[:, 0]) elon = np.max(sta.ix[:, 0]) slat = np.min(sta.ix[:, 1]) elat = np.max(sta.ix[:, 1]) nsta = len(sta.index) if (dlon is None): for i in range(nsta - 1): dlon = sta.ix[i, 0] - sta.ix[i + 1, 0] if dlon != 0: dlon = math.fabs(dlon) break if (dlat is None): for i in range(nsta - 1): dlat = sta.ix[i, 1] - sta.ix[i + 1, 1] if dlat != 0: dlat = math.fabs(dlat) break grd = bd.grid_data(bd.grid(slon, dlon, elon, slat, dlat, elat)) ig = ((sta.ix[:, 0] - grd.slon) // grd.dlon).astype(dtype='int16') jg = ((sta.ix[:, 1] - grd.slat) // grd.dlat).astype(dtype='int16') grd.dat[jg, ig] = sta.ix[:, 2] return grd
def expand_to_contain_another_grid(grd0, grid): grd0.reset() grid.reset() si = 0 sj = 0 ei = 0 ej = 0 if (grid.slon < grd0.grid.slon): si = int(math.ceil((grd0.grid.slon - grid.slon) / grd0.grid.dlon)) if (grid.slat < grd0.grid.slat): sj = int(math.ceil((grd0.grid.slat - grid.slat) / grd0.grid.dlat)) if (grid.elon > grd0.grid.elon): ei = int(math.ceil((-grd0.grid.elon + grid.elon) / grd0.grid.dlon)) if (grid.elat > grd0.grid.elat): ej = int(math.ceil((-grd0.grid.elat + grid.elat) / grd0.grid.dlat)) slon = grd0.grid.slon - si * grd0.grid.dlon slat = grd0.grid.slat - sj * grd0.grid.dlat elon = grd0.grid.elon + ei * grd0.grid.dlon elat = grd0.grid.elat + ej * grd0.grid.dlat grd1 = bd.grid_data( bd.grid(slon, grd0.grid.dlon, elon, slat, grd0.grid.dlat, elat)) grd1.dat[sj:(sj + grd0.grid.nlat), si:(si + grd0.grid.nlon)] = grd0.dat[:, :] return grd1
def explain_awx_bytes(byteArray): sat96 = struct.unpack("12s", byteArray[:12])[0] levl = np.frombuffer(byteArray[12:30], dtype='int16').astype(dtype="int32") formatstr = struct.unpack("8s", byteArray[30:38])[0] qualityflag = struct.unpack("h", byteArray[38:40])[0] satellite = struct.unpack("8s", byteArray[40:48])[0] lev2 = np.frombuffer(byteArray[48:104], dtype='int16').astype(dtype="int32") recordlen = levl[4] headnum = levl[5] datanum = levl[6] timenum = lev2[0:5] nlon = lev2[7] nlat = lev2[8] range = lev2[12:16].astype("float32") slat = range[0] / 100 elat = range[1] / 100 slon = range[2] / 100 elon = range[3] / 100 #nintels=lev2[20:22].astype("float32") dlon = (elon - slon) / (nlon - 1) dlat = (elat - slat) / (nlat - 1) grd = bd.grid_data(bd.grid(slon, dlon, elon, slat, dlat, elat)) colorlen = lev2[24] caliblen = lev2[25] geololen = lev2[26] #print(levl) #print(lev2) head_lenght = headnum * recordlen data_lenght = datanum * recordlen #print(head_lenght + data_lenght) #print( data_lenght) #print(grd.nlon * grd.nlat) #headrest = np.frombuffer(byteArray[:head_lenght], dtype='int8') data_awx = np.frombuffer(byteArray[head_lenght:(head_lenght + data_lenght)], dtype='int8') #print(headrest) if colorlen <= 0: calib = np.frombuffer(byteArray[104:(104 + 2048)], dtype='int16').astype(dtype="float32") else: #color = np.frombuffer(byteArray[104:(104+colorlen*2)], dtype='int16') calib = np.frombuffer( byteArray[(104 + colorlen * 2):(104 + colorlen * 2 + 2048)], dtype='int16').astype(dtype="float32") realcalib = calib / 100.0 realcalib[calib < 0] = (calib[calib < 0] + 65536) / 100.0 awx_index = np.empty(len(data_awx), dtype="int32") awx_index[:] = data_awx[:] awx_index[data_awx < 0] = data_awx[data_awx < 0] + 256 awx_index *= 4 real_data_awx = realcalib[awx_index] grd.dat = real_data_awx.reshape(grd.nlat, grd.nlon) grd.reset() return grd
def sta_to_grid_oa2(sta0, background, sm=1, effect_R=1000, rate_of_model=0): sta = ssf.remove_IV(sta0) sta = ssf.get_sta_in_grid(sta, background.grid) grd = background.copy() ig = ((sta.ix[:, 0] - grd.slon) // grd.dlon).astype(dtype='int16') jg = ((sta.ix[:, 1] - grd.slat) // grd.dlat).astype(dtype='int16') dx = (sta.ix[:, 0] - grd.slon) / grd.dlon - ig dy = (sta.ix[:, 1] - grd.slat) / grd.dlat - jg c00 = (1 - dx) * (1 - dy) c01 = dx * (1 - dy) c10 = (1 - dx) * dy c11 = dx * dy lat = np.arange(grd.nlat) * grd.dlat + grd.slat sr = 1 / np.power(np.cos(lat * math.pi / 180), 4) def targe(x): grdv = x.reshape(grd.nlat, grd.nlon) dx = grdv[:, :-2] + grdv[:, 2:] - 2 * grdv[:, 1:-1] cost1 = np.sum(dx * dx) dy = grdv[:-2, :] + grdv[2:, :] - 2 * grdv[1:-1, :] dy2 = dy * dy sum_dy = np.sum(dy2, axis=1) cost1 = cost1 + np.dot(sum_dy, sr[1:-1]) sta_g = c00 * grd.dat[jg, ig] + c01 * grd.dat[ jg, ig + 1] + c10 * grd.dat[jg + 1, ig] + c11 * grd.dat[jg + 1, ig + 1] error = sta.ix[:, 2] - sta_g cost2 = np.sum(error * error) cost = sm * cost1 + cost2 return cost def grads(x): grdv = x.reshape(grd.nlat, grd.nlon) g1 = np.zeros(grdv.shape) dx = 2 * (grdv[:, :-2] + grdv[:, 2:] - 2 * grdv[:, 1:-1]) g1[:, :-2] = dx g1[:, 2:] += dx g1[:, 1:-1] -= 2 * dx sr_expend = np.tile(sr[1:-1], [grd.nlon, 1]).T dy = 2 * (grdv[:-2, :] + grdv[2:, :] - 2 * grdv[1:-1, :]) dy_sr = dy * sr_expend g1[:-2, :] += dy_sr g1[2:, :] += dy_sr g1[1:-1, :] -= 2 * dy_sr g2 = np.zeros(grdv.shape) sta_g = c00 * grd.dat[jg, ig] + c01 * grd.dat[ jg, ig + 1] + c10 * grd.dat[jg + 1, ig] + c11 * grd.dat[jg + 1, ig + 1] d = 2 * (sta_g - sta.ix[:, 2]) g2[jg, ig] += d * c00 g2[jg, ig + 1] += d * c01 g2[jg + 1, ig] += d * c10 g2[jg + 1, ig + 1] += d * c11 g = sm * g1 + g2 return g.reshape(-1) x = grd.dat.reshape(-1) x_oa = frprmn2(x, targe, grads) grd = bd.grid_data(grd.grid) grd.dat = x_oa.reshape(grd.nlat, grd.nlon) return grd