def prepare_Y(self,mode): # 1 drought periods if mode != 'mix': out_dir = this_root+'random_forest\\{}\\'.format(mode) analysis.Tools().mk_dir(out_dir) print 'loading f_recovery_time...' f_recovery_time = this_root+'arr\\recovery_time\\{}_composite_recovery_time\\composite.npy'.format(mode) else: out_dir = this_root + 'random_forest\\{}\\'.format(mode) analysis.Tools().mk_dir(out_dir) print 'loading f_recovery_time...' f_recovery_time = this_root + 'arr\\recovery_time\\composite_3_modes\\composite_3_mode_recovery_time.npy' recovery_time = dict(np.load(f_recovery_time).item()) print 'done' Y = {} flag = 0 for pix in tqdm(recovery_time): vals = recovery_time[pix] # print vals for r_time,mark,date_range,drought_range in vals: if r_time == None: #r_time 为 TRUE continue flag += 1 start = date_range[0] end = start + r_time key = pix+'_'+mark+'_'+'{}.{}'.format(start,end) # print key Y[key] = r_time # print flag # flag=1192218 # flag=198075 np.save(out_dir+'Y',Y)
def insert_per_pix(self): mode = 'SWE_avg' # mode = 'SWE_max' fdir = this_root+'GLOBSWE\\per_pix\\'+mode+'\\' out_dir = this_root+'GLOBSWE\\per_pix\\'+mode+'_408\\' analysis.Tools().mk_dir(out_dir) # spatial_dic = {} for f in tqdm(os.listdir(fdir)): dic = dict(np.load(fdir+f).item()) data_dic = {} for pix in dic: val = dic[pix] if val[0] > 0: val = self.insert_nan_vals(val) data_dic[pix] = val # plt.plot(val) # print len(val) # # print f # plt.show() else: data_dic[pix] = [-999999]*408 # arr = analysis.DIC_and_TIF().pix_dic_to_spatial_arr(spatial_dic) # plt.imshow(arr) # plt.show() np.save(out_dir+f,data_dic)
def hdf_to_tif(self,data='swe_average'): # data = 'swe_maximum' # data = 'swe_average' # 获取nc空间坐标信息 nc = r'D:\project05\GLOBSWE\nc\198203.nc' ncin = Dataset(nc, 'r') x = np.array(ncin['x']) y = np.array(ncin['y']) longitude_start, latitude_start, pixelWidth, pixelHeight = x[0], y[0], x[1] - x[0], y[1] - y[0] hdf_dir = this_root + 'GLOBSWE\\download\\' out_dir = this_root + 'GLOBSWE\\hdf_to_tif\\' + data + '\\' analysis.Tools().mk_dir(out_dir, force=True) for f in os.listdir(hdf_dir): if f.endswith('.hdf'): print f fname = hdf_dir + f hdf = SD(fname) # print hdf.datasets() # swe = np.array(hdf.select(data)['fakeDim2']) swe = hdf.select(data)[:] # swe = np.ma.masked_where(swe == -1,swe) # plt.imshow(swe) # plt.show() # attr = hdf.attributes(full=1) # attNames = attr.keys() # attNames.sort() # print attNames # print attr['Spatial Resolution '] out_fname = f.split('_')[-2] + '.tif' to_raster.array2raster_polar(out_dir + out_fname, longitude_start, latitude_start, pixelWidth, pixelHeight, swe, ndv=-1) pass
def nc_to_tif(nc,outdir): # outdir = this_root+'SPEI\\tif\\' import analysis analysis.Tools().mk_dir(outdir) ncin = Dataset(nc, 'r') lat = ncin['lat'][::-1] lon = ncin['lon'] pixelWidth = lon[1]-lon[0] pixelHeight = lat[1]-lat[0] longitude_start = lon[0] latitude_start = lat[0] time = ncin.variables['time'] # print(time) # exit() # time_bounds = ncin.variables['time_bounds'] # print(time_bounds) start = datetime.datetime(1900, 01, 01) # a = start + datetime.timedelta(days=5459) # print(a) # print(len(time_bounds)) # print(len(time)) # for i in time: # print(i) # exit() # nc_dic = {} flag = 0 valid_year = [] for i in range(1982, 2016): valid_year.append(str(i)) for i in range(len(time)): flag += 1 # print(time[i]) date = start + datetime.timedelta(days=int(time[i])) year = str(date.year) month = '%02d' % date.month # day = '%02d'%date.day date_str = year + month if not date_str[:4] in valid_year: continue # print(date_str) arr = ncin.variables['tmp'][i][::-1] arr = np.array(arr) grid = arr < 99999 arr[np.logical_not(grid)] = -999999 newRasterfn = outdir+date_str+'.tif' to_raster.array2raster(newRasterfn,longitude_start,latitude_start,pixelWidth,pixelHeight,arr)
def unzip(self): fdir = this_root + 'GLOBSWE\\download\\' outdir = this_root + 'GLOBSWE\\nc\\' analysis.Tools().mk_dir(outdir) for f in os.listdir(fdir): if f.endswith('.gz'): print f try: f_gzip = gzip.GzipFile(fdir + f, "rb") content = f_gzip.read() with open(outdir + f.split('.')[0] + '.nc', 'wb') as fw: fw.write(content) except: pass
def main(): # n=12 # n='%02d'%n # nc = this_root+'SPEI\\download_from_web\\spei{}.nc'.format(n) # out_dir = this_root+'SPEI\\tif\\SPEI_{}\\'.format(n) # npz = this_root+'SPEI\\npz\\spei_3' # nc_to_npz(nc,npz) # run_nc_to_tif() # download_spei() # CRU nc = this_root+'CRU\\cru_ts4.02.1901.2017.tmp.dat.nc\\cru_ts4.02.1901.2017.tmp.dat.nc' outdir = this_root+'TMP\\tif\\' analysis.Tools().mk_dir(outdir,force=True) nc_to_tif(nc,outdir)
def kernel_download(self,params): year, date = params # url = 'http://www.globsnow.info/swe/archive_v2.0/' url = 'http://www.globsnow.info/swe/archive_v2.0/{}/L3B_monthly_SWE/' \ 'GlobSnow_SWE_L3B_monthly_{}_v2.0.nc.gz'.format(year, date) # print url name = '{}.gz'.format(date) out_dir = this_root + 'GLOBSWE\\download\\' f = out_dir + name analysis.Tools().mk_dir(out_dir, force=True) # check_zip(f) # if check_zip(f): # print f if not os.path.isfile(f): print f print url self.downloadFILE(url, f)
def cal_monthly_mean(self,fdir,outdir): # outdir = this_root + 'TMP\\mon_mean_tif\\' # fdir = this_root + 'TMP\\tif\\' analysis.Tools().mk_dir(outdir) for m in tqdm(range(1, 13)): # if m in range(6,10): # continue arrs_sum = 0. for y in range(1982, 2016): date = '{}{}'.format(y, '%02d' % m) tif = fdir + date + '.tif' if not os.path.isfile(tif): continue arr, originX, originY, pixelWidth, pixelHeight = analysis.to_raster.raster2array(tif) arrs_sum += arr mean_arr = arrs_sum / len(range(1982, 2016)) mean_arr = np.array(mean_arr, dtype=float) grid = mean_arr <= 0 mean_arr[grid] = np.nan analysis.DIC_and_TIF().arr_to_tif(mean_arr, outdir + '%02d.tif' % m)
def nc_to_tif(self,data='SWE_avg'): # data = 'SWE_avg' # data = 'SWE_max' # 可行 ncdir = this_root + 'GLOBSWE\\nc\\' out_dir = this_root + 'GLOBSWE\\nc_to_tif\\' + data + '\\' analysis.Tools().mk_dir(out_dir, force=True) for f in os.listdir(ncdir): print f nc = ncdir + f ncin = Dataset(nc, 'r') x = np.array(ncin['x']) y = np.array(ncin['y']) swe_avg = np.array(ncin[data]) swe_avg = np.array(swe_avg) longitude_start, latitude_start, pixelWidth, pixelHeight = x[0], y[0], x[1] - x[0], y[1] - y[0] fname = f.split('.')[0] + '.tif' to_raster.array2raster_polar(out_dir + fname, longitude_start, latitude_start, pixelWidth, pixelHeight, swe_avg, ndv=-1) # plt.imshow(swe_avg) # plt.show() # exit() pass
def cal_monthly_mean(fdir, outdir): # outdir = this_root + 'TMP\\mon_mean_tif\\' # fdir = this_root + 'TMP\\tif\\' analysis.Tools().mk_dir(outdir) for m in tqdm(range(1, 13)): arrs = [] for y in range(1982, 2016): date = '{}{}'.format(y, '%02d' % m) tif = fdir + date + '.tif' if not os.path.isfile(tif): continue arr, originX, originY, pixelWidth, pixelHeight = analysis.to_raster.raster2array( tif) arrs.append(arr) arrs = np.array(arrs, dtype=float) mean_arr = [] for i in range(360): temp = [] for j in range(720): pix_vals = [] for date in range(len(arrs)): val = arrs[date][i][j] if val > 0: pix_vals.append(val) if len(pix_vals) > 0: mean_val = np.mean(pix_vals) else: mean_val = np.nan temp.append(mean_val) mean_arr.append(temp) mean_arr = np.array(mean_arr) analysis.DIC_and_TIF().arr_to_tif(mean_arr, outdir + '%02d.tif' % m)
def data_transform(self): # 不可并行,内存不足 mode = 'SWE_max' # mode = 'SWE_avg' fdir = this_root+'GLOBSWE\\tif\\'+mode+'\\' outdir = this_root+'GLOBSWE\\per_pix\\'+mode+'\\' analysis.Tools().mk_dir(outdir,force=True) # 将空间图转换为数组 # per_pix_data flist = os.listdir(fdir) date_list = [] for y in range(1982,2016): for mon in range(1,13): if mon in range(5,10): continue date_list.append('{}{}'.format(y,'%02d'%mon)) # for i in date_list: # print i # exit() all_array = [] for d in tqdm(date_list, 'loading...'): for f in flist: # if not d in f: # continue if f.endswith('.tif'): if f.split('.')[0] == d: array, originX, originY, pixelWidth, pixelHeight = to_raster.raster2array(fdir + f) all_array.append(array) # print len(all_array) # exit() row = len(all_array[0]) col = len(all_array[0][0]) void_dic = {} void_dic_list = [] for r in range(row): for c in range(col): void_dic['%03d.%03d' % (r, c)] = [] void_dic_list.append('%03d.%03d' % (r, c)) # print(len(void_dic)) # exit() for r in tqdm(range(row), 'transforming...'): for c in range(col): for arr in all_array: val = arr[r][c] void_dic['%03d.%03d' % (r, c)].append(val) # for i in void_dic_list: # print(i) # exit() flag = 0 temp_dic = {} for key in tqdm(void_dic_list, 'saving...'): flag += 1 # print('saving ',flag,'/',len(void_dic)/100000) temp_dic[key] = void_dic[key] if flag % 10000 == 0: # print('\nsaving %02d' % (flag / 10000)+'\n') np.save(outdir + 'per_pix_dic_%03d' % (flag / 10000), temp_dic) temp_dic = {} np.save(outdir + 'per_pix_dic_%03d' % 0, temp_dic)
def nc_to_tif(nc,outdir): # outdir = this_root+'SPEI\\tif\\' import analysis analysis.Tools().mk_dir(outdir,force=1) ncin = Dataset(nc, 'r') # print(ncin.variables) # exit() lat = ncin['lat'] lon = ncin['lon'] pixelWidth = lon[1]-lon[0] pixelHeight = lat[1]-lat[0] longitude_start = lon[0] latitude_start = lat[0] time = ncin.variables['time'] # print(time) # exit() # time_bounds = ncin.variables['time_bounds'] # print(time_bounds) start = datetime.datetime(1900, 01, 01) # a = start + datetime.timedelta(days=5459) # print(a) # print(len(time_bounds)) # print(len(time)) # for i in time: # print(i) # exit() # nc_dic = {} flag = 0 # valid_year = [] # for i in range(1982, 2016): # valid_year.append(str(i)) # for i in time: # print(i) # exit() for i in range(len(time)): flag += 1 # print(time[i]) date = start + datetime.timedelta(days=int(time[i])) year = str(date.year) month = '%02d' % date.month # day = '%02d'%date.day date_str = year + month # if not date_str[:4] in valid_year: # continue # print(date_str) # exit() ndv = np.nan arr = ncin.variables['def'][i] for name,variable in ncin.variables.items(): for var in variable.ncattrs(): if var == 'missing_value': ndv = variable.getncattr(var) if np.isnan(ndv): raise IOError('no key missing_value') arr = np.array(arr) #### mask #### grid = arr == 32768 # print ndv arr[grid] = -999999 # arr[grid] = np.nan # plt.imshow(arr) # plt.colorbar() # plt.show() newRasterfn = outdir+date_str+'.tif' to_raster.array2raster(newRasterfn,longitude_start,latitude_start,pixelWidth,pixelHeight,arr)