def prepare_Y(self,mode):
# 1 drought periods
if mode != 'mix':
out_dir = this_root+'random_forest\\{}\\'.format(mode)
analysis.Tools().mk_dir(out_dir)
print 'loading f_recovery_time...'
f_recovery_time = this_root+'arr\\recovery_time\\{}_composite_recovery_time\\composite.npy'.format(mode)
else:
out_dir = this_root + 'random_forest\\{}\\'.format(mode)
analysis.Tools().mk_dir(out_dir)
print 'loading f_recovery_time...'
f_recovery_time = this_root + 'arr\\recovery_time\\composite_3_modes\\composite_3_mode_recovery_time.npy'
recovery_time = dict(np.load(f_recovery_time).item())
print 'done'
Y = {}
flag = 0
for pix in tqdm(recovery_time):
vals = recovery_time[pix]
# print vals
for r_time,mark,date_range,drought_range in vals:
if r_time == None: #r_time 为 TRUE
continue
flag += 1
start = date_range[0]
end = start + r_time
key = pix+'_'+mark+'_'+'{}.{}'.format(start,end)
# print key
Y[key] = r_time
# print flag
# flag=1192218
# flag=198075
np.save(out_dir+'Y',Y)
def insert_per_pix(self):
mode = 'SWE_avg'
# mode = 'SWE_max'
fdir = this_root+'GLOBSWE\\per_pix\\'+mode+'\\'
out_dir = this_root+'GLOBSWE\\per_pix\\'+mode+'_408\\'
analysis.Tools().mk_dir(out_dir)
# spatial_dic = {}
for f in tqdm(os.listdir(fdir)):
dic = dict(np.load(fdir+f).item())
data_dic = {}
for pix in dic:
val = dic[pix]
if val[0] > 0:
val = self.insert_nan_vals(val)
data_dic[pix] = val
# plt.plot(val)
# print len(val)
# # print f
# plt.show()
else:
data_dic[pix] = [-999999]*408
# arr = analysis.DIC_and_TIF().pix_dic_to_spatial_arr(spatial_dic)
# plt.imshow(arr)
# plt.show()
np.save(out_dir+f,data_dic)
def hdf_to_tif(self,data='swe_average'):
# data = 'swe_maximum'
# data = 'swe_average'
# 获取nc空间坐标信息
nc = r'D:\project05\GLOBSWE\nc\198203.nc'
ncin = Dataset(nc, 'r')
x = np.array(ncin['x'])
y = np.array(ncin['y'])
longitude_start, latitude_start, pixelWidth, pixelHeight = x[0], y[0], x[1] - x[0], y[1] - y[0]
hdf_dir = this_root + 'GLOBSWE\\download\\'
out_dir = this_root + 'GLOBSWE\\hdf_to_tif\\' + data + '\\'
analysis.Tools().mk_dir(out_dir, force=True)
for f in os.listdir(hdf_dir):
if f.endswith('.hdf'):
print f
fname = hdf_dir + f
hdf = SD(fname)
# print hdf.datasets()
# swe = np.array(hdf.select(data)['fakeDim2'])
swe = hdf.select(data)[:]
# swe = np.ma.masked_where(swe == -1,swe)
# plt.imshow(swe)
# plt.show()
# attr = hdf.attributes(full=1)
# attNames = attr.keys()
# attNames.sort()
# print attNames
# print attr['Spatial Resolution ']
out_fname = f.split('_')[-2] + '.tif'
to_raster.array2raster_polar(out_dir + out_fname, longitude_start, latitude_start, pixelWidth,
pixelHeight, swe, ndv=-1)
pass
def nc_to_tif(nc,outdir):
# outdir = this_root+'SPEI\\tif\\'
import analysis
analysis.Tools().mk_dir(outdir)
ncin = Dataset(nc, 'r')
lat = ncin['lat'][::-1]
lon = ncin['lon']
pixelWidth = lon[1]-lon[0]
pixelHeight = lat[1]-lat[0]
longitude_start = lon[0]
latitude_start = lat[0]
time = ncin.variables['time']
# print(time)
# exit()
# time_bounds = ncin.variables['time_bounds']
# print(time_bounds)
start = datetime.datetime(1900, 01, 01)
# a = start + datetime.timedelta(days=5459)
# print(a)
# print(len(time_bounds))
# print(len(time))
# for i in time:
# print(i)
# exit()
# nc_dic = {}
flag = 0
valid_year = []
for i in range(1982, 2016):
valid_year.append(str(i))
for i in range(len(time)):
flag += 1
# print(time[i])
date = start + datetime.timedelta(days=int(time[i]))
year = str(date.year)
month = '%02d' % date.month
# day = '%02d'%date.day
date_str = year + month
if not date_str[:4] in valid_year:
continue
# print(date_str)
arr = ncin.variables['tmp'][i][::-1]
arr = np.array(arr)
grid = arr < 99999
arr[np.logical_not(grid)] = -999999
newRasterfn = outdir+date_str+'.tif'
to_raster.array2raster(newRasterfn,longitude_start,latitude_start,pixelWidth,pixelHeight,arr)
def unzip(self):
fdir = this_root + 'GLOBSWE\\download\\'
outdir = this_root + 'GLOBSWE\\nc\\'
analysis.Tools().mk_dir(outdir)
for f in os.listdir(fdir):
if f.endswith('.gz'):
print f
try:
f_gzip = gzip.GzipFile(fdir + f, "rb")
content = f_gzip.read()
with open(outdir + f.split('.')[0] + '.nc', 'wb') as fw:
fw.write(content)
except:
pass
def main():
# n=12
# n='%02d'%n
# nc = this_root+'SPEI\\download_from_web\\spei{}.nc'.format(n)
# out_dir = this_root+'SPEI\\tif\\SPEI_{}\\'.format(n)
# npz = this_root+'SPEI\\npz\\spei_3'
# nc_to_npz(nc,npz)
# run_nc_to_tif()
# download_spei()
# CRU
nc = this_root+'CRU\\cru_ts4.02.1901.2017.tmp.dat.nc\\cru_ts4.02.1901.2017.tmp.dat.nc'
outdir = this_root+'TMP\\tif\\'
analysis.Tools().mk_dir(outdir,force=True)
nc_to_tif(nc,outdir)
def kernel_download(self,params):
year, date = params
# url = 'http://www.globsnow.info/swe/archive_v2.0/'
url = 'http://www.globsnow.info/swe/archive_v2.0/{}/L3B_monthly_SWE/' \
'GlobSnow_SWE_L3B_monthly_{}_v2.0.nc.gz'.format(year, date)
# print url
name = '{}.gz'.format(date)
out_dir = this_root + 'GLOBSWE\\download\\'
f = out_dir + name
analysis.Tools().mk_dir(out_dir, force=True)
# check_zip(f)
# if check_zip(f):
# print f
if not os.path.isfile(f):
print f
print url
self.downloadFILE(url, f)
def cal_monthly_mean(self,fdir,outdir):
# outdir = this_root + 'TMP\\mon_mean_tif\\'
# fdir = this_root + 'TMP\\tif\\'
analysis.Tools().mk_dir(outdir)
for m in tqdm(range(1, 13)):
# if m in range(6,10):
# continue
arrs_sum = 0.
for y in range(1982, 2016):
date = '{}{}'.format(y, '%02d' % m)
tif = fdir + date + '.tif'
if not os.path.isfile(tif):
continue
arr, originX, originY, pixelWidth, pixelHeight = analysis.to_raster.raster2array(tif)
arrs_sum += arr
mean_arr = arrs_sum / len(range(1982, 2016))
mean_arr = np.array(mean_arr, dtype=float)
grid = mean_arr <= 0
mean_arr[grid] = np.nan
analysis.DIC_and_TIF().arr_to_tif(mean_arr, outdir + '%02d.tif' % m)
def nc_to_tif(self,data='SWE_avg'):
# data = 'SWE_avg'
# data = 'SWE_max'
# 可行
ncdir = this_root + 'GLOBSWE\\nc\\'
out_dir = this_root + 'GLOBSWE\\nc_to_tif\\' + data + '\\'
analysis.Tools().mk_dir(out_dir, force=True)
for f in os.listdir(ncdir):
print f
nc = ncdir + f
ncin = Dataset(nc, 'r')
x = np.array(ncin['x'])
y = np.array(ncin['y'])
swe_avg = np.array(ncin[data])
swe_avg = np.array(swe_avg)
longitude_start, latitude_start, pixelWidth, pixelHeight = x[0], y[0], x[1] - x[0], y[1] - y[0]
fname = f.split('.')[0] + '.tif'
to_raster.array2raster_polar(out_dir + fname, longitude_start, latitude_start, pixelWidth, pixelHeight,
swe_avg, ndv=-1)
# plt.imshow(swe_avg)
# plt.show()
# exit()
pass
def cal_monthly_mean(fdir, outdir):
# outdir = this_root + 'TMP\\mon_mean_tif\\'
# fdir = this_root + 'TMP\\tif\\'
analysis.Tools().mk_dir(outdir)
for m in tqdm(range(1, 13)):
arrs = []
for y in range(1982, 2016):
date = '{}{}'.format(y, '%02d' % m)
tif = fdir + date + '.tif'
if not os.path.isfile(tif):
continue
arr, originX, originY, pixelWidth, pixelHeight = analysis.to_raster.raster2array(
tif)
arrs.append(arr)
arrs = np.array(arrs, dtype=float)
mean_arr = []
for i in range(360):
temp = []
for j in range(720):
pix_vals = []
for date in range(len(arrs)):
val = arrs[date][i][j]
if val > 0:
pix_vals.append(val)
if len(pix_vals) > 0:
mean_val = np.mean(pix_vals)
else:
mean_val = np.nan
temp.append(mean_val)
mean_arr.append(temp)
mean_arr = np.array(mean_arr)
analysis.DIC_and_TIF().arr_to_tif(mean_arr, outdir + '%02d.tif' % m)
def data_transform(self):
# 不可并行，内存不足
mode = 'SWE_max'
# mode = 'SWE_avg'
fdir = this_root+'GLOBSWE\\tif\\'+mode+'\\'
outdir = this_root+'GLOBSWE\\per_pix\\'+mode+'\\'
analysis.Tools().mk_dir(outdir,force=True)
# 将空间图转换为数组
# per_pix_data
flist = os.listdir(fdir)
date_list = []
for y in range(1982,2016):
for mon in range(1,13):
if mon in range(5,10):
continue
date_list.append('{}{}'.format(y,'%02d'%mon))
# for i in date_list:
# print i
# exit()
all_array = []
for d in tqdm(date_list, 'loading...'):
for f in flist:
# if not d in f:
# continue
if f.endswith('.tif'):
if f.split('.')[0] == d:
array, originX, originY, pixelWidth, pixelHeight = to_raster.raster2array(fdir + f)
all_array.append(array)
# print len(all_array)
# exit()
row = len(all_array[0])
col = len(all_array[0][0])
void_dic = {}
void_dic_list = []
for r in range(row):
for c in range(col):
void_dic['%03d.%03d' % (r, c)] = []
void_dic_list.append('%03d.%03d' % (r, c))
# print(len(void_dic))
# exit()
for r in tqdm(range(row), 'transforming...'):
for c in range(col):
for arr in all_array:
val = arr[r][c]
void_dic['%03d.%03d' % (r, c)].append(val)
# for i in void_dic_list:
# print(i)
# exit()
flag = 0
temp_dic = {}
for key in tqdm(void_dic_list, 'saving...'):
flag += 1
# print('saving ',flag,'/',len(void_dic)/100000)
temp_dic[key] = void_dic[key]
if flag % 10000 == 0:
# print('\nsaving %02d' % (flag / 10000)+'\n')
np.save(outdir + 'per_pix_dic_%03d' % (flag / 10000), temp_dic)
temp_dic = {}
np.save(outdir + 'per_pix_dic_%03d' % 0, temp_dic)
def nc_to_tif(nc,outdir):
# outdir = this_root+'SPEI\\tif\\'
import analysis
analysis.Tools().mk_dir(outdir,force=1)
ncin = Dataset(nc, 'r')
# print(ncin.variables)
# exit()
lat = ncin['lat']
lon = ncin['lon']
pixelWidth = lon[1]-lon[0]
pixelHeight = lat[1]-lat[0]
longitude_start = lon[0]
latitude_start = lat[0]
time = ncin.variables['time']
# print(time)
# exit()
# time_bounds = ncin.variables['time_bounds']
# print(time_bounds)
start = datetime.datetime(1900, 01, 01)
# a = start + datetime.timedelta(days=5459)
# print(a)
# print(len(time_bounds))
# print(len(time))
# for i in time:
# print(i)
# exit()
# nc_dic = {}
flag = 0
# valid_year = []
# for i in range(1982, 2016):
# valid_year.append(str(i))
# for i in time:
# print(i)
# exit()
for i in range(len(time)):
flag += 1
# print(time[i])
date = start + datetime.timedelta(days=int(time[i]))
year = str(date.year)
month = '%02d' % date.month
# day = '%02d'%date.day
date_str = year + month
# if not date_str[:4] in valid_year:
# continue
# print(date_str)
# exit()
ndv = np.nan
arr = ncin.variables['def'][i]
for name,variable in ncin.variables.items():
for var in variable.ncattrs():
if var == 'missing_value':
ndv = variable.getncattr(var)
if np.isnan(ndv):
raise IOError('no key missing_value')
arr = np.array(arr)
#### mask ####
grid = arr == 32768
# print ndv
arr[grid] = -999999
# arr[grid] = np.nan
# plt.imshow(arr)
# plt.colorbar()
# plt.show()
newRasterfn = outdir+date_str+'.tif'
to_raster.array2raster(newRasterfn,longitude_start,latitude_start,pixelWidth,pixelHeight,arr)
