LabelMap, PolarMap = MapDivide(GCZ_map, 35) mapUS = LabelMap[194:722, 984:2320] #Clip the region of interest labels = np.unique(mapUS)[1:] US_labels = [int(x) for x in labels] for jj in range(14, 20): Li = (LabelMap == jj).astype(np.uint8) rows, cols = np.where(Li == 1) [rl, ru, cl, cr] = [np.min(rows), np.max(rows), np.min(cols), np.max(cols)] print jj, [rl, ru, cl, cr] r17 = np.load( '/ssd-scratch/htranvie/Snow_dataset/filtered_multiparts/oyscacldf4_17.0.npy' ) t0 = date(2000, 2, 24) t_check = date(2003, 7, 6) t_start = t_check - timedelta(days=15) t_end = t_check + timedelta(days=14) ndays = (t_end - t_start).days / 2 + 1 dn_start1 = (t_start - t0).days dn_end1 = (t_end - t0).days oyscacldf4_17 = r17[dn_start1:dn_end1, :, :] oyscacldf4_18 = r18[dn_start1:dn_end1, :, :] oyscacldf4_17i = ClearCloud(oyscacldf4_17, t_start, t_end) oyscacldf4_18i = ClearCloud(oyscacldf4_18, t_start, t_end) oyscacldf4 = np.vstack([oyscacldf4_17i, oyscacldf4_18i]) oyscacldf4 = oyscacldf4[:, 984 - 897:] fvi = gdal.Open('result_US/SnowUS_' + t_check.strftime('%Y-%m-%d') + '.asc').ReadAsArray() oyscacldf4_fvi = fvi[216 - 194:657 - 194 + 1, :1546 - 984 + 1]
else: arr0 = gdal.Open(bin_files[0]).ReadAsArray() oyflood = np.concatenate([oyflood,np.zeros((1,arr_temp.shape[0],arr_temp.shape[1]))],axis=0) oyflood = np.concatenate([oyflood,arr0.reshape(-1,arr_temp.shape[0],arr_temp.shape[1])],axis=0) else: for bin_file in bin_files: arr1 = gdal.Open(bin_file).ReadAsArray() oyflood = np.concatenate([oyflood,arr1.reshape(-1,arr_temp.shape[0],arr_temp.shape[1])],axis=0) oyflood[oyflood==1]=2 oyflood[oyflood==-1]=1 scipy.io.savemat('/ssd-scratch/htranvie/Flood/data/oyflood',mdict={'oyflood':oyflood}) oyscacld = scipy.io.loadmat('/ssd-scratch/htranvie/Flood/data/oyflood.mat')['oyflood'] oyscacld1 = ndimage.zoom(oyscacld,(1,4,4),order=0) sca = ClearCloud(oyscacld1,date(2008,6,5),date(2008,6,5)+timedelta(days=29)) scipy.io.savemat('/ssd-scratch/htranvie/Flood/data/sca_py3',mdict={'sca_py3':sca}) sca = scipy.io.loadmat('/ssd-scratch/htranvie/Flood/data/sca_py2.mat')['sca_py2'] #for matlab sca = scipy.io.loadmat('/ssd-scratch/htranvie/Flood/data/sca2.mat')['sca2'] sca = np.swapaxes(np.swapaxes(sca,0,2),1,2) #Draw report t0 = date(2008,6,5) fig=plt.figure() m1 = 4 n = 6 for i in range(n*m1,n*m1+n): t = t0+timedelta(days=i/2) ax=fig.add_subplot(2,n,i-n*m1+1)
def BootstrapTest(oyscacld, ntimes): cld = scipy.io.loadmat('cloud_lib.mat')['cld'] cldpct = scipy.io.loadmat('cloud_lib.mat')['cldpct'] [nday, nr, nc] = oyscacld.shape pod = np.array([]).reshape(0, 4).astype(np.float32) far = np.array([]).reshape(0, 4).astype(np.float32) csi = np.array([]).reshape(0, 4).astype(np.float32) for jk in range(ntimes): lowcldday = np.zeros((nday, nr, nc)).astype(np.uint8) medcldday = np.zeros((nday, nr, nc)).astype(np.uint8) highcldday = np.zeros((nday, nr, nc)).astype(np.uint8) sevcldday = np.zeros((nday, nr, nc)).astype(np.uint8) for i in range(nday): ##Low cloud hindrance pctls = np.where((cldpct > 0.03) & (cldpct <= 0.05))[1] iii = np.random.randint(len(pctls), size=1)[0] lowcld = cld[:, :, pctls[iii]] lowcld1 = oyscacld[i, :, :] * 2 + lowcld lowcld1[lowcld1 == 3] = 1 lowcldday[i, :, :] = lowcld1 ##Med cloud hindrance pctls = np.where((cldpct > 0.07) & (cldpct <= 0.09))[1] iii = np.random.randint(len(pctls), size=1)[0] medcld = cld[:, :, pctls[iii]] medcld1 = oyscacld[i, :, :] * 2 + medcld medcld1[medcld1 == 3] = 1 medcldday[i, :, :] = medcld1 ##High cloud hindrance pctls = np.where((cldpct > 0.15) & (cldpct <= 0.2))[1] iii = np.random.randint(len(pctls), size=1)[0] highcld = cld[:, :, pctls[iii]] highcld1 = oyscacld[i, :, :] * 2 + highcld highcld1[highcld1 == 3] = 1 highcldday[i, :, :] = highcld1 ##Severe cloud hindrance pctls = np.where((cldpct > 0.2) & (cldpct <= 0.4))[1] iii = np.random.randint(len(pctls), size=1)[0] sevcld = cld[:, :, pctls[iii]] sevcld1 = oyscacld[i, :, :] * 2 + sevcld sevcld1[sevcld1 == 3] = 1 sevcldday[i, :, :] = sevcld1 sca1 = ClearCloud(lowcldday, date(2009, 9, 9), date(2009, 9, 10), date(2009, 9, 15)) sca2 = ClearCloud(medcldday, date(2009, 9, 9), date(2009, 9, 10), date(2009, 9, 15)) sca3 = ClearCloud(highcldday, date(2009, 9, 9), date(2009, 9, 10), date(2009, 9, 15)) sca4 = ClearCloud(sevcldday, date(2009, 9, 9), date(2009, 9, 10), date(2009, 9, 15)) lowhit = np.sum( np.logical_and(oyscacld[2, :, :] == 1, sca1[2, :, :] == 1)) lowmiss = np.sum( np.logical_and(oyscacld[2, :, :] == 1, sca1[2, :, :] == 0)) lowfalse = np.sum( np.logical_and(oyscacld[2, :, :] == 0, sca1[2, :, :] == 1)) medhit = np.sum( np.logical_and(oyscacld[2, :, :] == 1, sca2[2, :, :] == 1)) medmiss = np.sum( np.logical_and(oyscacld[2, :, :] == 1, sca2[2, :, :] == 0)) medfalse = np.sum( np.logical_and(oyscacld[2, :, :] == 0, sca2[2, :, :] == 1)) highhit = np.sum( np.logical_and(oyscacld[2, :, :] == 1, sca3[2, :, :] == 1)) highmiss = np.sum( np.logical_and(oyscacld[2, :, :] == 1, sca3[2, :, :] == 0)) highfalse = np.sum( np.logical_and(oyscacld[2, :, :] == 0, sca3[2, :, :] == 1)) sevhit = np.sum( np.logical_and(oyscacld[2, :, :] == 1, sca4[2, :, :] == 1)) sevmiss = np.sum( np.logical_and(oyscacld[2, :, :] == 1, sca4[2, :, :] == 0)) sevfalse = np.sum( np.logical_and(oyscacld[2, :, :] == 0, sca4[2, :, :] == 1)) pod = np.vstack([ pod, np.array([ lowhit / float(lowhit + lowmiss), medhit / float(medhit + medmiss), highhit / float(highhit + highmiss), sevhit / float(sevhit + sevmiss) ]) ]) far = np.vstack([ far, np.array([ lowfalse / float(lowhit + lowfalse), medfalse / float(medhit + medfalse), highfalse / float(highhit + highfalse), sevfalse / float(sevhit + sevfalse) ]) ]) csi = np.vstack([ csi, np.array([ lowhit / float(lowhit + lowmiss + lowfalse), medhit / float(medhit + medmiss + medfalse), highhit / float(highhit + highmiss + highfalse), sevhit / float(sevhit + sevmiss + sevfalse) ]) ]) return pod, far, csi
import scipy.ndimage d0 = date(2013, 6, 7) for i in [0, 48]: #for i in [377,409,425,441]: d1 = d0 + timedelta(days=i) # if os.path.isfile('Cloud_free/MOD_CF_'+d1.strftime('%Y%m%d')+'.npy'): # continue selectedDays = [(d1+timedelta(days=-x-1)).strftime('%Y%m%d') for x in range(7)][::-1]+\ [(d1+timedelta(days=x)).strftime('%Y%m%d') for x in range(8)] refl_in = np.array([]).reshape(0, 1494, 1836) count = 0 for j, day in enumerate(selectedDays): if os.path.isfile('results/MOD_' + day + '_bin.tif'): arr = io.imread('results/MOD_' + day + '_bin.tif') else: count += 1 arr = np.ones((1494, 1836)) * -1 refl_in = np.vstack([arr[np.newaxis, :, :], refl_in]) if count > 7: continue refl_in[refl_in == 1] = 2 refl_in[refl_in == -1] = 1 #L = scipy.ndimage.zoom(refl_in,(1,2,2),order=1) refl_vi = ClearCloud(refl_in, d1 + timedelta(days=-7), d1 + timedelta(days=7)) plt.clf() plt.imshow(np.ma.masked_where(refl_vi == -99, refl_vi)) plt.savefig('Cloud_free/img/h' + d1.strftime('%Y%m%d') + '.png') np.save('Cloud_free/MOD_CF_' + d1.strftime('%Y%m%d'), refl_vi)
def CloudRemoval(yyyy1, mm1, dd1, labeli): t0 = date(2000, 2, 24) t_start = date(yyyy1, mm1, dd1) t_end = t_start + timedelta(days=29) ndays = (t_end - t_start).days / 2 + 1 if not os.path.isdir('/ssd-scratch/htranvie/Snow_dataset/results/' + str(labeli)): os.mkdir('/ssd-scratch/htranvie/Snow_dataset/results/' + str(labeli)) print(t_start + timedelta(days=ndays)).strftime('%Y-%m-%d') if os.path.isfile('/ssd-scratch/htranvie/Snow_dataset/results/' + str(labeli) + '/result_' + str(labeli) + '_' + (t_start + timedelta(days=ndays)).strftime('%Y-%m-%d') + '.npy'): return GCZ_map, classes = GetClimateZone() LabelMap, PolarMap = MapDivide(GCZ_map, 35) labels = np.unique(LabelMap)[1:] Li = (LabelMap == labeli).astype(np.uint8) rows, cols = np.where(Li == 1) [rl, ru, cl, cu] = [np.min(rows), np.max(rows), np.min(cols), np.max(cols)] Li_mini = Li[rl:ru + 1, cl:cu + 1] Polar_mini = PolarMap[rl:ru + 1, cl:cu + 1] dn_start1 = (t_start - t0).days dn_end1 = (t_end - t0).days oyscacldf4i = np.load( '/ssd-scratch/htranvie/Snow_dataset/merged_multiparts/oyscacldf4_' + str(labeli) + '.npy')[dn_start1:dn_end1, :, :] Polar_add = np.logical_and(PolarMap == 1, LabelMap == 0)[rl:ru + 1, cl:cu + 1].astype(np.uint8) start_tot = time.time() start = time.time() oyscacldfi = ClearCloud(oyscacldf4i, t_start, t_end) if oyscacldfi == 'need to divide array': oyscacldfi_1 = ClearCloud(oyscacldf4i[:, :, :(cu + 1 - cl) / 2], t_start, t_end) if oyscacldfi_1 == 'need to divide array': oyscacldfi_10 = ClearCloud(oyscacldf4i[:, :, :(cu + 1 - cl) / 4], t_start, t_end) if oyscacldfi_10 == 'need to divide array': oyscacldfi_100 = ClearCloud( oyscacldf4i[:, :(ru + 1 - rl) / 2, :(cu + 1 - cl) / 4], t_start, t_end) oyscacldfi_101 = ClearCloud( oyscacldf4i[:, (ru + 1 - rl) / 2:, :(cu + 1 - cl) / 4], t_start, t_end) oyscacldfi_10 = np.vstack([oyscacldfi_100, oyscacldfi_101]) oyscacldfi_11 = ClearCloud( oyscacldf4i[:, :, (cu + 1 - cl) / 4:(cu + 1 - cl) / 2], t_start, t_end) if oyscacldfi_11 == 'need to divide array': oyscacldfi_110 = ClearCloud( oyscacldf4i[:, :(ru + 1 - rl) / 2, (cu + 1 - cl) / 4:(cu + 1 - cl) / 2], t_start, t_end) oyscacldfi_111 = ClearCloud( oyscacldf4i[:, (ru + 1 - rl) / 2:, (cu + 1 - cl) / 4:(cu + 1 - cl) / 2], t_start, t_end) oyscacldfi_11 = np.vstack([oyscacldfi_110, oyscacldfi_111]) oyscacldfi_1 = np.hstack([oyscacldfi_10, oyscacldfi_11]) oyscacldfi_2 = ClearCloud(oyscacldf4i[:, :, (cu + 1 - cl) / 2:], t_start, t_end) if oyscacldfi_2 == 'need to divide array': oyscacldfi_20 = ClearCloud( oyscacldf4i[:, :, (cu + 1 - cl) / 2:3 * (cu + 1 - cl) / 4], t_start, t_end) if oyscacldfi_20 == 'need to divide array': oyscacldfi_200 = ClearCloud( oyscacldf4i[:, :(ru + 1 - rl) / 2, (cu + 1 - cl) / 2:3 * (cu + 1 - cl) / 4], t_start, t_end) oyscacldfi_201 = ClearCloud( oyscacldf4i[:, (ru + 1 - rl) / 2:, (cu + 1 - cl) / 2:3 * (cu + 1 - cl) / 4], t_start, t_end) oyscacldfi_20 = np.vstack([oyscacldfi_200, oyscacldfi_201]) oyscacldfi_21 = ClearCloud( oyscacldf4i[:, :, 3 * (cu + 1 - cl) / 4:], t_start, t_end) if oyscacldfi_21 == 'need to divide array': oyscacldfi_210 = ClearCloud( oyscacldf4i[:, :(ru + 1 - rl) / 2, (cu + 1 - cl) / 2:3 * (cu + 1 - cl) / 4:], t_start, t_end) oyscacldfi_211 = ClearCloud( oyscacldf4i[:, (ru + 1 - rl) / 2:, 3 * (cu + 1 - cl) / 4:], t_start, t_end) oyscacldfi_21 = np.vstack([oyscacldfi_210, oyscacldfi_211]) oyscacldfi_2 = np.hstack([oyscacldfi_20, oyscacldfi_21]) oyscacldfi = np.hstack([oyscacldfi_1, oyscacldfi_2]) V = np.logical_and(oyscacldfi == 0, Polar_mini == 1) oyscacldfi[V] = 1 end6 = time.time() print 'VI process ' + str(end6 - start) + ' s' oyscacldfi = oyscacldfi + Polar_add end_tot = time.time() print 'Total time ' + str(end_tot - start_tot) + ' s' np.save( '/ssd-scratch/htranvie/Snow_dataset/merged_multiparts/26/result_' + str(labeli) + '_' + (t_start + timedelta(days=ndays)).strftime('%Y-%m-%d'), oyscacldfi)
#dem_arr = io.imread('elevation/dem_new.tif') oyscacld = np.array([]).reshape(0, 2139, 1499) miss = 0 count = 0 for i in range(ndays): for pre in ['MOD', 'MYD']: t = (t_start + timedelta(days=i)).strftime('%Y%m%d') res_files = glob('results/' + pre + '_' + t + '_bindem.tif') if not res_files: miss += 1 continue elif len(res_files) == 1: res_file = res_files[0] else: res_file = [x for x in res_files if 'new' in x][0] mod_arr = gdal.Open(res_file).ReadAsArray() mod_arr[mod_arr == 1] = 2 mod_arr[mod_arr == -1] = 1 mod_arr[river_area == -99] = -99 oyscacld = np.vstack( [oyscacld, mod_arr[np.newaxis, 1272:, 1900:3399]]) count += 1 #L = scipy.ndimage.zoom(oyscacld,(1,2,2),order=0) #high_lakes = np.logical_and(oyscacld==2,dem_arr>=300) #oyscacld[high_lakes] = 0 refl_vi = ClearCloud(oyscacld) #refl_vi[high_lakes[ndays,:,:]] = 1 scipy.io.savemat('Cloud_free/refl_vi' + t_mid.strftime('%Y%m%d') + '.mat', mdict={'refl_vi' + t_mid.strftime('%Y%m%d'): refl_vi})
def CloudRemoval(yyyy0, mm0, dd0, yyyy1, mm1, dd1, yyyy2, mm2, dd2): t0 = date(yyyy0, mm0, dd0) t_start = date(yyyy1, mm1, dd1) t_end = date(yyyy2, mm2, dd2) ndays = (t_end - t_start).days / 2 + 1 dn_start1 = (t_start - t0).days dn_end1 = (t_end - t0).days + 1 GCZ_map, classes = GetClimateZone() LabelMap, PolarMap = MapDivide(GCZ_map, 35) labels = np.unique(LabelMap)[1:] oyscacldf4 = np.load('data/after_filter_global101516031517.npy')[ dn_start1:dn_end1, :, :] oyscacldf1 = np.load('data/merged_global101516031517.npy')[ dn_start1:dn_end1, :, :] n, nr, nc = oyscacldf1.shape arr1 = np.zeros((nr, nc)) Polar_add = np.logical_and(PolarMap == 1, LabelMap == 0).astype(np.uint8) start_tot = time.time() for i in labels: print str(i) Li = (LabelMap == i).astype(np.uint8) rows, cols = np.where(Li == 1) [rl, ru, cl, cu] = [np.min(rows), np.max(rows), np.min(cols), np.max(cols)] Li_mini = Li[rl:ru + 1, cl:cu + 1] Polar_mini = PolarMap[rl:ru + 1, cl:cu + 1] oyscacldf4i = oyscacldf4[:, rl:ru + 1, cl:cu + 1] start = time.time() oyscacldfi = ClearCloud(oyscacldf4i, t_start, t_end) if oyscacldfi == 'need to divide array': oyscacldfi_1 = ClearCloud(oyscacldf4i[:, :, :(cu + 1 - cl) / 2], t_start, t_end) if oyscacldfi_1 == 'need to divide array': oyscacldfi_10 = ClearCloud( oyscacldf4i[:, :, :(cu + 1 - cl) / 4], t_start, t_end) if oyscacldfi_10 == 'need to divide array': oyscacldfi_100 = ClearCloud( oyscacldf4i[:, :(ru + 1 - rl) / 2, :(cu + 1 - cl) / 4], t_start, t_end) oyscacldfi_101 = ClearCloud( oyscacldf4i[:, (ru + 1 - rl) / 2:, :(cu + 1 - cl) / 4], t_start, t_end) oyscacldfi_10 = np.vstack([oyscacldfi_100, oyscacldfi_101]) oyscacldfi_11 = ClearCloud( oyscacldf4i[:, :, (cu + 1 - cl) / 4:(cu + 1 - cl) / 2], t_start, t_end) if oyscacldfi_11 == 'need to divide array': oyscacldfi_110 = ClearCloud( oyscacldf4i[:, :(ru + 1 - rl) / 2, (cu + 1 - cl) / 4:(cu + 1 - cl) / 2], t_start, t_end) oyscacldfi_111 = ClearCloud( oyscacldf4i[:, (ru + 1 - rl) / 2:, (cu + 1 - cl) / 4:(cu + 1 - cl) / 2], t_start, t_end) oyscacldfi_11 = np.vstack([oyscacldfi_110, oyscacldfi_111]) oyscacldfi_1 = np.hstack([oyscacldfi_10, oyscacldfi_11]) oyscacldfi_2 = ClearCloud(oyscacldf4i[:, :, (cu + 1 - cl) / 2:], t_start, t_end) if oyscacldfi_2 == 'need to divide array': oyscacldfi_20 = ClearCloud( oyscacldf4i[:, :, (cu + 1 - cl) / 2:3 * (cu + 1 - cl) / 4], t_start, t_end) if oyscacldfi_20 == 'need to divide array': oyscacldfi_200 = ClearCloud( oyscacldf4i[:, :(ru + 1 - rl) / 2, (cu + 1 - cl) / 2:3 * (cu + 1 - cl) / 4], t_start, t_end) oyscacldfi_201 = ClearCloud( oyscacldf4i[:, (ru + 1 - rl) / 2:, (cu + 1 - cl) / 2:3 * (cu + 1 - cl) / 4], t_start, t_end) oyscacldfi_20 = np.vstack([oyscacldfi_200, oyscacldfi_201]) oyscacldfi_21 = ClearCloud( oyscacldf4i[:, :, 3 * (cu + 1 - cl) / 4:], t_start, t_end) if oyscacldfi_21 == 'need to divide array': oyscacldfi_210 = ClearCloud( oyscacldf4i[:, :(ru + 1 - rl) / 2, (cu + 1 - cl) / 2:3 * (cu + 1 - cl) / 4:], t_start, t_end) oyscacldfi_211 = ClearCloud( oyscacldf4i[:, (ru + 1 - rl) / 2:, 3 * (cu + 1 - cl) / 4:], t_start, t_end) oyscacldfi_21 = np.vstack([oyscacldfi_210, oyscacldfi_211]) oyscacldfi_2 = np.hstack([oyscacldfi_20, oyscacldfi_21]) oyscacldfi = np.hstack([oyscacldfi_1, oyscacldfi_2]) V = np.logical_and( np.logical_and(oyscacldf1[ndays, rl:ru + 1, cl:cu + 1] == 1, oyscacldfi == 0), Polar_mini == 1) oyscacldfi[V] = 1 end6 = time.time() print 'VI process ' + str(end6 - start) + ' s' arr1[Li == 1] = oyscacldfi[Li_mini == 1] f = plt.figure() plt.imshow(arr1) f.savefig('h' + (t_start + timedelta(days=ndays)).strftime('%Y-%m-%d') + '.png') arr_tot = arr1 + Polar_add arr_tot = arr_tot.astype(np.uint8) end_tot = time.time() print 'Total time ' + str(end_tot - start_tot) + ' s' plotWhole( arr_tot, 'final_' + (t_start + timedelta(days=ndays)).strftime('%Y-%m-%d')) np.save( 'data/result_' + (t_start + timedelta(days=ndays)).strftime('%Y-%m-%d'), arr_tot)