def __init__(self, start_date=date(1990, 1, 1), end_date=date(1990, 12, 31), regin="AUS", lr_transform=None, hr_transform=None, shuffle=True, args=None): print("=> BARRA_R & ACCESS_S1 loading") print("=> from " + start_date.strftime("%Y/%m/%d") + " to " + end_date.strftime("%Y/%m/%d") + "") self.file_BARRA_dir = args.file_BARRA_dir self.file_ACCESS_dir = args.file_ACCESS_dir self.args = args self.lr_transform = lr_transform self.hr_transform = hr_transform self.start_date = start_date self.end_date = end_date self.regin = regin self.leading_time_we_use = args.leading_time_we_use self.ensemble_access = [ 'e01', 'e02', 'e03', 'e04', 'e05', 'e06', 'e07', 'e08', 'e09', 'e10', 'e11' ] self.ensemble = [] for i in range(args.ensemble): self.ensemble.append(self.ensemble_access[i]) self.dates = self.date_range(start_date, end_date) self.filename_list = self.get_filename_with_time_order( args.file_ACCESS_dir + "pr/daily/") if not os.path.exists(args.file_ACCESS_dir + "pr/daily/"): print(args.file_ACCESS_dir + "pr/daily/") print("no file or no permission") _, _, date_for_BARRA, time_leading = self.filename_list[0] if shuffle: random.shuffle(self.filename_list) if not os.path.exists( "/g/data/ma05/BARRA_R/v1/forecast/spec/accum_prcp/1990/01/accum_prcp-fc-spec-PT1H-BARRA_R-v1-19900109T0600Z.sub.nc" ): print(self.file_BARRA_dir) print("no file or no permission!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") data_high = dpt.read_barra_data_fc_get_lat_lon(self.file_BARRA_dir, date_for_BARRA) self.lat = data_high[1] self.lon = data_high[1] self.shape = (79, 94) if self.args.dem: data_dem = dpt.add_lat_lon( dpt.read_dem(args.file_DEM_dir + "dem-9s1.tif")) self.dem_data = dpt.interp_tensor_2d( dpt.map_aust_old(data_dem, xrarray=False), self.shape)
def __getitem__(self, idx): ''' from filename idx get id return lr,hr ''' #read_data filemame[idx] access_filename_pr, date_for_BARRA, time_leading = self.filename_list[ idx] # print(type(date_for_BARRA)) # low_filename,high_filename,time_leading=self.filename_list[idx] data_low = dpt.read_access_data(access_filename_pr, idx=time_leading) lr_raw = dpt.map_aust(data_low, domain=args.domain, xrarray=False) # domain = [train_data.lon.data.min(), train_data.lon.data.max(), train_data.lat.data.min(), train_data.lat.data.max()] # print(domain) data_high = dpt.read_barra_data_fc(self.file_BARRA_dir, date_for_BARRA, nine2nine=True) label = dpt.map_aust(data_high, domain=args.domain, xrarray=False) #,domain=domain) lr = dpt.interp_tensor_2d(lr_raw, (78, 100)) if self.transform: #channel 数量需要整理!! if self.args.channels == 27: return self.transform(lr * 86400), self.transform( self.dem_data), self.transform(lr_psl), self.transform( lr_zg), self.transform(lr_tasmax), self.transform( lr_tasmin), self.transform(label), torch.tensor( int(date_for_BARRA.strftime( "%Y%m%d"))), torch.tensor(time_leading) elif self.args.channels == 5: return self.transform(lr * 86400), self.transform( self.dem_data), self.transform(lr_psl), self.transform( lr_tasmax), self.transform(lr_tasmin), self.transform( label), torch.tensor( int(date_for_BARRA.strftime( "%Y%m%d"))), torch.tensor(time_leading) if self.args.channels == 2: return self.transform(lr * 86400), self.transform( self.dem_data), self.transform(label), torch.tensor( int(date_for_BARRA.strftime("%Y%m%d"))), torch.tensor( time_leading) else: return lr * 86400, label, torch.tensor( int(date_for_BARRA.strftime("%Y%m%d"))), torch.tensor( time_leading)
def __getitem__(self, idx): ''' from filename idx get id return lr,hr ''' #read_data filemame[idx] access_filename, date_for_BARRA, time_leading = self.filename_list[idx] data_low = dpt.read_access_data(access_filename, idx=time_leading) lr_raw = dpt.map_aust(data_low, domain=args.domain, xrarray=False) lr = dpt.interp_tensor_2d(lr_raw, (78, 100)) if self.transform: return self.transform(np.expand_dims(lr, axis=3) * 86400) else: return np.expand_dims(lr, axis=3) * 86400, np.expand_dims( label, axis=3), torch.tensor(int(date_for_BARRA.strftime( "%Y%m%d"))), torch.tensor(time_leading)
def __getitem__(self, idx): ''' from filename idx get id return lr,hr ''' t = time.time() #read_data filemame[idx] access_filename_pr, en, access_date, date_for_BARRA, time_leading = self.filename_list[ idx] # print(type(date_for_BARRA)) # low_filename,high_filename,time_leading=self.filename_list[idx] lr = dpt.read_access_data( access_filename_pr, idx=time_leading).data[82:144, 134:188] * 86400 # lr=dpt.map_aust(lr,domain=self.args.domain,xrarray=False) lr = np.expand_dims(dpt.interp_tensor_2d(lr, self.shape), axis=2) lr.dtype = "float32" data_high = dpt.read_barra_data_fc(self.file_BARRA_dir, date_for_BARRA, nine2nine=True) label = dpt.map_aust(data_high, domain=self.args.domain, xrarray=False) #,domain=domain) if self.args.zg: access_filename_zg = self.args.file_ACCESS_dir + "zg/daily/" + en + "/" + "da_zg_" + access_date.strftime( "%Y%m%d") + "_" + en + ".nc" lr_zg = dpt.read_access_zg(access_filename_zg, idx=time_leading).data[:][83:145, 135:188] lr_zg = dpt.interp_tensor_3d(lr_zg, self.shape) if self.args.psl: access_filename_psl = self.args.file_ACCESS_dir + "psl/daily/" + en + "/" + "da_psl_" + access_date.strftime( "%Y%m%d") + "_" + en + ".nc" lr_psl = dpt.read_access_data(access_filename_psl, var_name="psl", idx=time_leading).data[82:144, 134:188] lr_psl = dpt.interp_tensor_2d(lr_psl, self.shape) if self.args.tasmax: access_filename_tasmax = self.args.file_ACCESS_dir + "tasmax/daily/" + en + "/" + "da_tasmax_" + access_date.strftime( "%Y%m%d") + "_" + en + ".nc" lr_tasmax = dpt.read_access_data(access_filename_tasmax, var_name="tasmax", idx=time_leading).data[82:144, 134:188] lr_tasmax = dpt.interp_tensor_2d(lr_tasmax, self.shape) if self.args.tasmin: access_filename_tasmin = self.args.file_ACCESS_dir + "tasmin/daily/" + en + "/" + "da_tasmin_" + access_date.strftime( "%Y%m%d") + "_" + en + ".nc" lr_tasmin = dpt.read_access_data(access_filename_tasmin, var_name="tasmin", idx=time_leading).data[82:144, 134:188] lr_tasmin = dpt.interp_tensor_2d(lr_tasmin, self.shape) # if self.args.dem: # # print("add dem data") # lr=np.concatenate((lr,np.expand_dims(self.dem_data,axis=2)),axis=2) # print("end loading one data,time cost %f"%(time.time()-t)) if self.transform: #channel 数量需要整理!! if self.args.channels == 27: return self.transform(lr), self.transform( self.dem_data), self.transform(lr_psl), self.transform( lr_zg), self.transform(lr_tasmax), self.transform( lr_tasmin), self.transform(label), torch.tensor( int(date_for_BARRA.strftime( "%Y%m%d"))), torch.tensor(time_leading) if self.args.channels == 2: return self.transform(lr * 86400), self.transform( self.dem_data), self.transform(label), torch.tensor( int(date_for_BARRA.strftime("%Y%m%d"))), torch.tensor( time_leading) else: return lr * 86400, label, torch.tensor( int(date_for_BARRA.strftime("%Y%m%d"))), torch.tensor( time_leading)
def main(): import netCDF4 as nc sys = platform.system() init_date = date(1970, 1, 1) start_date = date(1990, 1, 1) end_date = date(2012, 12, 25) #if 929 is true we should substract 1 day if sys == "Windows": init_date = date(1970, 1, 1) start_date = date(1990, 1, 1) end_date = date(2012, 12, 25) #if 929 is true we should substract 1 day file_ACCESS_dir = "H:/climate/access-s1/" file_BARRA_dir = "D:/dataset/accum_prcp/" # args.file_ACCESS_dir="E:/climate/access-s1/" # args.file_BARRA_dir="C:/Users/JIA059/barra/" file_DEM_dir = "../DEM/" else: file_ACCESS_dir_pr = "/g/data/ub7/access-s1/hc/raw_model/atmos/pr/daily/" file_ACCESS_dir = "/g/data/ub7/access-s1/hc/raw_model/atmos/" # training_name="temp01" file_BARRA_dir = "/g/data/ma05/BARRA_R/v1/forecast/spec/accum_prcp/" nine2nine = True date_minus_one = 1 leading_time_we_use = 7 ensemble = ['e10', 'e11'] var_name = "pr" dates = [ start_date + timedelta(x) for x in range((end_date - start_date).days + 1) ] file_list = get_filename_with_time_order( file_ACCESS_dir + var_name + '/daily/', ensemble, dates, var_name) time_leading = 7 lat_name = "lat" lon_name = "lon" # print(file_list) for i in file_list: # print(i) data = Dataset(i[0], 'r') var = data[var_name][:7, 82:144, 134:188] lat = data[lat_name][:][82:144] lon = data[lon_name][:][134:188] # print(var.shape) data.close() # lr=dpt.read_access_data(i,idx=time_leading).data[82:144,134:188]*86400 result = np.zeros((7, 79, 94)) for idx, j in enumerate(var): result[idx] = dpt.interp_tensor_2d(j, (79, 94)) if not os.path.exists('../data/' + var_name + '/daily/' + i[1]): os.mkdir('../data/' + var_name + '/daily/' + i[1]) f_w = nc.Dataset('../data/' + var_name + '/daily/' + i[1] + "/da_" + var_name + "_" + i[2].strftime("%Y%m%d") + "_" + i[1] + '.nc', 'w', format='NETCDF4') f_w.createDimension('lat', 79) f_w.createDimension('lon', 94) f_w.createDimension('time', time_leading) f_w.createVariable('lat', np.float32, ('lat')) f_w.createVariable('lon', np.float32, ('lon')) f_w.createVariable('time', np.int, ('time')) f_w.variables['lat'][:] = np.zeros((79)) f_w.variables['lon'][:] = np.zeros((94)) f_w.variables['time'][:] = np.zeros((7)) f_w.createVariable(var_name, np.float32, ('time', 'lat', 'lon')) f_w.variables[var_name][:] = result f_w.close()
def __getitem__(self, idx): ''' from filename idx get id return lr,hr ''' # t=time.time() #read_data filemame[idx] access_filename_pr, access_date, date_for_BARRA, time_leading = self.filename_list[ idx] # print(type(date_for_BARRA)) # low_filename,high_filename,time_leading=self.filename_list[idx] data_low = dpt.read_access_data(access_filename_pr, idx=time_leading) lr_raw = dpt.map_aust(data_low, domain=self.args.domain, xrarray=False) # domain = [train_data.lon.data.min(), train_data.lon.data.max(), train_data.lat.data.min(), train_data.lat.data.max()] # print(domain) data_high = dpt.read_barra_data_fc(self.file_BARRA_dir, date_for_BARRA, nine2nine=True) label = dpt.map_aust(data_high, domain=self.args.domain, xrarray=False) #,domain=domain) lr = dpt.interp_tensor_2d(lr_raw, (78, 100)) if self.args.zg: access_filename_zg = self.args.file_ACCESS_dir + "zg/daily/" + en + "/" + "da_zg_" + access_date.strftime( "%Y%m%d") + "_" + en + ".nc" data_zg = dpt.read_access_zg(access_filename_zg, idx=time_leading) data_zg_aus = map_aust(data_zg, xrarray=False) lr_zg = dpt.interp_tensor_3d(data_zg_aus, (78, 100)) if self.args.psl: access_filename_psl = self.args.file_ACCESS_dir + "psl/daily/" + en + "/" + "da_psl_" + access_date.strftime( "%Y%m%d") + "_" + en + ".nc" data_psl = dpt.read_access_data(access_filename_psl, idx=time_leading) data_psl_aus = map_aust(data_psl, xrarray=False) lr_psl = dpt.interp_tensor_2d(data_psl_aus, (78, 100)) if self.args.tasmax: access_filename_tasmax = self.args.file_ACCESS_dir + "tasmax/daily/" + en + "/" + "da_tasmax_" + access_date.strftime( "%Y%m%d") + "_" + en + ".nc" data_tasmax = dpt.read_access_data(access_filename_tasmax, idx=time_leading) data_tasmax_aus = map_aust(data_tasmax, xrarray=False) lr_tasmax = dpt.interp_tensor_2d(data_tasmax_aus, (78, 100)) if self.args.tasmax: access_filename_tasmin = self.args.file_ACCESS_dir + "tasmin/daily/" + en + "/" + "da_tasmin_" + access_date.strftime( "%Y%m%d") + "_" + en + ".nc" data_tasmin = dpt.read_access_data(access_filename_tasmin, idx=time_leading) data_tasmin_aus = map_aust(data_tasmin, xrarray=False) lr_tasmin = dpt.interp_tensor_2d(data_tasmin_aus, (78, 100)) # print("end loading one data,time cost %f"%(time.time()-t)) if self.transform: #channel 数量需要整理!! return self.transform( lr * 86400), self.transform(label), torch.tensor( int(date_for_BARRA.strftime("%Y%m%d"))), torch.tensor( time_leading) else: return lr * 86400, label, torch.tensor( int(date_for_BARRA.strftime("%Y%m%d"))), torch.tensor( time_leading)