def get_lakefraction(path='data/geophys_lam_na_0.5deg_170x158_class', margin=20): r = RPN(path) data = r.get_first_record_for_name_and_level('VF', level=3) r.close() return data[margin:-margin, margin:-margin, 0] pass
def test(): rObj = RPN('write.rpn', mode='w') nx = 20 ny = 40 data = np.zeros((nx, ny)) for i in range(nx): for j in range(ny): data[i, j] = i**2 + j**2 print('before ', data.shape, data.min(), data.max(), data.mean()) plt.figure() plt.title('before') plt.pcolormesh(data) plt.colorbar() rObj.write_2D_field('test', level=1, data=data, grid_type='') rObj.close() rObj = RPN('write.rpn') x = rObj.get_first_record_for_name('test') rObj.close() print('after ', x.shape, x.min(), x.max(), x.mean()) plt.figure() plt.title('after') plt.pcolormesh(x) plt.colorbar() plt.show()
def calculate_monthly_sum( data_folder='/home/huziy/skynet3_rech1/gemclim/quebec/Samples', month=6, year=1999, prefix='quebec_220x220_', var_name='GWDI', level=-1): suffix = name_format % (year, month) folder_for_month = os.path.join(data_folder, prefix + suffix) count = 0.0 result_data = None if not os.path.isdir(folder_for_month): return 0, 0 for file in os.listdir(folder_for_month): if not file.startswith('pm'): continue the_path = os.path.join(folder_for_month, file) print(the_path) r = RPN(the_path) the_data = r.get_first_record_for_name_and_level(var_name, level=level) if result_data == None: result_data = the_data else: result_data += the_data count += 1 r.close() return result_data, count pass
def calculate_monthly_sum( data_folder="/home/huziy/skynet3_rech1/gemclim/quebec/Samples", month=6, year=1999, prefix="quebec_220x220_", var_name="GWDI", level=-1, ): suffix = name_format % (year, month) folder_for_month = os.path.join(data_folder, prefix + suffix) count = 0.0 result_data = None if not os.path.isdir(folder_for_month): return 0, 0 for file in os.listdir(folder_for_month): if not file.startswith("pm"): continue the_path = os.path.join(folder_for_month, file) print(the_path) r = RPN(the_path) the_data = r.get_first_record_for_name_and_level(var_name, level=level) if result_data == None: result_data = the_data else: result_data += the_data count += 1 r.close() return result_data, count pass
def test(): rObj = RPN('write.rpn', mode = 'w') nx = 20 ny = 40 data = np.zeros((nx, ny)) for i in range(nx): for j in range(ny): data[i, j] = i ** 2 + j ** 2 print('before ', data.shape, data.min(), data.max(), data.mean()) plt.figure() plt.title('before') plt.pcolormesh(data) plt.colorbar() rObj.write_2D_field('test', level = 1, data = data, grid_type = '') rObj.close() rObj = RPN('write.rpn') x = rObj.get_first_record_for_name('test') rObj.close() print('after ', x.shape, x.min(), x.max(), x.mean()) plt.figure() plt.title('after') plt.pcolormesh(x) plt.colorbar() plt.show()
def get_lon_lat( path='/home/huziy/skynet3_rech1/gemclim/quebec/Samples/quebec_220x220_199912/pm1999050100_00068760p' ): r = RPN(path) lons, lats = r.get_longitudes_and_latitudes() r.close() return lons, lats
def read_data(folder="data/1950-1960-ECHO-G-profiles"): ni, nj, id_to_hor_indices, land_sea_mask = read_coordinates() nz = -1 all_data = [] for year_folder in os.listdir(folder): year_folder_path = os.path.join(folder, year_folder) if not os.path.isdir(year_folder_path): continue data = None for file in os.listdir(year_folder_path): file_path = os.path.join(year_folder_path, file) f = open(file_path) lines = [x.strip() for x in f.readlines()] lines = list(filter(lambda x: len(x) > 0, lines)) if data is None: nz = len(lines) data = np.zeros((ni, nj, nz)) the_id = int(re.findall("\d+", file)[0]) i, j = id_to_hor_indices[the_id] profile = [float(x.split()[-1]) for x in lines] data[i, j, :] = np.array(profile)[:] f.close() all_data.append(np.fliplr(data)) mean_data = np.mean(all_data, axis=0) rpn_obj = RPN("data/soil_profiles/{0}.rpn".format("profile_200"), mode="w") rpn_obj.write_2D_field(grid_type="A", ig=[0, 0, 0, 0], data=np.fliplr(land_sea_mask), name="MASK") for level in range(nz): #longitudinal grid length is 360/NI. For such a grid, #IG1 contains the domain of the grid: 0: Global #1: Northern Hemisphere #2: Southern Hemisphere IG2 contains the orientation of the grid: #0: South -> North (pt (1,1) is at the bottom of the grid) #1: North -> South (pt (1,1) is at the top of the grid) IG3 should be 0. #IG4 should be 0. rpn_obj.write_2D_field( grid_type="A", ig=[0, 0, 0, 0], data=mean_data[:, :, level], level=level, level_kind=level_kinds.HEIGHT_METERS_OVER_SEA_LEVEL, name="TBAR") rpn_obj.close()
def calculate_time_integral( data_folder='data/crcm_sim_with_lakes/data_selected', start_date=None, end_date=None, var_name='FV', level=7, level_kind=level_kinds.ARBITRARY, name_pattern='Coupled11_36cpu_Test_C_%Y%m', dt=3 * 60 * 60, file_size_limit_bytes=None): date = start_date time_integral = None while date <= end_date: current_folder = os.path.join(data_folder, date.strftime(name_pattern)) for file in os.listdir(current_folder): if not file.startswith('pm'): #skip non physics files continue file_path = os.path.join(current_folder, file) if file_size_limit_bytes != None: if os.path.getsize(file_path) < file_size_limit_bytes: continue r = RPN(file_path) print('current folder ', current_folder) print('reading file {0}'.format(file)) data = r.get_first_record_for_name_and_level(varname=var_name, level=level, level_kind=level_kind) data = data[:, :, 0] if date == start_date: time_integral = data else: time_integral += data r.close() #add month if date.month == 12: date = datetime(date.year + 1, 1, date.day, date.hour, date.minute) else: date = datetime(date.year, date.month + 1, date.day, date.hour, date.minute) return time_integral * dt pass
def read_data(folder = "data/1950-1960-ECHO-G-profiles"): ni, nj, id_to_hor_indices, land_sea_mask = read_coordinates() nz = -1 all_data = [] for year_folder in os.listdir(folder): year_folder_path = os.path.join(folder, year_folder) if not os.path.isdir(year_folder_path): continue data = None for file in os.listdir(year_folder_path): file_path = os.path.join(year_folder_path, file) f = open(file_path) lines = [x.strip() for x in f.readlines()] lines = list(filter(lambda x: len(x) > 0, lines)) if data is None: nz = len(lines) data = np.zeros((ni, nj, nz)) the_id = int(re.findall("\d+", file)[0]) i, j = id_to_hor_indices[the_id] profile = [float(x.split()[-1]) for x in lines] data[i, j, :] = np.array(profile)[:] f.close() all_data.append( np.fliplr( data )) mean_data = np.mean(all_data, axis = 0) rpn_obj = RPN("data/soil_profiles/{0}.rpn".format("profile_200"), mode="w") rpn_obj.write_2D_field(grid_type="A", ig = [0, 0, 0, 0],data=np.fliplr(land_sea_mask), name = "MASK") for level in range(nz): #longitudinal grid length is 360/NI. For such a grid, #IG1 contains the domain of the grid: 0: Global #1: Northern Hemisphere #2: Southern Hemisphere IG2 contains the orientation of the grid: #0: South -> North (pt (1,1) is at the bottom of the grid) #1: North -> South (pt (1,1) is at the top of the grid) IG3 should be 0. #IG4 should be 0. rpn_obj.write_2D_field(grid_type="A", ig = [0, 0, 0, 0], data= mean_data[:,:, level], level = level, level_kind=level_kinds.HEIGHT_METERS_OVER_SEA_LEVEL, name="TBAR" ) rpn_obj.close()
def calculate_seasonal_mean(data_path = "", field_name = "STFL", file_prefix = None, months = None): """ calculates seasonal means, months - list of months when the averaging is performed 1 = Jan, ..., 12 = Dec TODO: implement """ result = None field_count = 0.0 lons, lats = None, None for monthFolder in os.listdir(data_path): monthPath = os.path.join(data_path, monthFolder) for fName in os.listdir(monthPath): if file_prefix is not None: if not fName.startswith(file_prefix): continue rObj = RPN(os.path.join(monthPath, fName)) field = rObj.get_first_record_for_name(field_name) vDate = rObj.get_current_validity_date() originDate = rObj.get_dateo_of_last_read_record() print("-" * 10) print("validity date, origin date", vDate, originDate) print("-" * 10) if result is None: result = field lons, lats = rObj.get_longitudes_and_latitudes() else: result = (field + result * field_count) / ( field_count + 1.0 ) rObj.close() field_count += 1.0 pass
def calculate_seasonal_mean(data_path="", field_name="STFL", file_prefix=None, months=None): """ calculates seasonal means, months - list of months when the averaging is performed 1 = Jan, ..., 12 = Dec TODO: implement """ result = None field_count = 0.0 lons, lats = None, None for monthFolder in os.listdir(data_path): monthPath = os.path.join(data_path, monthFolder) for fName in os.listdir(monthPath): if file_prefix is not None: if not fName.startswith(file_prefix): continue rObj = RPN(os.path.join(monthPath, fName)) field = rObj.get_first_record_for_name(field_name) vDate = rObj.get_current_validity_date() originDate = rObj.get_dateo_of_last_read_record() print("-" * 10) print("validity date, origin date", vDate, originDate) print("-" * 10) if result is None: result = field lons, lats = rObj.get_longitudes_and_latitudes() else: result = (field + result * field_count) / (field_count + 1.0) rObj.close() field_count += 1.0 pass
def calculate_time_integral(data_folder = 'data/crcm_sim_with_lakes/data_selected', start_date = None, end_date = None, var_name = 'FV', level = 7, level_kind = level_kinds.ARBITRARY, name_pattern = 'Coupled11_36cpu_Test_C_%Y%m', dt = 3 * 60 * 60, file_size_limit_bytes = None ): date = start_date time_integral = None while date <= end_date: current_folder = os.path.join(data_folder, date.strftime(name_pattern)) for file in os.listdir(current_folder): if not file.startswith('pm'): #skip non physics files continue file_path = os.path.join(current_folder, file) if file_size_limit_bytes != None: if os.path.getsize(file_path) < file_size_limit_bytes: continue r = RPN(file_path) print('current folder ', current_folder) print('reading file {0}'.format(file)) data = r.get_first_record_for_name_and_level(varname = var_name, level = level, level_kind = level_kind) data = data[:,:,0] if date == start_date: time_integral = data else: time_integral += data r.close() #add month if date.month == 12: date = datetime(date.year + 1, 1, date.day, date.hour, date.minute) else: date = datetime(date.year, date.month + 1, date.day, date.hour, date.minute) return time_integral * dt pass
def calculate_mean_field(data_path = "", field_name = "STFL", file_prefix = None): """ Calculates annual mean field from rpn files data_path = path to the Samples folder """ result = None field_count = 0.0 lons, lats = None, None for monthFolder in os.listdir(data_path): monthPath = os.path.join(data_path, monthFolder) for fName in os.listdir(monthPath): if file_prefix is not None: if not fName.startswith(file_prefix): continue rObj = RPN(os.path.join(monthPath, fName)) field = rObj.get_first_record_for_name(field_name) vDate = rObj.get_current_validity_date() originDate = rObj.get_dateo_of_last_read_record() print("-" * 10) print("validity date, origin date", vDate, originDate) print(rObj.get_datetime_for_the_last_read_record()) print("-" * 10) if result is None: result = field lons, lats = rObj.get_longitudes_and_latitudes() else: result = (field + result * field_count) / ( field_count + 1.0 ) rObj.close() field_count += 1.0 return lons, lats, result
def calculate_mean_field(data_path="", field_name="STFL", file_prefix=None): """ Calculates annual mean field from rpn files data_path = path to the Samples folder """ result = None field_count = 0.0 lons, lats = None, None for monthFolder in os.listdir(data_path): monthPath = os.path.join(data_path, monthFolder) for fName in os.listdir(monthPath): if file_prefix is not None: if not fName.startswith(file_prefix): continue rObj = RPN(os.path.join(monthPath, fName)) field = rObj.get_first_record_for_name(field_name) vDate = rObj.get_current_validity_date() originDate = rObj.get_dateo_of_last_read_record() print("-" * 10) print("validity date, origin date", vDate, originDate) print(rObj.get_datetime_for_the_last_read_record()) print("-" * 10) if result is None: result = field lons, lats = rObj.get_longitudes_and_latitudes() else: result = (field + result * field_count) / (field_count + 1.0) rObj.close() field_count += 1.0 return lons, lats, result
def get_lon_lat(path="/home/huziy/skynet3_rech1/gemclim/quebec/Samples/quebec_220x220_199912/pm1999050100_00068760p"): r = RPN(path) lons, lats = r.get_longitudes_and_latitudes() r.close() return lons, lats
def get_lon_lat(path='data/pm1957090100_00589248p'): print('reading lons and lats from the file %s' % path) r = RPN(path) lons, lats = r.get_longitudes_and_latitudes() r.close() return lons, lats
def get_lon_lat(path = 'data/pm1957090100_00589248p'): print('reading lons and lats from the file %s' % path) r = RPN(path) lons, lats = r.get_longitudes_and_latitudes() r.close() return lons, lats
def get_lakefraction(path = 'data/geophys_lam_na_0.5deg_170x158_class', margin = 20): r = RPN(path) data = r.get_first_record_for_name_and_level('VF', level = 3) r.close() return data[margin:-margin, margin:-margin, 0] pass