def get_lakefraction(path='data/geophys_lam_na_0.5deg_170x158_class',
margin=20):
r = RPN(path)
data = r.get_first_record_for_name_and_level('VF', level=3)
r.close()
return data[margin:-margin, margin:-margin, 0]
pass
def test():
rObj = RPN('write.rpn', mode='w')
nx = 20
ny = 40
data = np.zeros((nx, ny))
for i in range(nx):
for j in range(ny):
data[i, j] = i**2 + j**2
print('before ', data.shape, data.min(), data.max(), data.mean())
plt.figure()
plt.title('before')
plt.pcolormesh(data)
plt.colorbar()
rObj.write_2D_field('test', level=1, data=data, grid_type='')
rObj.close()
rObj = RPN('write.rpn')
x = rObj.get_first_record_for_name('test')
rObj.close()
print('after ', x.shape, x.min(), x.max(), x.mean())
plt.figure()
plt.title('after')
plt.pcolormesh(x)
plt.colorbar()
plt.show()
def calculate_monthly_sum(
data_folder='/home/huziy/skynet3_rech1/gemclim/quebec/Samples',
month=6,
year=1999,
prefix='quebec_220x220_',
var_name='GWDI',
level=-1):
suffix = name_format % (year, month)
folder_for_month = os.path.join(data_folder, prefix + suffix)
count = 0.0
result_data = None
if not os.path.isdir(folder_for_month):
return 0, 0
for file in os.listdir(folder_for_month):
if not file.startswith('pm'):
continue
the_path = os.path.join(folder_for_month, file)
print(the_path)
r = RPN(the_path)
the_data = r.get_first_record_for_name_and_level(var_name, level=level)
if result_data == None:
result_data = the_data
else:
result_data += the_data
count += 1
r.close()
return result_data, count
pass
def calculate_monthly_sum(
data_folder="/home/huziy/skynet3_rech1/gemclim/quebec/Samples",
month=6,
year=1999,
prefix="quebec_220x220_",
var_name="GWDI",
level=-1,
):
suffix = name_format % (year, month)
folder_for_month = os.path.join(data_folder, prefix + suffix)
count = 0.0
result_data = None
if not os.path.isdir(folder_for_month):
return 0, 0
for file in os.listdir(folder_for_month):
if not file.startswith("pm"):
continue
the_path = os.path.join(folder_for_month, file)
print(the_path)
r = RPN(the_path)
the_data = r.get_first_record_for_name_and_level(var_name, level=level)
if result_data == None:
result_data = the_data
else:
result_data += the_data
count += 1
r.close()
return result_data, count
pass
def test():
rObj = RPN('write.rpn', mode = 'w')
nx = 20
ny = 40
data = np.zeros((nx, ny))
for i in range(nx):
for j in range(ny):
data[i, j] = i ** 2 + j ** 2
print('before ', data.shape, data.min(), data.max(), data.mean())
plt.figure()
plt.title('before')
plt.pcolormesh(data)
plt.colorbar()
rObj.write_2D_field('test', level = 1, data = data, grid_type = '')
rObj.close()
rObj = RPN('write.rpn')
x = rObj.get_first_record_for_name('test')
rObj.close()
print('after ', x.shape, x.min(), x.max(), x.mean())
plt.figure()
plt.title('after')
plt.pcolormesh(x)
plt.colorbar()
plt.show()
def get_lon_lat(
path='/home/huziy/skynet3_rech1/gemclim/quebec/Samples/quebec_220x220_199912/pm1999050100_00068760p'
):
r = RPN(path)
lons, lats = r.get_longitudes_and_latitudes()
r.close()
return lons, lats
def read_data(folder="data/1950-1960-ECHO-G-profiles"):
ni, nj, id_to_hor_indices, land_sea_mask = read_coordinates()
nz = -1
all_data = []
for year_folder in os.listdir(folder):
year_folder_path = os.path.join(folder, year_folder)
if not os.path.isdir(year_folder_path):
continue
data = None
for file in os.listdir(year_folder_path):
file_path = os.path.join(year_folder_path, file)
f = open(file_path)
lines = [x.strip() for x in f.readlines()]
lines = list(filter(lambda x: len(x) > 0, lines))
if data is None:
nz = len(lines)
data = np.zeros((ni, nj, nz))
the_id = int(re.findall("\d+", file)[0])
i, j = id_to_hor_indices[the_id]
profile = [float(x.split()[-1]) for x in lines]
data[i, j, :] = np.array(profile)[:]
f.close()
all_data.append(np.fliplr(data))
mean_data = np.mean(all_data, axis=0)
rpn_obj = RPN("data/soil_profiles/{0}.rpn".format("profile_200"), mode="w")
rpn_obj.write_2D_field(grid_type="A",
ig=[0, 0, 0, 0],
data=np.fliplr(land_sea_mask),
name="MASK")
for level in range(nz):
#longitudinal grid length is 360/NI. For such a grid,
#IG1 contains the domain of the grid: 0: Global
#1: Northern Hemisphere
#2: Southern Hemisphere IG2 contains the orientation of the grid:
#0: South -> North (pt (1,1) is at the bottom of the grid)
#1: North -> South (pt (1,1) is at the top of the grid) IG3 should be 0.
#IG4 should be 0.
rpn_obj.write_2D_field(
grid_type="A",
ig=[0, 0, 0, 0],
data=mean_data[:, :, level],
level=level,
level_kind=level_kinds.HEIGHT_METERS_OVER_SEA_LEVEL,
name="TBAR")
rpn_obj.close()
def calculate_time_integral(
data_folder='data/crcm_sim_with_lakes/data_selected',
start_date=None,
end_date=None,
var_name='FV',
level=7,
level_kind=level_kinds.ARBITRARY,
name_pattern='Coupled11_36cpu_Test_C_%Y%m',
dt=3 * 60 * 60,
file_size_limit_bytes=None):
date = start_date
time_integral = None
while date <= end_date:
current_folder = os.path.join(data_folder, date.strftime(name_pattern))
for file in os.listdir(current_folder):
if not file.startswith('pm'): #skip non physics files
continue
file_path = os.path.join(current_folder, file)
if file_size_limit_bytes != None:
if os.path.getsize(file_path) < file_size_limit_bytes:
continue
r = RPN(file_path)
print('current folder ', current_folder)
print('reading file {0}'.format(file))
data = r.get_first_record_for_name_and_level(varname=var_name,
level=level,
level_kind=level_kind)
data = data[:, :, 0]
if date == start_date:
time_integral = data
else:
time_integral += data
r.close()
#add month
if date.month == 12:
date = datetime(date.year + 1, 1, date.day, date.hour, date.minute)
else:
date = datetime(date.year, date.month + 1, date.day, date.hour,
date.minute)
return time_integral * dt
pass
def read_data(folder = "data/1950-1960-ECHO-G-profiles"):
ni, nj, id_to_hor_indices, land_sea_mask = read_coordinates()
nz = -1
all_data = []
for year_folder in os.listdir(folder):
year_folder_path = os.path.join(folder, year_folder)
if not os.path.isdir(year_folder_path):
continue
data = None
for file in os.listdir(year_folder_path):
file_path = os.path.join(year_folder_path, file)
f = open(file_path)
lines = [x.strip() for x in f.readlines()]
lines = list(filter(lambda x: len(x) > 0, lines))
if data is None:
nz = len(lines)
data = np.zeros((ni, nj, nz))
the_id = int(re.findall("\d+", file)[0])
i, j = id_to_hor_indices[the_id]
profile = [float(x.split()[-1]) for x in lines]
data[i, j, :] = np.array(profile)[:]
f.close()
all_data.append( np.fliplr( data ))
mean_data = np.mean(all_data, axis = 0)
rpn_obj = RPN("data/soil_profiles/{0}.rpn".format("profile_200"), mode="w")
rpn_obj.write_2D_field(grid_type="A", ig = [0, 0, 0, 0],data=np.fliplr(land_sea_mask), name = "MASK")
for level in range(nz):
#longitudinal grid length is 360/NI. For such a grid,
#IG1 contains the domain of the grid: 0: Global
#1: Northern Hemisphere
#2: Southern Hemisphere IG2 contains the orientation of the grid:
#0: South -> North (pt (1,1) is at the bottom of the grid)
#1: North -> South (pt (1,1) is at the top of the grid) IG3 should be 0.
#IG4 should be 0.
rpn_obj.write_2D_field(grid_type="A", ig = [0, 0, 0, 0],
data= mean_data[:,:, level], level = level, level_kind=level_kinds.HEIGHT_METERS_OVER_SEA_LEVEL,
name="TBAR"
)
rpn_obj.close()
def calculate_seasonal_mean(data_path = "", field_name = "STFL", file_prefix = None, months = None):
"""
calculates seasonal means,
months - list of months when the averaging is performed 1 = Jan, ..., 12 = Dec
TODO: implement
"""
result = None
field_count = 0.0
lons, lats = None, None
for monthFolder in os.listdir(data_path):
monthPath = os.path.join(data_path, monthFolder)
for fName in os.listdir(monthPath):
if file_prefix is not None:
if not fName.startswith(file_prefix):
continue
rObj = RPN(os.path.join(monthPath, fName))
field = rObj.get_first_record_for_name(field_name)
vDate = rObj.get_current_validity_date()
originDate = rObj.get_dateo_of_last_read_record()
print("-" * 10)
print("validity date, origin date", vDate, originDate)
print("-" * 10)
if result is None:
result = field
lons, lats = rObj.get_longitudes_and_latitudes()
else:
result = (field + result * field_count) / ( field_count + 1.0 )
rObj.close()
field_count += 1.0
pass
def calculate_seasonal_mean(data_path="",
field_name="STFL",
file_prefix=None,
months=None):
"""
calculates seasonal means,
months - list of months when the averaging is performed 1 = Jan, ..., 12 = Dec
TODO: implement
"""
result = None
field_count = 0.0
lons, lats = None, None
for monthFolder in os.listdir(data_path):
monthPath = os.path.join(data_path, monthFolder)
for fName in os.listdir(monthPath):
if file_prefix is not None:
if not fName.startswith(file_prefix):
continue
rObj = RPN(os.path.join(monthPath, fName))
field = rObj.get_first_record_for_name(field_name)
vDate = rObj.get_current_validity_date()
originDate = rObj.get_dateo_of_last_read_record()
print("-" * 10)
print("validity date, origin date", vDate, originDate)
print("-" * 10)
if result is None:
result = field
lons, lats = rObj.get_longitudes_and_latitudes()
else:
result = (field + result * field_count) / (field_count + 1.0)
rObj.close()
field_count += 1.0
pass
def calculate_time_integral(data_folder = 'data/crcm_sim_with_lakes/data_selected',
start_date = None, end_date = None, var_name = 'FV', level = 7,
level_kind = level_kinds.ARBITRARY,
name_pattern = 'Coupled11_36cpu_Test_C_%Y%m',
dt = 3 * 60 * 60,
file_size_limit_bytes = None
):
date = start_date
time_integral = None
while date <= end_date:
current_folder = os.path.join(data_folder, date.strftime(name_pattern))
for file in os.listdir(current_folder):
if not file.startswith('pm'): #skip non physics files
continue
file_path = os.path.join(current_folder, file)
if file_size_limit_bytes != None:
if os.path.getsize(file_path) < file_size_limit_bytes:
continue
r = RPN(file_path)
print('current folder ', current_folder)
print('reading file {0}'.format(file))
data = r.get_first_record_for_name_and_level(varname = var_name, level = level, level_kind = level_kind)
data = data[:,:,0]
if date == start_date:
time_integral = data
else:
time_integral += data
r.close()
#add month
if date.month == 12:
date = datetime(date.year + 1, 1, date.day, date.hour, date.minute)
else:
date = datetime(date.year, date.month + 1, date.day, date.hour, date.minute)
return time_integral * dt
pass
def calculate_mean_field(data_path = "", field_name = "STFL", file_prefix = None):
"""
Calculates annual mean field from rpn files
data_path = path to the Samples folder
"""
result = None
field_count = 0.0
lons, lats = None, None
for monthFolder in os.listdir(data_path):
monthPath = os.path.join(data_path, monthFolder)
for fName in os.listdir(monthPath):
if file_prefix is not None:
if not fName.startswith(file_prefix):
continue
rObj = RPN(os.path.join(monthPath, fName))
field = rObj.get_first_record_for_name(field_name)
vDate = rObj.get_current_validity_date()
originDate = rObj.get_dateo_of_last_read_record()
print("-" * 10)
print("validity date, origin date", vDate, originDate)
print(rObj.get_datetime_for_the_last_read_record())
print("-" * 10)
if result is None:
result = field
lons, lats = rObj.get_longitudes_and_latitudes()
else:
result = (field + result * field_count) / ( field_count + 1.0 )
rObj.close()
field_count += 1.0
return lons, lats, result
def calculate_mean_field(data_path="", field_name="STFL", file_prefix=None):
"""
Calculates annual mean field from rpn files
data_path = path to the Samples folder
"""
result = None
field_count = 0.0
lons, lats = None, None
for monthFolder in os.listdir(data_path):
monthPath = os.path.join(data_path, monthFolder)
for fName in os.listdir(monthPath):
if file_prefix is not None:
if not fName.startswith(file_prefix):
continue
rObj = RPN(os.path.join(monthPath, fName))
field = rObj.get_first_record_for_name(field_name)
vDate = rObj.get_current_validity_date()
originDate = rObj.get_dateo_of_last_read_record()
print("-" * 10)
print("validity date, origin date", vDate, originDate)
print(rObj.get_datetime_for_the_last_read_record())
print("-" * 10)
if result is None:
result = field
lons, lats = rObj.get_longitudes_and_latitudes()
else:
result = (field + result * field_count) / (field_count + 1.0)
rObj.close()
field_count += 1.0
return lons, lats, result
def get_lon_lat(path="/home/huziy/skynet3_rech1/gemclim/quebec/Samples/quebec_220x220_199912/pm1999050100_00068760p"):
r = RPN(path)
lons, lats = r.get_longitudes_and_latitudes()
r.close()
return lons, lats
def get_lon_lat(path='data/pm1957090100_00589248p'):
print('reading lons and lats from the file %s' % path)
r = RPN(path)
lons, lats = r.get_longitudes_and_latitudes()
r.close()
return lons, lats
def get_lon_lat(path = 'data/pm1957090100_00589248p'):
print('reading lons and lats from the file %s' % path)
r = RPN(path)
lons, lats = r.get_longitudes_and_latitudes()
r.close()
return lons, lats
def get_lakefraction(path = 'data/geophys_lam_na_0.5deg_170x158_class', margin = 20):
r = RPN(path)
data = r.get_first_record_for_name_and_level('VF', level = 3)
r.close()
return data[margin:-margin, margin:-margin, 0]
pass
