def plot_raw_data_new(proc_id):
"""
This code makes two plots: 1) model and increment 2) model, increment and observations
Depending on which plot you want to make, it reads field_names and fields_names_obs from the PreprocConfig file
:param proc_id:
:return:
"""
config = get_preproc_config()
input_folder_tsis = config[PreprocParams.input_folder_tsis]
input_folder_forecast = config[PreprocParams.input_folder_hycom]
input_folder_obs = config[PreprocParams.input_folder_obs]
output_folder = config[PreprocParams.imgs_output_folder]
YEARS = config[PreprocParams.YEARS]
MONTHS = config[PreprocParams.MONTHS]
fields = config[PreprocParams.fields_names]
fields_obs = config[PreprocParams.fields_names_obs]
plot_modes = config[PreprocParams.plot_modes_per_field]
layers = config[PreprocParams.layers_to_plot]
img_viz = EOAImageVisualizer(output_folder=output_folder,
disp_images=False)
# Iterate current year
for c_year in YEARS:
# Iterate current month
for c_month in MONTHS:
try:
days_of_month, days_of_year = get_days_from_month(c_month)
# Reads the data (DA, Free run, and observations)
increment_files, increment_paths = get_hycom_file_name(
input_folder_tsis, c_year, c_month)
hycom_files, hycom_paths = get_hycom_file_name(
input_folder_forecast, c_year, c_month, day_idx=2)
obs_files, obs_paths = get_obs_file_names(
input_folder_obs, c_year, c_month)
except Exception as e:
print(F"Failed to find any file for date {c_year}-{c_month}")
continue
# This for is fixed to be able to run in parallel
for c_day_of_month, c_day_of_year in enumerate(days_of_year):
if (c_day_of_month % NUM_PROC) == proc_id:
# Makes regular expression of the current desired file
re_tsis = F'incupd.{c_year}_{c_day_of_year:03d}\S*.a'
re_hycom = F'020_archv.{c_year}_{c_day_of_year:03d}\S*.a'
# re_hycom = F'archv.{c_year}_{c_day_of_year:03d}\S*.a'
# re_obs = F'tsis_obs_ias_{c_year}{c_month:02d}{c_day_of_month+1:02d}\S*.nc'
re_obs = F'tsis_obs_gomb4_{c_year}{c_month:02d}{c_day_of_month+1:02d}\S*.nc'
try:
# Gets the proper index of the file for the three cases
increment_file_idx = [
i for i, file in enumerate(increment_files)
if re.search(re_tsis, file) != None
][0]
hycom_file_idx = [
i for i, file in enumerate(hycom_files)
if re.search(re_hycom, file) != None
][0]
obs_file_idx = [
i for i, file in enumerate(obs_files)
if re.search(re_obs, file) != None
][0]
except Exception as e:
print(
F"ERROR: The file for date {c_year} - {c_month} - {(c_day_of_month+1)} doesn't exist: {e}"
)
continue
print(
F" =============== Working with: {increment_files[increment_file_idx]} ============= "
)
print(
F"Available fields on increment: {read_field_names(increment_paths[increment_file_idx])}"
)
print(
F"Available fields on model: {read_field_names(hycom_paths[hycom_file_idx])}"
)
ds = xr.open_dataset(obs_paths[obs_file_idx])
print(
F"Available fields on observations: {print(list(ds.keys()))}"
)
model_state_np_fields = read_hycom_fields(
hycom_paths[hycom_file_idx], fields, layers=layers)
increment_np_fields = read_hycom_fields(
increment_paths[increment_file_idx],
fields,
layers=layers)
# obs_np_fields = read_netcdf(obs_paths[obs_file_idx], fields_obs, rename_fields=fields)
obs_np_fields = read_netcdf(obs_paths[obs_file_idx],
fields_obs)
# Iterate over the fields defined in PreprocConfig and plot them
for idx_field, c_field_name in enumerate(fields):
increment_np_c_field = increment_np_fields[
c_field_name]
nan_indx = increment_np_c_field == 0
increment_np_c_field[nan_indx] = np.nan
model_state_np_c_field = model_state_np_fields[
c_field_name]
# diff_increment_vs_fo = increment_np_c_field - model_state_np_c_field
# In these 2 cases, we only compute it for the surface layer
# diff_obs_vs_hycom = obs_np_c_field - model_state_np_c_field[0]
# obs_np_c_field[502,609] - model_state_np_c_field[0][502,609]
# diff_obs_vs_da = obs_np_c_field - increment_np_c_field[0]
# mse_hycom_vs_da = mse(increment_np_c_field, model_state_np_c_field)
# mse_obs_vs_hycom = mse(obs_np_c_field, model_state_np_c_field[0])
# mse_obs_vs_da = mse(obs_np_c_field, increment_np_c_field[0])
if c_field_name == "thknss":
divide = 9806
model_state_np_c_field = model_state_np_c_field / divide
increment_np_c_field = increment_np_c_field / divide
if c_field_name == "srfhgt":
inc = increment_np_c_field
else:
inc = (model_state_np_c_field -
increment_np_c_field)
# ======================= Only Background state and TSIS increment ==================
try:
title = F"{c_field_name} {c_year}_{c_month:02d}_{(c_day_of_month+1):02d}"
img_viz.plot_3d_data_np(
[model_state_np_c_field, inc],
# img_viz.plot_3d_data_np([model_state_np_c_field, increment_np_c_field],
var_names=['HYCOM', 'Increment (TSIS)'],
title=title,
file_name_prefix=
F'ModelAndIncrement_{c_field_name}_{c_year}_{c_month:02d}_{(c_day_of_month+1):02d}',
z_lavels_names=layers,
flip_data=True,
plot_mode=plot_modes[idx_field])
except Exception as e:
print(F"Failed for field: {c_field_name}: {e}")
def img_generation_hycom(proc_id):
"""
Makes images of the available data (Free run, DA and Observations)
:param proc_id:
:return:
"""
config = get_preproc_config()
input_folder_tsis = config[PreprocParams.input_folder_tsis]
input_folder_forecast = config[PreprocParams.input_folder_hycom]
input_folder_obs = config[PreprocParams.input_folder_obs]
output_folder = config[PreprocParams.imgs_output_folder]
YEARS = config[PreprocParams.YEARS]
MONTHS = config[PreprocParams.MONTHS]
fields = config[PreprocParams.fields_names]
fields_obs = config[PreprocParams.fields_names_obs]
plot_modes = config[PreprocParams.plot_modes_per_field]
layers = config[PreprocParams.layers_to_plot]
img_viz = EOAImageVisualizer(output_folder=output_folder,
disp_images=False)
# Iterate current year
for c_year in YEARS:
# Iterate current month
for c_month in MONTHS:
try:
days_of_month, days_of_year = get_days_from_month(c_month)
# Reads the data (DA, Free run, and observations)
hycom_files, hycom_paths = get_hycom_file_name(
input_folder_forecast, c_year, c_month)
except Exception as e:
print(F"Failed to find any file for date {c_year}-{c_month}")
continue
# This for is fixed to be able to run in parallel
for c_day_of_month, c_day_of_year in enumerate(days_of_year):
if (c_day_of_month % NUM_PROC) == proc_id:
# Makes regular expression of the current desired file
re_hycom = F'archv.{c_year}_{c_day_of_year:03d}\S*.a'
try:
# Gets the proper index of the file for the three cases
hycom_file_idx = [
i for i, file in enumerate(hycom_files)
if re.search(re_hycom, file) != None
][0]
except Exception as e:
print(
F"ERROR: The file for date {c_year} - {c_month} - {c_day_of_month} doesn't exist: {e}"
)
continue
print(
F" =============== Working with: {hycom_files[hycom_file_idx]} ============= "
)
print(
F"Available fields: {read_field_names(hycom_paths[hycom_file_idx])}"
)
model_state_np_fields = read_hycom_fields(
hycom_paths[hycom_file_idx], fields, layers=layers)
for idx_field, c_field_name in enumerate(fields):
model_state_np_c_field = model_state_np_fields[
c_field_name]
title = F"{c_field_name} {c_year}_{c_month:02d}_{(c_day_of_month+1):02d}"
# ======================= Only Fredatae HYCOM, TSIS, Observations ==================
img_viz.plot_3d_data_np(
[model_state_np_c_field],
var_names=[F'HYCOM'],
title=title,
file_name_prefix=
F'HYCOM_{c_field_name}_{c_year}_{c_month:02d}_{c_day_of_month:02d}',
z_lavels_names=layers,
flip_data=True,
plot_mode=plot_modes[idx_field])
def plot_raw_data(proc_id):
"""
Makes images of the available data (Free run, DA and Observations)
:param proc_id:
:return:
"""
config = get_preproc_config()
input_folder_tsis = config[PreprocParams.input_folder_tsis]
input_folder_forecast = config[PreprocParams.input_folder_hycom]
input_folder_obs = config[PreprocParams.input_folder_obs]
output_folder = config[PreprocParams.imgs_output_folder]
YEARS = config[PreprocParams.YEARS]
MONTHS = config[PreprocParams.MONTHS]
fields = config[PreprocParams.fields_names]
fields_obs = config[PreprocParams.fields_names_obs]
plot_modes = config[PreprocParams.plot_modes_per_field]
layers = config[PreprocParams.layers_to_plot]
img_viz = EOAImageVisualizer(output_folder=output_folder,
disp_images=False)
# Iterate current year
for c_year in YEARS:
# Iterate current month
for c_month in MONTHS:
try:
days_of_month, days_of_year = get_days_from_month(c_month)
# Reads the data (DA, Free run, and observations)
increment_files, increment_paths = get_hycom_file_name(
input_folder_tsis, c_year, c_month)
hycom_files, hycom_paths = get_hycom_file_name(
input_folder_forecast, c_year, c_month)
obs_files, obs_paths = get_obs_file_names(
input_folder_obs, c_year, c_month)
except Exception as e:
print(F"Failed to find any file for date {c_year}-{c_month}")
continue
# This for is fixed to be able to run in parallel
for c_day_of_month, c_day_of_year in enumerate(days_of_year):
if (c_day_of_month % NUM_PROC) == proc_id:
# Makes regular expression of the current desired file
re_tsis = F'incupd.{c_year}_{c_day_of_year:03d}\S*.a'
re_hycom = F'archv.{c_year}_{c_day_of_year:03d}\S*.a'
re_obs = F'tsis_obs_ias_{c_year}{c_month:02d}{c_day_of_month+1:02d}\S*.nc'
try:
# Gets the proper index of the file for the three cases
increment_file_idx = [
i for i, file in enumerate(increment_files)
if re.search(re_tsis, file) != None
][0]
hycom_file_idx = [
i for i, file in enumerate(hycom_files)
if re.search(re_hycom, file) != None
][0]
obs_file_idx = [
i for i, file in enumerate(obs_files)
if re.search(re_obs, file) != None
][0]
except Exception as e:
print(
F"ERROR: The file for date {c_year} - {c_month} - {(c_day_of_month+1)} doesn't exist: {e}"
)
continue
print(
F" =============== Working with: {increment_files[increment_file_idx]} ============= "
)
print(
F"Available fields on increment: {read_field_names(increment_paths[increment_file_idx])}"
)
increment_np_fields = read_hycom_fields(
increment_paths[increment_file_idx],
fields,
layers=layers)
model_state_np_fields = read_hycom_fields(
hycom_paths[hycom_file_idx], fields, layers=layers)
obs_np_fields = read_netcdf(obs_paths[obs_file_idx],
fields_obs,
layers=[0],
rename_fields=fields)
for idx_field, c_field_name in enumerate(fields):
increment_np_c_field = increment_np_fields[
c_field_name]
nan_indx = increment_np_c_field == 0
increment_np_c_field[nan_indx] = np.nan
model_state_np_c_field = model_state_np_fields[
c_field_name]
obs_np_c_field = obs_np_fields[c_field_name]
# diff_increment_vs_fo = increment_np_c_field - model_state_np_c_field
# In these 2 cases, we only compute it for the surface layer
# diff_obs_vs_hycom = obs_np_c_field - model_state_np_c_field[0]
obs_np_c_field[502,
609] - model_state_np_c_field[0][502,
609]
# diff_obs_vs_da = obs_np_c_field - increment_np_c_field[0]
# mse_hycom_vs_da = mse(increment_np_c_field, model_state_np_c_field)
# mse_obs_vs_hycom = mse(obs_np_c_field, model_state_np_c_field[0])
# mse_obs_vs_da = mse(obs_np_c_field, increment_np_c_field[0])
title = F"{c_field_name} {c_year}_{c_month:02d}_{(c_day_of_month+1):02d}"
# ======================= Only Fredatae HYCOM, TSIS, Observations ==================
img_viz.plot_3d_data_np(
[
np.expand_dims(obs_np_c_field, 0),
model_state_np_c_field, increment_np_c_field
],
var_names=[
F'Observations', 'HYCOM', 'Increment (TSIS)'
],
title=title,
file_name_prefix=
F'Summary_{c_field_name}_{c_year}_{c_month:02d}_{(c_day_of_month+1):02d}',
z_lavels_names=layers,
flip_data=True,
plot_mode=plot_modes[idx_field])