def graph_bar():
from netCDF4 import Dataset
for index_l, index_path in get_subdirs(indices_output_dir):
if index_l != index_in['name']:
continue
avg6190 = {}
files_to_plot = {}
for region, region_path in get_subdirs(index_path):
if 'ignore' in index_in and region in index_in['ignore']:
continue
avg6190[region] = None
files_to_plot[region] = []
for rcp, rcp_path in get_subdirs(region_path):
for file, file_path in get_subfiles(rcp_path):
if file.startswith("._"):
continue
elif file.endswith("ts.nc"):
files_to_plot[region].append(file_path)
elif file.endswith("Avg6190.nc"):
fh = Dataset(file_path, 'r')
avg6190[region] = fh.variables[index_in['cdo_name']][0, 0, 0]
fh.close()
plot_bar_index(index=index_in, files=files_to_plot, png_name_in=indices_graphs_out_dir+"/"+index_l+"/"+index_in['name']+'_bar.png', avg6190=avg6190)
def loop_models():
# loop all the index we want to obtain, and the required param for that index
for index, req_param in zip(index_array, req_param_array):
print("Calculate %s using %s" % (index, req_param))
# loop the experiment folders we have
for experiment in experiment_array:
# loop of all models inside the experiment folder
for model, model_path in get_subdirs(experiment):
# loop of all parameters inside each model
for param, param_path in get_subdirs(model_path):
# if the param is the required one for the index
if param == req_param:
for region, region_path in get_subdirs(param_path):
# loop all files inside the param path
for file, file_path in get_subfiles(region_path):
if file.startswith("._"):
pass # does nothing
elif file.endswith(".nc"): # check if file is .nc
output_dir = indices_output_dir + '/' + index + '/' + region + '/models/' + model
if "historical" in experiment:
absolute_index(index_in=index, param_in=param, nc_histo_in=file_path, out_dir_in=output_dir)
elif "rcp45" in experiment:
absolute_index(index_in=index, param_in=param, nc_rcp45_in=file_path, out_dir_in=output_dir)
elif "rcp85" in experiment:
absolute_index(index_in=index, param_in=param, nc_rcp85_in=file_path, out_dir_in=output_dir)
def merge_index(periods=False, ts=False):
# loop indices folders until we find the one we want to merge
for index_l, index_path in get_subdirs(indices_output_dir):
if index_l != index_in['name']:
continue
for region, region_path in get_subdirs(index_path):
# loop rcps folders until we find the one we want to merge
for rcp, rcp_path in get_subdirs(region_path):
if rcp != args.rcp:
continue
# for function in index_in['merge_functions']:
# function(cdo=cdo, rcp_path=rcp_path, index=index_in)
if periods:
merge_periods(cdo=cdo, rcp_path=rcp_path, index=index_in)
if ts:
merge_ts(cdo=cdo, rcp_path=rcp_path, index=index_in)
def loop_models():
# loop the required folders (for indices that require two parameters)
for experiment in experiment_set:
# loop of all models inside the experiment folder
for model, model_path in get_subdirs(experiment):
param_path_list = {}
if 'ignore' in index_in and [model, args.rcp] in index_in['ignore']: # this model/rcp should not be used for this index
debug(model+' Ignored ')
continue
# loop of all parameters inside each model
for param, param_path in get_subdirs(model_path):
# if the param is the required one for the index
if param in index_in['param']:
for region, region_path in get_subdirs(param_path):
if 'ignore' in index_in and region in index_in['ignore']:
continue
if region not in param_path_list.keys():
param_path_list[region] = [''] * (len(index_in['param']) + 1) # 1 more for the out_dir
# loop all files inside the param path
for file, file_path in get_subfiles(region_path):
if file.startswith("._"):
continue # does nothing
elif file.endswith(".nc"): # check if file is .nc
output_dir = None
if "rcp45" in experiment:
output_dir = indices_output_dir + '/' + index_in['name'] + '/' + region + '/rcp45/models/' + model
elif "rcp85" in experiment:
output_dir = indices_output_dir + '/' + index_in['name'] + '/' + region + '/rcp85/models/' + model
pathlib.Path(output_dir).mkdir(parents=True, exist_ok=True)
if len(index_in['param']) == 1:
for function in index_in['loop_functions']:
function(index=index_in, cdo=cdo, out_dir=output_dir, nc_in=file_path)
else: # index with multiple param
param_path_list[region][-1] = output_dir
param_path_list[region][index_in['param'].index(param)] = file_path
if len(index_in['param']) > 1:
for region, paths in param_path_list.items():
for function in index_in['loop_functions']:
if paths[-1] != '' and paths[0] != '' and paths[1] != '':
function(index=index_in, cdo=cdo, out_dir=paths[-1], nc_in=paths[0], nc_in2=paths[1])
else:
debug(clean("Missign param for out_dir: " + paths[-1]))
def graph_index():
bounds = [-0.7, -0.6, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7] # 11 colors
colors = ['red', 'firebrick', 'tomato', 'salmon', 'lightcoral',
'lightsalmon', 'papayawhip', 'snow', 'paleturquoise', 'skyblue', 'lightskyblue',
'steelblue', 'dodgerblue', 'blue', 'darkblue']
for index, index_path in get_subdirs(indices_output_dir):
param = index+"ETCCDI"
for region, region_path in get_subdirs(index_path):
for season, season_path in get_subdirs(region_path):
for file, file_path in get_subfiles(season_path):
if file.startswith("._"):
pass
elif file.endswith("sub.nc"): # if files ends with _sub.nc is the result of a substraction, and needs to be graphed
for period in periods:
if period in file:
title = index + " in the " + period + " for season " + season + " in region " + region
plot_basemap_regions(file_path, file_path.replace(".nc", ".png"), param, region, title, cdo, bounds, colors, colors[-1], colors[0])
def merge_index():
for index, index_path in get_subdirs(indices_output_dir):
for region, region_path in get_subdirs(index_path):
for season in seasons:
pathlib.Path(region_path + '/' + season).mkdir(parents=True, exist_ok=True)
nc_ensmean_reference = ''
for period in periods:
index_file_pathlib_list = pathlib.Path(region_path + '/models').rglob('*' + season + '*/*' + period + '*.nc')
index_all_models = ''
for index_file_pathlib in index_file_pathlib_list:
index_file = str(index_file_pathlib)
if (index_file.startswith("._")) or ("regrid" in index_file):
continue # jump to next file
index_file_regrid = index_file.replace('.nc', '_regrid.nc')
target_grid_file = region_path + '/target_grid.nc'
cdo.remapbil(target_grid_file, input=index_file, output=index_file_regrid, froce=False)
index_all_models += index_file_regrid + ' '
if index_all_models == '':
print("No files found for Season %s and period %s" % (season, period))
else:
nc_ensmean_out = region_path + '/' + season + '/' + period + '.nc'
try:
print(nc_ensmean_out)
cdo.ensmean(input=index_all_models, output=nc_ensmean_out, options='-f nc', force=True, returnCdf=False)
if period == "reference":
nc_ensmean_reference = nc_ensmean_out
elif nc_ensmean_reference != '': # substract reference from period. Only for non reference period, if reference file has been found.
nc_ensmean_sub_out = region_path + '/' + season + '/' + period + '_sub.nc'
print(nc_ensmean_sub_out)
cdo.sub(input=nc_ensmean_out + " " + nc_ensmean_reference, output=nc_ensmean_sub_out, options='-f nc', force=True, returnCdf=False)
except CDOException:
print("CDO error")
'''
regionArray = ['Andes', 'Alpine']
boxAndes = [283-1, 288+1, 0, 8.5+1] # long1, long2, lat1, lat2
boxAlpine = [5-1, 14+1, 44.5-1, 48.5+1]
boxesArray = [boxAndes, boxAlpine]
paramArray = []
nc_files_dir = "../nc_files/"
proyect_dir = "RCP8_5_converted/"
max_models = 45
# loop the regionArray and boxesArray together
for region, box in zip(regionArray, boxesArray):
i_models = 0
# loop of all models inside the cmip5 proyect dir
for model, model_path in get_subdirs(nc_files_dir+proyect_dir):
i_models = i_models + 1
if(i_models > max_models):
break
# loop of all parameters inside each model
for param, param_path in get_subdirs(model_path):
# create array of params
if param not in paramArray:
paramArray.append(param) # create an array to plot with all the models saved
# loop all files inside the param path
for file, file_path in get_subfiles(param_path):
if file.endswith(".nc"): # check if file is .nc_files_dir
# ploth the time series of the spatial avg
avg_time_series(file_path, param, region, box, model, False)
break
cdo.degub = True
project_dir = "/Users/danielaquintero/Downloads/tmasmin"
project_dir_converted = "/Users/danielaquintero/Downloads/tmasmin_converted" # "../nc_files/rcp85_pr_daily_converted/"
file_path_aux = "/Users/danielaquintero/Downloads/file_path_aux.nc"
# project_dir = "/Volumes/wd_personal/rcp85_tmasmin"
# project_dir_converted = "../nc_files/rcp85_tmasmin_daily_converted/"
# file_path_aux = "/Volumes/SONY_FAT32/file_path_aux.nc"
max_models = 45
i_models = 0
# model_array = ['EC-EARTH']
# loop of all models inside the cmip5 project dir
for model, model_path in get_subdirs(project_dir):
# if model not in model_array:
# continue
print(model)
i_models = i_models + 1
if(i_models > max_models):
break
model_path_converted = model_path.replace(project_dir, project_dir_converted)
check_and_create(model_path_converted)
# loop of all parameters inside each model
for param, param_path in get_subdirs(model_path): # me da todo lo que hay dentro de la carpeta modelo (osea parametros)
param_path_converted = param_path.replace(project_dir, project_dir_converted)
def merge_ts(cdo, rcp_path, index): # ts= timeseries
from indices_misc import logger, clean
from useful_functions import get_subdirs
from useful_functions import get_subfiles
from indices_graph_functions import plot_time_series
array_all_models = ""
file_path_list = [] # Just to plot all models
array_all_models_avg6190 = ""
for model, model_path in get_subdirs(rcp_path + "/models/"):
for file, file_path in get_subfiles(model_path):
if file.startswith("._"):
continue
elif file.endswith(
"ts.nc"
): # check if end is ts.nc which means it is a time series and needs to be ensembled
if index[
'do_anom']: # if index requires anomalie, before merging, calculate it
nc_avg_61_90 = file_path.replace('_ts.nc', '_avg_61_90.nc')
nc_anomal = file_path.replace('_ts.nc', '_ts_anomal.nc')
year_range = "-selyear,1961/1990"
avg_61_90_val = cdo.timmean(
input=year_range + " " + file_path,
output=nc_avg_61_90,
force=False,
options='-f nc',
returnCdf=True).variables[index['cdo_name']][0, 0, 0]
cdo.subc(avg_61_90_val,
input=file_path,
force=False,
output=nc_anomal
) # substract the timeman of period 61-90
array_all_models += nc_anomal + ' '
file_path_list.append(
nc_anomal) # only to plot individually for debbugging
array_all_models_avg6190 += nc_avg_61_90 + ' '
else: # no anomaly required
array_all_models += file_path + ' '
file_path_list.append(
file_path) # only to plot individually for debbugging
if array_all_models == '':
logger.debug(clean(("No files found in %s" % rcp_path)))
else:
if index['do_anom']:
plot_time_series(index=index,
file_path_in_array=file_path_list,
png_name_in=rcp_path + '/' + index['name'] +
"_allModels_ts_anom.png")
else:
plot_time_series(index,
file_path_list,
png_name_in=rcp_path + '/' + index['name'] +
"_allModels_ts.png")
percentil_array = ["25", "50", "75"] # median = 50th percentil
for percentil in percentil_array:
nc_ensmean_out = rcp_path + '/' + index[
'name'] + '_percent_' + percentil + '_ts.nc'
if percentil == "mean":
cdo.ensmean(input=array_all_models,
output=nc_ensmean_out,
options='-f nc',
force=False,
returnCdf=False)
else:
cdo.enspctl(percentil,
input=array_all_models,
output=nc_ensmean_out,
options='-f nc',
force=False,
returnCdf=False) # Ensemble percentiles
if index['do_anom']:
nc_ensmean_out_medianOfAvg6190 = rcp_path + '/' + index[
'name'] + '_percent_50_ts_medianOfAvg6190.nc'
cdo.ensmean(input=array_all_models_avg6190,
output=nc_ensmean_out_medianOfAvg6190,
options='-f nc',
force=False,
returnCdf=False)
# find anomalie (this result is not really used)
nc_avg_61_90 = nc_ensmean_out.replace('_ts.nc',
'_avg_61_90.nc')
nc_anomal = nc_ensmean_out.replace('_ts.nc', '_ts_anomal.nc')
year_range = "-selyear,1961/1990"
avg_61_90_val = cdo.timmean(
input=year_range + " " + nc_ensmean_out,
output=nc_avg_61_90,
options='-f nc',
force=False,
returnCdf=True).variables[index['cdo_name']][0, 0, 0]
cdo.subc(avg_61_90_val, input=nc_ensmean_out,
output=nc_anomal) # substract il file 61-90
logger.debug(clean((nc_anomal)))
from cdo import *
from useful_functions import get_subdirs
from useful_functions import get_subfiles
from useful_functions import check
import pathlib
import argparse
import sys
nc_dir = "/Users/danielaquintero/Documents/tesis/cmip5_eval/nc_files/"
index_dir = "indices/wsdieca"
cdo = Cdo()
cdo.degub = True
for region, region_path in get_subdirs(nc_dir+index_dir):
for model_rcp85, model_rcp85_path in get_subdirs(region_path+"/rcp85/models/"):
file_rcp85_merge = ""
file_rcp45_merge = ""
for file_rcp85, file_rcp85_path in get_subfiles(model_rcp85_path):
if file_rcp85.startswith("._"):
continue # go to next file
if file_rcp85.endswith("ts.nc"): # check if file is .nc
file_rcp85_merge = file_rcp85_path
model_rcp45_path = model_rcp85_path.replace("rcp85", "rcp45")
if not check(model_rcp45_path):
print(model_rcp45_path)
print("NO RCP45 equivalent\n")
continue
else:
The nc files will be analyzed for different regions (for now two)
'''
nc_files_dir = "/Volumes/wd_tesis/"
proyect_dir = "tasmin_tasmax_historical_converted/"
max_models = 45
i_models = 0
regionArray = []
paramArray = []
# loop of all models inside the cmip5 proyect dir
for model, model_path in get_subdirs(nc_files_dir+proyect_dir):
i_models = i_models + 1
if(i_models > max_models):
break
# loop of all parameters inside each model
for param, param_path in get_subdirs(model_path):
for region, region_path in get_subdirs(param_path):
for subparam, subparam_path in get_subdirs(region_path):
# loop all files inside the param path
for file, file_path in get_subfiles(subparam_path):
if file.endswith("timemax_fldmax_annual.nc"): # check if file is .nc_files_dir
if region in file:
# ploth the time series of the spatial avg
print(file_path)
def graph_map():
import numpy as np
from netCDF4 import Dataset
for index_l, index_path in get_subdirs(indices_output_dir):
if index_l != index_in['name']:
continue
for region, region_path in get_subdirs(index_path):
if 'ignore' in index_in and region in index_in['ignore']:
continue
# first find the minumum and maximum for the index
max_list = []
min_list = []
min_list_ref = []
max_list_ref = []
max_list_rel = []
min_list_rel = []
for rcp, rcp_path in get_subdirs(region_path):
for season, season_path in get_subdirs(rcp_path):
for file, file_path in get_subfiles(season_path):
if file.startswith("._"):
pass
elif (file.endswith("sub.nc") and index_in['do_anom']) or (file.endswith('.nc') and 'reference' not in file and not index_in['do_anom']):
if any(period in file for period in period_name_array):
fh = Dataset(file_path, 'r')
param = fh.variables[index_in['cdo_name']][0:, :, :]
max_list.append(np.amax(param))
min_list.append(np.amin(param))
fh.close()
elif file.endswith("sub_rel.nc"):
if any(period in file for period in period_name_array):
fh = Dataset(file_path, 'r')
param = fh.variables[index_in['cdo_name']][0:, :, :]
max_list_rel.append(np.amax(param))
min_list_rel.append(np.amin(param))
fh.close()
elif 'reference' in file and file.endswith('.nc'):
fh = Dataset(file_path, 'r')
param = fh.variables[index_in['cdo_name']][0:, :, :]
max_list_ref.append(np.amax(param))
min_list_ref.append(np.amin(param))
fh.close()
# The min and max may be extremes that would make the graph hard to read.
# Instead use percentiles.
min_sub = np.percentile(np.array(min_list), index_in['min_perc'])
max_sub = np.percentile(np.array(max_list), index_in['max_perc'])
min_ref = min(min_list_ref)
max_ref = max(max_list_ref)
if not index_in['do_anom']: # for indices with implicit anomaly, it is better to use the same scale for reference period and other periods
min_ref = min_sub
max_ref = max_sub
if (index_in['do_rel']):
min_rel = np.percentile(np.array(min_list_rel), index_in['min_perc_rel'])
max_rel = np.percentile(np.array(max_list_rel), index_in['max_perc_rel'])
# plot
for rcp, rcp_path in get_subdirs(region_path):
if rcp != args.rcp:
continue
for season, season_path in get_subdirs(rcp_path):
season_name = ''
if season != 'ANN':
season_name = " (" + season + ")"
for file, file_path in get_subfiles(season_path):
png_name_latex = indices_graphs_out_dir+"/"+index_l+"/"+region+"/"+rcp+"/"+season+"/"+file.replace('.nc', '.png')
for period in period_name_array:
if period in file:
title = period_title_array[period_name_array.index(period)] + season_name
if 'future' in period:
title += ' (' + str.upper(rcp).replace('5', '.5') + ')'
elif '45' in rcp:
png_name_latex = None # 45 are not needed in latex expect the future one
break
if file.startswith("._"):
pass
elif (file.endswith("sub.nc") and index_in['do_anom']) or (file.endswith('.nc') and 'reference' not in file and not index_in['do_anom']):
plot_basemap_regions(index_in, file_path, file_path.replace(".nc", ".png"), region, title, min_sub, max_sub, png_name_latex)
elif 'reference' in file and file.endswith('.nc'):
plot_basemap_regions(index_in, file_path, file_path.replace(".nc", ".png"), region, title, min_ref, max_ref, None)
elif 'ignore' in index_in and region in index_in['ignore']:
continue
elif file.endswith("sub_rel.nc") and index_in['do_rel']:
if 'future' not in file:
png_name_latex = None # rel graphs only needed for future
plot_basemap_regions(index_in, file_path, file_path.replace(".nc", ".png"), region, title, min_rel, max_rel, png_name_latex)
def graph_ts():
import numpy as np
from netCDF4 import Dataset
for index_l, index_path in get_subdirs(indices_output_dir):
if index_l != index_in['name']:
continue
# first find the minumum and maximum for the index
max_list_models = []
min_list_models = []
max_list = {}
min_list = {}
avg6190 = {}
models_plot = {}
for region, region_path in get_subdirs(index_path):
if 'ignore' in index_in and region in index_in['ignore']:
continue
avg6190[region] = None
models_plot[region] = []
max_list[region] = []
min_list[region] = []
for rcp, rcp_path in get_subdirs(region_path):
for file, file_path in get_subfiles(rcp_path):
if file.startswith("._"):
continue
elif file.endswith("ts.nc"):
fh = Dataset(file_path, 'r')
param = fh.variables[index_in['cdo_name']][:, 0, 0]
max_list[region].append(np.amax(param))
min_list[region].append(np.amin(param))
fh.close()
elif file.endswith("Avg6190.nc"):
fh = Dataset(file_path, 'r')
avg6190[region] = fh.variables[index_in['cdo_name']][0, 0, 0]
fh.close()
# all models
for model, model_path in get_subdirs(rcp_path+"/models/"):
for file, file_path in get_subfiles(model_path):
if file.startswith("._"):
continue
elif (file.endswith("ts_anomal.nc") and index_in['do_anom']) or (file.endswith('ts.nc') and not index_in['do_anom']):
fh = Dataset(file_path, 'r')
param = fh.variables[index_in['cdo_name']][:, 0, 0]
max_list_models.append(np.amax(param))
min_list_models.append(np.amin(param))
fh.close()
models_plot[region].append(file_path)
# plot
for region, region_path in get_subdirs(index_path):
if 'ignore' in index_in and region in index_in['ignore']:
continue
files_to_plot = []
for rcp, rcp_path in get_subdirs(region_path):
for file, file_path in get_subfiles(rcp_path):
if file.startswith("._"):
continue
elif file.endswith("ts.nc"):
files_to_plot.append(file_path)
debug(clean((file_path)))
plot_time_series(index_in, files_to_plot, models_plot_array=models_plot[region], region=region,
png_name_in=region_path+"/"+index_in['name']+'_'+region+'_Models_ts.png',
png_name_latex=indices_graphs_out_dir+"/"+index_l+"/"+region+"/"+index_in['name']+'_'+region+'_Models_ts.png',
min=min(min_list[region]), max=max(max_list[region]), avg6190=avg6190[region])
plot_time_series(index_in, files_to_plot, region=region,
png_name_in=region_path+"/"+index_in['name']+'_'+region+'_ts.png', min=min(min_list[region]), max=max(max_list[region]), avg6190=avg6190[region])
elif args.rcp85:
rcp_path = rcp85_path
merge_path = histo_rcp85_merge
else:
parser.print_help()
sys.exit()
else:
parser.print_help()
sys.exit()
cdo = Cdo()
if args.verb:
cdo.degub = True
for model_histo, model_histo_path in get_subdirs(nc_dir + histo_path):
# loop of all parameters inside each model
for param_histo, param_histo_path in get_subdirs(model_histo_path):
# loop all regions inside each parameter
for region_histo, region_histo_path in get_subdirs(param_histo_path):
# loop all files inside the param path
file_histo_merge = ""
file_rcp_merge = ""
for file_histo, file_histo_path in get_subfiles(region_histo_path):
if file_histo.startswith("._"):
continue # go to next file
if file_histo.endswith(".nc"): # check if file is .nc
file_histo_merge = file_histo_path
region_rcp_path = region_histo_path.replace(histo_path, rcp_path)
