def generate_periods(cdo, index, out_dir, nc_in=None, nc_in2=None):
'''
Generate a map of the index, for the required time periods, starting from matrix
'''
from indices_misc import logger, clean
from indices_misc import period_range_array, period_name_array
from useful_functions import get_subfiles
import pathlib
for season in index['seasons']:
for year_range, name in zip(period_range_array, period_name_array):
cdo_year_command = "-selyear," + year_range
for file, file_path in get_subfiles(out_dir):
if file.startswith("._"):
continue # does nothing
elif file.endswith(
index['name'] + ".nc"
): # if file ends with index.nc, it is the pure idex file, with matrix values (temp and field)
pathlib.Path(out_dir + "/" + season + "/").mkdir(
parents=True, exist_ok=True)
nc_out = out_dir + "/" + season + "/" + pathlib.Path(
file_path).stem + "_" + name + ".nc"
logger.debug(clean((nc_out)))
logger.debug(clean(("")))
cdo.timmean(input="-setreftime,1850-01-01,00:00:00 " +
cdo_year_command + " " + file_path,
output=nc_out,
options='-f nc',
force=False,
returnCdf=False)
def loop_models():
# loop all the index we want to obtain, and the required param for that index
for index, req_param in zip(index_array, req_param_array):
print("Calculate %s using %s" % (index, req_param))
# loop the experiment folders we have
for experiment in experiment_array:
# loop of all models inside the experiment folder
for model, model_path in get_subdirs(experiment):
# loop of all parameters inside each model
for param, param_path in get_subdirs(model_path):
# if the param is the required one for the index
if param == req_param:
for region, region_path in get_subdirs(param_path):
# loop all files inside the param path
for file, file_path in get_subfiles(region_path):
if file.startswith("._"):
pass # does nothing
elif file.endswith(".nc"): # check if file is .nc
output_dir = indices_output_dir + '/' + index + '/' + region + '/models/' + model
if "historical" in experiment:
absolute_index(index_in=index, param_in=param, nc_histo_in=file_path, out_dir_in=output_dir)
elif "rcp45" in experiment:
absolute_index(index_in=index, param_in=param, nc_rcp45_in=file_path, out_dir_in=output_dir)
elif "rcp85" in experiment:
absolute_index(index_in=index, param_in=param, nc_rcp85_in=file_path, out_dir_in=output_dir)
def graph_bar():
from netCDF4 import Dataset
for index_l, index_path in get_subdirs(indices_output_dir):
if index_l != index_in['name']:
continue
avg6190 = {}
files_to_plot = {}
for region, region_path in get_subdirs(index_path):
if 'ignore' in index_in and region in index_in['ignore']:
continue
avg6190[region] = None
files_to_plot[region] = []
for rcp, rcp_path in get_subdirs(region_path):
for file, file_path in get_subfiles(rcp_path):
if file.startswith("._"):
continue
elif file.endswith("ts.nc"):
files_to_plot[region].append(file_path)
elif file.endswith("Avg6190.nc"):
fh = Dataset(file_path, 'r')
avg6190[region] = fh.variables[index_in['cdo_name']][0, 0, 0]
fh.close()
plot_bar_index(index=index_in, files=files_to_plot, png_name_in=indices_graphs_out_dir+"/"+index_l+"/"+index_in['name']+'_bar.png', avg6190=avg6190)
def delete_days(cdo, index, out_dir, nc_in, nc_in2=None):
'''
Delete wrongly generated days in rx5day index monthly
'''
from indices_misc import logger, clean
from useful_functions import get_subfiles
import pathlib
from os import rename
for file, file_path in get_subfiles(out_dir):
if file.startswith("._"):
continue # does nothing
elif file.endswith(
index['name'] + ".nc"
): # if file ends with index.nc, it is the pure index file
nc_out_days = out_dir + "/" + pathlib.Path(
file_path).stem + "_withDays.nc"
rename(file_path, nc_out_days)
logger.debug(clean((file_path)))
cdo.delete("day=17",
input="-delete,month=6,hour=12 " + nc_out_days,
output=file_path,
options='-f nc',
force=False,
returnCdf=False)
def loop_models():
# loop the required folders (for indices that require two parameters)
for experiment in experiment_set:
# loop of all models inside the experiment folder
for model, model_path in get_subdirs(experiment):
param_path_list = {}
if 'ignore' in index_in and [model, args.rcp] in index_in['ignore']: # this model/rcp should not be used for this index
debug(model+' Ignored ')
continue
# loop of all parameters inside each model
for param, param_path in get_subdirs(model_path):
# if the param is the required one for the index
if param in index_in['param']:
for region, region_path in get_subdirs(param_path):
if 'ignore' in index_in and region in index_in['ignore']:
continue
if region not in param_path_list.keys():
param_path_list[region] = [''] * (len(index_in['param']) + 1) # 1 more for the out_dir
# loop all files inside the param path
for file, file_path in get_subfiles(region_path):
if file.startswith("._"):
continue # does nothing
elif file.endswith(".nc"): # check if file is .nc
output_dir = None
if "rcp45" in experiment:
output_dir = indices_output_dir + '/' + index_in['name'] + '/' + region + '/rcp45/models/' + model
elif "rcp85" in experiment:
output_dir = indices_output_dir + '/' + index_in['name'] + '/' + region + '/rcp85/models/' + model
pathlib.Path(output_dir).mkdir(parents=True, exist_ok=True)
if len(index_in['param']) == 1:
for function in index_in['loop_functions']:
function(index=index_in, cdo=cdo, out_dir=output_dir, nc_in=file_path)
else: # index with multiple param
param_path_list[region][-1] = output_dir
param_path_list[region][index_in['param'].index(param)] = file_path
if len(index_in['param']) > 1:
for region, paths in param_path_list.items():
for function in index_in['loop_functions']:
if paths[-1] != '' and paths[0] != '' and paths[1] != '':
function(index=index_in, cdo=cdo, out_dir=paths[-1], nc_in=paths[0], nc_in2=paths[1])
else:
debug(clean("Missign param for out_dir: " + paths[-1]))
def graph_index():
bounds = [-0.7, -0.6, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7] # 11 colors
colors = ['red', 'firebrick', 'tomato', 'salmon', 'lightcoral',
'lightsalmon', 'papayawhip', 'snow', 'paleturquoise', 'skyblue', 'lightskyblue',
'steelblue', 'dodgerblue', 'blue', 'darkblue']
for index, index_path in get_subdirs(indices_output_dir):
param = index+"ETCCDI"
for region, region_path in get_subdirs(index_path):
for season, season_path in get_subdirs(region_path):
for file, file_path in get_subfiles(season_path):
if file.startswith("._"):
pass
elif file.endswith("sub.nc"): # if files ends with _sub.nc is the result of a substraction, and needs to be graphed
for period in periods:
if period in file:
title = index + " in the " + period + " for season " + season + " in region " + region
plot_basemap_regions(file_path, file_path.replace(".nc", ".png"), param, region, title, cdo, bounds, colors, colors[-1], colors[0])
def generate_ts(cdo, index, out_dir, nc_in=None, nc_in2=None):
'''
Generate timeseries of the index, for the whole area, starting from matrix
'''
from indices_misc import logger, clean
from useful_functions import get_subfiles
import pathlib
for file, file_path in get_subfiles(out_dir):
if file.startswith("._"):
continue # does nothing
elif file.endswith(
index['name'] + ".nc"
): # if file ends with index.nc, it is the pure idex file, with matrix values (temp and field)
nc_out = out_dir + "/" + pathlib.Path(file_path).stem + "_ts.nc"
logger.debug(clean((nc_out)))
logger.debug(clean(("")))
cdo.fldmean(input="-setreftime,1850-01-01,00:00:00 " + file_path,
output=nc_out,
options='-f nc',
force=False,
returnCdf=False)
# loop the regionArray and boxesArray together
for region, box in zip(regionArray, boxesArray):
i_models = 0
# loop of all models inside the cmip5 proyect dir
for model, model_path in get_subdirs(nc_files_dir+proyect_dir):
i_models = i_models + 1
if(i_models > max_models):
break
# loop of all parameters inside each model
for param, param_path in get_subdirs(model_path):
# create array of params
if param not in paramArray:
paramArray.append(param) # create an array to plot with all the models saved
# loop all files inside the param path
for file, file_path in get_subfiles(param_path):
if file.endswith(".nc"): # check if file is .nc_files_dir
# ploth the time series of the spatial avg
avg_time_series(file_path, param, region, box, model, False)
break
# https://matplotlib.org/examples/color/colormaps_reference.html
colormap = plt.cm.tab20
# after ploting all models, set colors and legend for all the figures
modelsHigh = ['MIROC4h', 'CESM1_BGC', 'CESM1_CAM5', 'CESM1_FASTCHEM',
'CCSM4', 'CNRM-CM5', 'CNRM-CM5-2', 'EC-EARTH',
'CMCC-CM', 'MIROC5', 'MRI-CGCM3', 'MRI-ESM1']
modelsHighColors = ['red', 'royalblue', 'black', 'rebeccapurple',
'brown', 'cyan', 'darkorange', 'purple',
break
model_path_converted = model_path.replace(project_dir, project_dir_converted)
check_and_create(model_path_converted)
# loop of all parameters inside each model
for param, param_path in get_subdirs(model_path): # me da todo lo que hay dentro de la carpeta modelo (osea parametros)
param_path_converted = param_path.replace(project_dir, project_dir_converted)
check_and_create(param_path_converted)
for region, region_path in get_subdirs(param_path):
region_path_converted = region_path.replace(project_dir, project_dir_converted)
check_and_create(region_path_converted)
# loop all files inside the param path
for file, file_path in get_subfiles(region_path):
if file.startswith("._"):
pass # does nothing
# print("Starts with ._" , file)
elif file.endswith(".nc"): # check if file is .nc
file_path_converted = file_path.replace(project_dir, project_dir_converted)
if os.path.exists(file_path_converted):
print("Aready exists", file_path_converted)
else:
print("Create", file_path_converted)
# try:
if (param == 'tas') or (param == 'tasmax') or (param == 'tasmin'):
convertTime(cdo, file_path, file_path_converted)
for region in regionArray:
i_models = 0
# loop of all models inside the cmip5 proyect dir
for model, model_path in get_subdirs(nc_files_dir+proyect_dir):
i_models = i_models + 1
if(i_models > max_models):
break
# loop of all parameters inside each model
for param, param_path in get_subdirs(model_path):
# create array of params
if param not in paramArray:
paramArray.append(param)
# loop all files inside the param path
for file, file_path in get_subfiles(param_path+'/avg/'):
if file.endswith("anomal.nc"): # check if file is .nc_files_dir
if region in file:
# ploth the time series of the spatial avg
file_path_normal = file_path.replace("anomal", "ymean" )
data_anom = Dataset(file_path, mode='r')
data_norm = Dataset(file_path_normal, mode='r')
plot_time_series(data_anom, param, region)
plot_time_series(data_norm, param, region)
data_anom.close()
data_norm.close()
# https://matplotlib.org/examples/color/colormaps_reference.html
colormap = plt.cm.tab20
# after ploting all models, set colors and legend for all the figures
def merge_ts(cdo, rcp_path, index): # ts= timeseries
from indices_misc import logger, clean
from useful_functions import get_subdirs
from useful_functions import get_subfiles
from indices_graph_functions import plot_time_series
array_all_models = ""
file_path_list = [] # Just to plot all models
array_all_models_avg6190 = ""
for model, model_path in get_subdirs(rcp_path + "/models/"):
for file, file_path in get_subfiles(model_path):
if file.startswith("._"):
continue
elif file.endswith(
"ts.nc"
): # check if end is ts.nc which means it is a time series and needs to be ensembled
if index[
'do_anom']: # if index requires anomalie, before merging, calculate it
nc_avg_61_90 = file_path.replace('_ts.nc', '_avg_61_90.nc')
nc_anomal = file_path.replace('_ts.nc', '_ts_anomal.nc')
year_range = "-selyear,1961/1990"
avg_61_90_val = cdo.timmean(
input=year_range + " " + file_path,
output=nc_avg_61_90,
force=False,
options='-f nc',
returnCdf=True).variables[index['cdo_name']][0, 0, 0]
cdo.subc(avg_61_90_val,
input=file_path,
force=False,
output=nc_anomal
) # substract the timeman of period 61-90
array_all_models += nc_anomal + ' '
file_path_list.append(
nc_anomal) # only to plot individually for debbugging
array_all_models_avg6190 += nc_avg_61_90 + ' '
else: # no anomaly required
array_all_models += file_path + ' '
file_path_list.append(
file_path) # only to plot individually for debbugging
if array_all_models == '':
logger.debug(clean(("No files found in %s" % rcp_path)))
else:
if index['do_anom']:
plot_time_series(index=index,
file_path_in_array=file_path_list,
png_name_in=rcp_path + '/' + index['name'] +
"_allModels_ts_anom.png")
else:
plot_time_series(index,
file_path_list,
png_name_in=rcp_path + '/' + index['name'] +
"_allModels_ts.png")
percentil_array = ["25", "50", "75"] # median = 50th percentil
for percentil in percentil_array:
nc_ensmean_out = rcp_path + '/' + index[
'name'] + '_percent_' + percentil + '_ts.nc'
if percentil == "mean":
cdo.ensmean(input=array_all_models,
output=nc_ensmean_out,
options='-f nc',
force=False,
returnCdf=False)
else:
cdo.enspctl(percentil,
input=array_all_models,
output=nc_ensmean_out,
options='-f nc',
force=False,
returnCdf=False) # Ensemble percentiles
if index['do_anom']:
nc_ensmean_out_medianOfAvg6190 = rcp_path + '/' + index[
'name'] + '_percent_50_ts_medianOfAvg6190.nc'
cdo.ensmean(input=array_all_models_avg6190,
output=nc_ensmean_out_medianOfAvg6190,
options='-f nc',
force=False,
returnCdf=False)
# find anomalie (this result is not really used)
nc_avg_61_90 = nc_ensmean_out.replace('_ts.nc',
'_avg_61_90.nc')
nc_anomal = nc_ensmean_out.replace('_ts.nc', '_ts_anomal.nc')
year_range = "-selyear,1961/1990"
avg_61_90_val = cdo.timmean(
input=year_range + " " + nc_ensmean_out,
output=nc_avg_61_90,
options='-f nc',
force=False,
returnCdf=True).variables[index['cdo_name']][0, 0, 0]
cdo.subc(avg_61_90_val, input=nc_ensmean_out,
output=nc_anomal) # substract il file 61-90
logger.debug(clean((nc_anomal)))
from useful_functions import check
import pathlib
import argparse
import sys
nc_dir = "/Users/danielaquintero/Documents/tesis/cmip5_eval/nc_files/"
index_dir = "indices/wsdieca"
cdo = Cdo()
cdo.degub = True
for region, region_path in get_subdirs(nc_dir+index_dir):
for model_rcp85, model_rcp85_path in get_subdirs(region_path+"/rcp85/models/"):
file_rcp85_merge = ""
file_rcp45_merge = ""
for file_rcp85, file_rcp85_path in get_subfiles(model_rcp85_path):
if file_rcp85.startswith("._"):
continue # go to next file
if file_rcp85.endswith("ts.nc"): # check if file is .nc
file_rcp85_merge = file_rcp85_path
model_rcp45_path = model_rcp85_path.replace("rcp85", "rcp45")
if not check(model_rcp45_path):
print(model_rcp45_path)
print("NO RCP45 equivalent\n")
continue
else:
for file_rcp45, file_rcp45_path in get_subfiles(model_rcp45_path):
if file_rcp45.startswith("._"):
continue # go to next file
def graph_map():
import numpy as np
from netCDF4 import Dataset
for index_l, index_path in get_subdirs(indices_output_dir):
if index_l != index_in['name']:
continue
for region, region_path in get_subdirs(index_path):
if 'ignore' in index_in and region in index_in['ignore']:
continue
# first find the minumum and maximum for the index
max_list = []
min_list = []
min_list_ref = []
max_list_ref = []
max_list_rel = []
min_list_rel = []
for rcp, rcp_path in get_subdirs(region_path):
for season, season_path in get_subdirs(rcp_path):
for file, file_path in get_subfiles(season_path):
if file.startswith("._"):
pass
elif (file.endswith("sub.nc") and index_in['do_anom']) or (file.endswith('.nc') and 'reference' not in file and not index_in['do_anom']):
if any(period in file for period in period_name_array):
fh = Dataset(file_path, 'r')
param = fh.variables[index_in['cdo_name']][0:, :, :]
max_list.append(np.amax(param))
min_list.append(np.amin(param))
fh.close()
elif file.endswith("sub_rel.nc"):
if any(period in file for period in period_name_array):
fh = Dataset(file_path, 'r')
param = fh.variables[index_in['cdo_name']][0:, :, :]
max_list_rel.append(np.amax(param))
min_list_rel.append(np.amin(param))
fh.close()
elif 'reference' in file and file.endswith('.nc'):
fh = Dataset(file_path, 'r')
param = fh.variables[index_in['cdo_name']][0:, :, :]
max_list_ref.append(np.amax(param))
min_list_ref.append(np.amin(param))
fh.close()
# The min and max may be extremes that would make the graph hard to read.
# Instead use percentiles.
min_sub = np.percentile(np.array(min_list), index_in['min_perc'])
max_sub = np.percentile(np.array(max_list), index_in['max_perc'])
min_ref = min(min_list_ref)
max_ref = max(max_list_ref)
if not index_in['do_anom']: # for indices with implicit anomaly, it is better to use the same scale for reference period and other periods
min_ref = min_sub
max_ref = max_sub
if (index_in['do_rel']):
min_rel = np.percentile(np.array(min_list_rel), index_in['min_perc_rel'])
max_rel = np.percentile(np.array(max_list_rel), index_in['max_perc_rel'])
# plot
for rcp, rcp_path in get_subdirs(region_path):
if rcp != args.rcp:
continue
for season, season_path in get_subdirs(rcp_path):
season_name = ''
if season != 'ANN':
season_name = " (" + season + ")"
for file, file_path in get_subfiles(season_path):
png_name_latex = indices_graphs_out_dir+"/"+index_l+"/"+region+"/"+rcp+"/"+season+"/"+file.replace('.nc', '.png')
for period in period_name_array:
if period in file:
title = period_title_array[period_name_array.index(period)] + season_name
if 'future' in period:
title += ' (' + str.upper(rcp).replace('5', '.5') + ')'
elif '45' in rcp:
png_name_latex = None # 45 are not needed in latex expect the future one
break
if file.startswith("._"):
pass
elif (file.endswith("sub.nc") and index_in['do_anom']) or (file.endswith('.nc') and 'reference' not in file and not index_in['do_anom']):
plot_basemap_regions(index_in, file_path, file_path.replace(".nc", ".png"), region, title, min_sub, max_sub, png_name_latex)
elif 'reference' in file and file.endswith('.nc'):
plot_basemap_regions(index_in, file_path, file_path.replace(".nc", ".png"), region, title, min_ref, max_ref, None)
elif 'ignore' in index_in and region in index_in['ignore']:
continue
elif file.endswith("sub_rel.nc") and index_in['do_rel']:
if 'future' not in file:
png_name_latex = None # rel graphs only needed for future
plot_basemap_regions(index_in, file_path, file_path.replace(".nc", ".png"), region, title, min_rel, max_rel, png_name_latex)
def graph_ts():
import numpy as np
from netCDF4 import Dataset
for index_l, index_path in get_subdirs(indices_output_dir):
if index_l != index_in['name']:
continue
# first find the minumum and maximum for the index
max_list_models = []
min_list_models = []
max_list = {}
min_list = {}
avg6190 = {}
models_plot = {}
for region, region_path in get_subdirs(index_path):
if 'ignore' in index_in and region in index_in['ignore']:
continue
avg6190[region] = None
models_plot[region] = []
max_list[region] = []
min_list[region] = []
for rcp, rcp_path in get_subdirs(region_path):
for file, file_path in get_subfiles(rcp_path):
if file.startswith("._"):
continue
elif file.endswith("ts.nc"):
fh = Dataset(file_path, 'r')
param = fh.variables[index_in['cdo_name']][:, 0, 0]
max_list[region].append(np.amax(param))
min_list[region].append(np.amin(param))
fh.close()
elif file.endswith("Avg6190.nc"):
fh = Dataset(file_path, 'r')
avg6190[region] = fh.variables[index_in['cdo_name']][0, 0, 0]
fh.close()
# all models
for model, model_path in get_subdirs(rcp_path+"/models/"):
for file, file_path in get_subfiles(model_path):
if file.startswith("._"):
continue
elif (file.endswith("ts_anomal.nc") and index_in['do_anom']) or (file.endswith('ts.nc') and not index_in['do_anom']):
fh = Dataset(file_path, 'r')
param = fh.variables[index_in['cdo_name']][:, 0, 0]
max_list_models.append(np.amax(param))
min_list_models.append(np.amin(param))
fh.close()
models_plot[region].append(file_path)
# plot
for region, region_path in get_subdirs(index_path):
if 'ignore' in index_in and region in index_in['ignore']:
continue
files_to_plot = []
for rcp, rcp_path in get_subdirs(region_path):
for file, file_path in get_subfiles(rcp_path):
if file.startswith("._"):
continue
elif file.endswith("ts.nc"):
files_to_plot.append(file_path)
debug(clean((file_path)))
plot_time_series(index_in, files_to_plot, models_plot_array=models_plot[region], region=region,
png_name_in=region_path+"/"+index_in['name']+'_'+region+'_Models_ts.png',
png_name_latex=indices_graphs_out_dir+"/"+index_l+"/"+region+"/"+index_in['name']+'_'+region+'_Models_ts.png',
min=min(min_list[region]), max=max(max_list[region]), avg6190=avg6190[region])
plot_time_series(index_in, files_to_plot, region=region,
png_name_in=region_path+"/"+index_in['name']+'_'+region+'_ts.png', min=min(min_list[region]), max=max(max_list[region]), avg6190=avg6190[region])
parser.print_help()
sys.exit()
cdo = Cdo()
if args.verb:
cdo.degub = True
for model_histo, model_histo_path in get_subdirs(nc_dir + histo_path):
# loop of all parameters inside each model
for param_histo, param_histo_path in get_subdirs(model_histo_path):
# loop all regions inside each parameter
for region_histo, region_histo_path in get_subdirs(param_histo_path):
# loop all files inside the param path
file_histo_merge = ""
file_rcp_merge = ""
for file_histo, file_histo_path in get_subfiles(region_histo_path):
if file_histo.startswith("._"):
continue # go to next file
if file_histo.endswith(".nc"): # check if file is .nc
file_histo_merge = file_histo_path
region_rcp_path = region_histo_path.replace(histo_path, rcp_path)
if not check(region_rcp_path):
print(file_histo_merge)
print("NO RCP equivalent\n")
continue
else:
for file_rcp, file_rcp_path in get_subfiles(region_rcp_path):
if file_rcp.startswith("._"):
continue # go to next file
