def generate_periods(cdo, index, out_dir, nc_in=None, nc_in2=None): ''' Generate a map of the index, for the required time periods, starting from matrix ''' from indices_misc import logger, clean from indices_misc import period_range_array, period_name_array from useful_functions import get_subfiles import pathlib for season in index['seasons']: for year_range, name in zip(period_range_array, period_name_array): cdo_year_command = "-selyear," + year_range for file, file_path in get_subfiles(out_dir): if file.startswith("._"): continue # does nothing elif file.endswith( index['name'] + ".nc" ): # if file ends with index.nc, it is the pure idex file, with matrix values (temp and field) pathlib.Path(out_dir + "/" + season + "/").mkdir( parents=True, exist_ok=True) nc_out = out_dir + "/" + season + "/" + pathlib.Path( file_path).stem + "_" + name + ".nc" logger.debug(clean((nc_out))) logger.debug(clean((""))) cdo.timmean(input="-setreftime,1850-01-01,00:00:00 " + cdo_year_command + " " + file_path, output=nc_out, options='-f nc', force=False, returnCdf=False)
def loop_models(): # loop all the index we want to obtain, and the required param for that index for index, req_param in zip(index_array, req_param_array): print("Calculate %s using %s" % (index, req_param)) # loop the experiment folders we have for experiment in experiment_array: # loop of all models inside the experiment folder for model, model_path in get_subdirs(experiment): # loop of all parameters inside each model for param, param_path in get_subdirs(model_path): # if the param is the required one for the index if param == req_param: for region, region_path in get_subdirs(param_path): # loop all files inside the param path for file, file_path in get_subfiles(region_path): if file.startswith("._"): pass # does nothing elif file.endswith(".nc"): # check if file is .nc output_dir = indices_output_dir + '/' + index + '/' + region + '/models/' + model if "historical" in experiment: absolute_index(index_in=index, param_in=param, nc_histo_in=file_path, out_dir_in=output_dir) elif "rcp45" in experiment: absolute_index(index_in=index, param_in=param, nc_rcp45_in=file_path, out_dir_in=output_dir) elif "rcp85" in experiment: absolute_index(index_in=index, param_in=param, nc_rcp85_in=file_path, out_dir_in=output_dir)
def graph_bar(): from netCDF4 import Dataset for index_l, index_path in get_subdirs(indices_output_dir): if index_l != index_in['name']: continue avg6190 = {} files_to_plot = {} for region, region_path in get_subdirs(index_path): if 'ignore' in index_in and region in index_in['ignore']: continue avg6190[region] = None files_to_plot[region] = [] for rcp, rcp_path in get_subdirs(region_path): for file, file_path in get_subfiles(rcp_path): if file.startswith("._"): continue elif file.endswith("ts.nc"): files_to_plot[region].append(file_path) elif file.endswith("Avg6190.nc"): fh = Dataset(file_path, 'r') avg6190[region] = fh.variables[index_in['cdo_name']][0, 0, 0] fh.close() plot_bar_index(index=index_in, files=files_to_plot, png_name_in=indices_graphs_out_dir+"/"+index_l+"/"+index_in['name']+'_bar.png', avg6190=avg6190)
def delete_days(cdo, index, out_dir, nc_in, nc_in2=None): ''' Delete wrongly generated days in rx5day index monthly ''' from indices_misc import logger, clean from useful_functions import get_subfiles import pathlib from os import rename for file, file_path in get_subfiles(out_dir): if file.startswith("._"): continue # does nothing elif file.endswith( index['name'] + ".nc" ): # if file ends with index.nc, it is the pure index file nc_out_days = out_dir + "/" + pathlib.Path( file_path).stem + "_withDays.nc" rename(file_path, nc_out_days) logger.debug(clean((file_path))) cdo.delete("day=17", input="-delete,month=6,hour=12 " + nc_out_days, output=file_path, options='-f nc', force=False, returnCdf=False)
def loop_models(): # loop the required folders (for indices that require two parameters) for experiment in experiment_set: # loop of all models inside the experiment folder for model, model_path in get_subdirs(experiment): param_path_list = {} if 'ignore' in index_in and [model, args.rcp] in index_in['ignore']: # this model/rcp should not be used for this index debug(model+' Ignored ') continue # loop of all parameters inside each model for param, param_path in get_subdirs(model_path): # if the param is the required one for the index if param in index_in['param']: for region, region_path in get_subdirs(param_path): if 'ignore' in index_in and region in index_in['ignore']: continue if region not in param_path_list.keys(): param_path_list[region] = [''] * (len(index_in['param']) + 1) # 1 more for the out_dir # loop all files inside the param path for file, file_path in get_subfiles(region_path): if file.startswith("._"): continue # does nothing elif file.endswith(".nc"): # check if file is .nc output_dir = None if "rcp45" in experiment: output_dir = indices_output_dir + '/' + index_in['name'] + '/' + region + '/rcp45/models/' + model elif "rcp85" in experiment: output_dir = indices_output_dir + '/' + index_in['name'] + '/' + region + '/rcp85/models/' + model pathlib.Path(output_dir).mkdir(parents=True, exist_ok=True) if len(index_in['param']) == 1: for function in index_in['loop_functions']: function(index=index_in, cdo=cdo, out_dir=output_dir, nc_in=file_path) else: # index with multiple param param_path_list[region][-1] = output_dir param_path_list[region][index_in['param'].index(param)] = file_path if len(index_in['param']) > 1: for region, paths in param_path_list.items(): for function in index_in['loop_functions']: if paths[-1] != '' and paths[0] != '' and paths[1] != '': function(index=index_in, cdo=cdo, out_dir=paths[-1], nc_in=paths[0], nc_in2=paths[1]) else: debug(clean("Missign param for out_dir: " + paths[-1]))
def graph_index(): bounds = [-0.7, -0.6, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7] # 11 colors colors = ['red', 'firebrick', 'tomato', 'salmon', 'lightcoral', 'lightsalmon', 'papayawhip', 'snow', 'paleturquoise', 'skyblue', 'lightskyblue', 'steelblue', 'dodgerblue', 'blue', 'darkblue'] for index, index_path in get_subdirs(indices_output_dir): param = index+"ETCCDI" for region, region_path in get_subdirs(index_path): for season, season_path in get_subdirs(region_path): for file, file_path in get_subfiles(season_path): if file.startswith("._"): pass elif file.endswith("sub.nc"): # if files ends with _sub.nc is the result of a substraction, and needs to be graphed for period in periods: if period in file: title = index + " in the " + period + " for season " + season + " in region " + region plot_basemap_regions(file_path, file_path.replace(".nc", ".png"), param, region, title, cdo, bounds, colors, colors[-1], colors[0])
def generate_ts(cdo, index, out_dir, nc_in=None, nc_in2=None): ''' Generate timeseries of the index, for the whole area, starting from matrix ''' from indices_misc import logger, clean from useful_functions import get_subfiles import pathlib for file, file_path in get_subfiles(out_dir): if file.startswith("._"): continue # does nothing elif file.endswith( index['name'] + ".nc" ): # if file ends with index.nc, it is the pure idex file, with matrix values (temp and field) nc_out = out_dir + "/" + pathlib.Path(file_path).stem + "_ts.nc" logger.debug(clean((nc_out))) logger.debug(clean((""))) cdo.fldmean(input="-setreftime,1850-01-01,00:00:00 " + file_path, output=nc_out, options='-f nc', force=False, returnCdf=False)
# loop the regionArray and boxesArray together for region, box in zip(regionArray, boxesArray): i_models = 0 # loop of all models inside the cmip5 proyect dir for model, model_path in get_subdirs(nc_files_dir+proyect_dir): i_models = i_models + 1 if(i_models > max_models): break # loop of all parameters inside each model for param, param_path in get_subdirs(model_path): # create array of params if param not in paramArray: paramArray.append(param) # create an array to plot with all the models saved # loop all files inside the param path for file, file_path in get_subfiles(param_path): if file.endswith(".nc"): # check if file is .nc_files_dir # ploth the time series of the spatial avg avg_time_series(file_path, param, region, box, model, False) break # https://matplotlib.org/examples/color/colormaps_reference.html colormap = plt.cm.tab20 # after ploting all models, set colors and legend for all the figures modelsHigh = ['MIROC4h', 'CESM1_BGC', 'CESM1_CAM5', 'CESM1_FASTCHEM', 'CCSM4', 'CNRM-CM5', 'CNRM-CM5-2', 'EC-EARTH', 'CMCC-CM', 'MIROC5', 'MRI-CGCM3', 'MRI-ESM1'] modelsHighColors = ['red', 'royalblue', 'black', 'rebeccapurple', 'brown', 'cyan', 'darkorange', 'purple',
break model_path_converted = model_path.replace(project_dir, project_dir_converted) check_and_create(model_path_converted) # loop of all parameters inside each model for param, param_path in get_subdirs(model_path): # me da todo lo que hay dentro de la carpeta modelo (osea parametros) param_path_converted = param_path.replace(project_dir, project_dir_converted) check_and_create(param_path_converted) for region, region_path in get_subdirs(param_path): region_path_converted = region_path.replace(project_dir, project_dir_converted) check_and_create(region_path_converted) # loop all files inside the param path for file, file_path in get_subfiles(region_path): if file.startswith("._"): pass # does nothing # print("Starts with ._" , file) elif file.endswith(".nc"): # check if file is .nc file_path_converted = file_path.replace(project_dir, project_dir_converted) if os.path.exists(file_path_converted): print("Aready exists", file_path_converted) else: print("Create", file_path_converted) # try: if (param == 'tas') or (param == 'tasmax') or (param == 'tasmin'): convertTime(cdo, file_path, file_path_converted)
for region in regionArray: i_models = 0 # loop of all models inside the cmip5 proyect dir for model, model_path in get_subdirs(nc_files_dir+proyect_dir): i_models = i_models + 1 if(i_models > max_models): break # loop of all parameters inside each model for param, param_path in get_subdirs(model_path): # create array of params if param not in paramArray: paramArray.append(param) # loop all files inside the param path for file, file_path in get_subfiles(param_path+'/avg/'): if file.endswith("anomal.nc"): # check if file is .nc_files_dir if region in file: # ploth the time series of the spatial avg file_path_normal = file_path.replace("anomal", "ymean" ) data_anom = Dataset(file_path, mode='r') data_norm = Dataset(file_path_normal, mode='r') plot_time_series(data_anom, param, region) plot_time_series(data_norm, param, region) data_anom.close() data_norm.close() # https://matplotlib.org/examples/color/colormaps_reference.html colormap = plt.cm.tab20 # after ploting all models, set colors and legend for all the figures
def merge_ts(cdo, rcp_path, index): # ts= timeseries from indices_misc import logger, clean from useful_functions import get_subdirs from useful_functions import get_subfiles from indices_graph_functions import plot_time_series array_all_models = "" file_path_list = [] # Just to plot all models array_all_models_avg6190 = "" for model, model_path in get_subdirs(rcp_path + "/models/"): for file, file_path in get_subfiles(model_path): if file.startswith("._"): continue elif file.endswith( "ts.nc" ): # check if end is ts.nc which means it is a time series and needs to be ensembled if index[ 'do_anom']: # if index requires anomalie, before merging, calculate it nc_avg_61_90 = file_path.replace('_ts.nc', '_avg_61_90.nc') nc_anomal = file_path.replace('_ts.nc', '_ts_anomal.nc') year_range = "-selyear,1961/1990" avg_61_90_val = cdo.timmean( input=year_range + " " + file_path, output=nc_avg_61_90, force=False, options='-f nc', returnCdf=True).variables[index['cdo_name']][0, 0, 0] cdo.subc(avg_61_90_val, input=file_path, force=False, output=nc_anomal ) # substract the timeman of period 61-90 array_all_models += nc_anomal + ' ' file_path_list.append( nc_anomal) # only to plot individually for debbugging array_all_models_avg6190 += nc_avg_61_90 + ' ' else: # no anomaly required array_all_models += file_path + ' ' file_path_list.append( file_path) # only to plot individually for debbugging if array_all_models == '': logger.debug(clean(("No files found in %s" % rcp_path))) else: if index['do_anom']: plot_time_series(index=index, file_path_in_array=file_path_list, png_name_in=rcp_path + '/' + index['name'] + "_allModels_ts_anom.png") else: plot_time_series(index, file_path_list, png_name_in=rcp_path + '/' + index['name'] + "_allModels_ts.png") percentil_array = ["25", "50", "75"] # median = 50th percentil for percentil in percentil_array: nc_ensmean_out = rcp_path + '/' + index[ 'name'] + '_percent_' + percentil + '_ts.nc' if percentil == "mean": cdo.ensmean(input=array_all_models, output=nc_ensmean_out, options='-f nc', force=False, returnCdf=False) else: cdo.enspctl(percentil, input=array_all_models, output=nc_ensmean_out, options='-f nc', force=False, returnCdf=False) # Ensemble percentiles if index['do_anom']: nc_ensmean_out_medianOfAvg6190 = rcp_path + '/' + index[ 'name'] + '_percent_50_ts_medianOfAvg6190.nc' cdo.ensmean(input=array_all_models_avg6190, output=nc_ensmean_out_medianOfAvg6190, options='-f nc', force=False, returnCdf=False) # find anomalie (this result is not really used) nc_avg_61_90 = nc_ensmean_out.replace('_ts.nc', '_avg_61_90.nc') nc_anomal = nc_ensmean_out.replace('_ts.nc', '_ts_anomal.nc') year_range = "-selyear,1961/1990" avg_61_90_val = cdo.timmean( input=year_range + " " + nc_ensmean_out, output=nc_avg_61_90, options='-f nc', force=False, returnCdf=True).variables[index['cdo_name']][0, 0, 0] cdo.subc(avg_61_90_val, input=nc_ensmean_out, output=nc_anomal) # substract il file 61-90 logger.debug(clean((nc_anomal)))
from useful_functions import check import pathlib import argparse import sys nc_dir = "/Users/danielaquintero/Documents/tesis/cmip5_eval/nc_files/" index_dir = "indices/wsdieca" cdo = Cdo() cdo.degub = True for region, region_path in get_subdirs(nc_dir+index_dir): for model_rcp85, model_rcp85_path in get_subdirs(region_path+"/rcp85/models/"): file_rcp85_merge = "" file_rcp45_merge = "" for file_rcp85, file_rcp85_path in get_subfiles(model_rcp85_path): if file_rcp85.startswith("._"): continue # go to next file if file_rcp85.endswith("ts.nc"): # check if file is .nc file_rcp85_merge = file_rcp85_path model_rcp45_path = model_rcp85_path.replace("rcp85", "rcp45") if not check(model_rcp45_path): print(model_rcp45_path) print("NO RCP45 equivalent\n") continue else: for file_rcp45, file_rcp45_path in get_subfiles(model_rcp45_path): if file_rcp45.startswith("._"): continue # go to next file
def graph_map(): import numpy as np from netCDF4 import Dataset for index_l, index_path in get_subdirs(indices_output_dir): if index_l != index_in['name']: continue for region, region_path in get_subdirs(index_path): if 'ignore' in index_in and region in index_in['ignore']: continue # first find the minumum and maximum for the index max_list = [] min_list = [] min_list_ref = [] max_list_ref = [] max_list_rel = [] min_list_rel = [] for rcp, rcp_path in get_subdirs(region_path): for season, season_path in get_subdirs(rcp_path): for file, file_path in get_subfiles(season_path): if file.startswith("._"): pass elif (file.endswith("sub.nc") and index_in['do_anom']) or (file.endswith('.nc') and 'reference' not in file and not index_in['do_anom']): if any(period in file for period in period_name_array): fh = Dataset(file_path, 'r') param = fh.variables[index_in['cdo_name']][0:, :, :] max_list.append(np.amax(param)) min_list.append(np.amin(param)) fh.close() elif file.endswith("sub_rel.nc"): if any(period in file for period in period_name_array): fh = Dataset(file_path, 'r') param = fh.variables[index_in['cdo_name']][0:, :, :] max_list_rel.append(np.amax(param)) min_list_rel.append(np.amin(param)) fh.close() elif 'reference' in file and file.endswith('.nc'): fh = Dataset(file_path, 'r') param = fh.variables[index_in['cdo_name']][0:, :, :] max_list_ref.append(np.amax(param)) min_list_ref.append(np.amin(param)) fh.close() # The min and max may be extremes that would make the graph hard to read. # Instead use percentiles. min_sub = np.percentile(np.array(min_list), index_in['min_perc']) max_sub = np.percentile(np.array(max_list), index_in['max_perc']) min_ref = min(min_list_ref) max_ref = max(max_list_ref) if not index_in['do_anom']: # for indices with implicit anomaly, it is better to use the same scale for reference period and other periods min_ref = min_sub max_ref = max_sub if (index_in['do_rel']): min_rel = np.percentile(np.array(min_list_rel), index_in['min_perc_rel']) max_rel = np.percentile(np.array(max_list_rel), index_in['max_perc_rel']) # plot for rcp, rcp_path in get_subdirs(region_path): if rcp != args.rcp: continue for season, season_path in get_subdirs(rcp_path): season_name = '' if season != 'ANN': season_name = " (" + season + ")" for file, file_path in get_subfiles(season_path): png_name_latex = indices_graphs_out_dir+"/"+index_l+"/"+region+"/"+rcp+"/"+season+"/"+file.replace('.nc', '.png') for period in period_name_array: if period in file: title = period_title_array[period_name_array.index(period)] + season_name if 'future' in period: title += ' (' + str.upper(rcp).replace('5', '.5') + ')' elif '45' in rcp: png_name_latex = None # 45 are not needed in latex expect the future one break if file.startswith("._"): pass elif (file.endswith("sub.nc") and index_in['do_anom']) or (file.endswith('.nc') and 'reference' not in file and not index_in['do_anom']): plot_basemap_regions(index_in, file_path, file_path.replace(".nc", ".png"), region, title, min_sub, max_sub, png_name_latex) elif 'reference' in file and file.endswith('.nc'): plot_basemap_regions(index_in, file_path, file_path.replace(".nc", ".png"), region, title, min_ref, max_ref, None) elif 'ignore' in index_in and region in index_in['ignore']: continue elif file.endswith("sub_rel.nc") and index_in['do_rel']: if 'future' not in file: png_name_latex = None # rel graphs only needed for future plot_basemap_regions(index_in, file_path, file_path.replace(".nc", ".png"), region, title, min_rel, max_rel, png_name_latex)
def graph_ts(): import numpy as np from netCDF4 import Dataset for index_l, index_path in get_subdirs(indices_output_dir): if index_l != index_in['name']: continue # first find the minumum and maximum for the index max_list_models = [] min_list_models = [] max_list = {} min_list = {} avg6190 = {} models_plot = {} for region, region_path in get_subdirs(index_path): if 'ignore' in index_in and region in index_in['ignore']: continue avg6190[region] = None models_plot[region] = [] max_list[region] = [] min_list[region] = [] for rcp, rcp_path in get_subdirs(region_path): for file, file_path in get_subfiles(rcp_path): if file.startswith("._"): continue elif file.endswith("ts.nc"): fh = Dataset(file_path, 'r') param = fh.variables[index_in['cdo_name']][:, 0, 0] max_list[region].append(np.amax(param)) min_list[region].append(np.amin(param)) fh.close() elif file.endswith("Avg6190.nc"): fh = Dataset(file_path, 'r') avg6190[region] = fh.variables[index_in['cdo_name']][0, 0, 0] fh.close() # all models for model, model_path in get_subdirs(rcp_path+"/models/"): for file, file_path in get_subfiles(model_path): if file.startswith("._"): continue elif (file.endswith("ts_anomal.nc") and index_in['do_anom']) or (file.endswith('ts.nc') and not index_in['do_anom']): fh = Dataset(file_path, 'r') param = fh.variables[index_in['cdo_name']][:, 0, 0] max_list_models.append(np.amax(param)) min_list_models.append(np.amin(param)) fh.close() models_plot[region].append(file_path) # plot for region, region_path in get_subdirs(index_path): if 'ignore' in index_in and region in index_in['ignore']: continue files_to_plot = [] for rcp, rcp_path in get_subdirs(region_path): for file, file_path in get_subfiles(rcp_path): if file.startswith("._"): continue elif file.endswith("ts.nc"): files_to_plot.append(file_path) debug(clean((file_path))) plot_time_series(index_in, files_to_plot, models_plot_array=models_plot[region], region=region, png_name_in=region_path+"/"+index_in['name']+'_'+region+'_Models_ts.png', png_name_latex=indices_graphs_out_dir+"/"+index_l+"/"+region+"/"+index_in['name']+'_'+region+'_Models_ts.png', min=min(min_list[region]), max=max(max_list[region]), avg6190=avg6190[region]) plot_time_series(index_in, files_to_plot, region=region, png_name_in=region_path+"/"+index_in['name']+'_'+region+'_ts.png', min=min(min_list[region]), max=max(max_list[region]), avg6190=avg6190[region])
parser.print_help() sys.exit() cdo = Cdo() if args.verb: cdo.degub = True for model_histo, model_histo_path in get_subdirs(nc_dir + histo_path): # loop of all parameters inside each model for param_histo, param_histo_path in get_subdirs(model_histo_path): # loop all regions inside each parameter for region_histo, region_histo_path in get_subdirs(param_histo_path): # loop all files inside the param path file_histo_merge = "" file_rcp_merge = "" for file_histo, file_histo_path in get_subfiles(region_histo_path): if file_histo.startswith("._"): continue # go to next file if file_histo.endswith(".nc"): # check if file is .nc file_histo_merge = file_histo_path region_rcp_path = region_histo_path.replace(histo_path, rcp_path) if not check(region_rcp_path): print(file_histo_merge) print("NO RCP equivalent\n") continue else: for file_rcp, file_rcp_path in get_subfiles(region_rcp_path): if file_rcp.startswith("._"): continue # go to next file