def graph_bar(): from netCDF4 import Dataset for index_l, index_path in get_subdirs(indices_output_dir): if index_l != index_in['name']: continue avg6190 = {} files_to_plot = {} for region, region_path in get_subdirs(index_path): if 'ignore' in index_in and region in index_in['ignore']: continue avg6190[region] = None files_to_plot[region] = [] for rcp, rcp_path in get_subdirs(region_path): for file, file_path in get_subfiles(rcp_path): if file.startswith("._"): continue elif file.endswith("ts.nc"): files_to_plot[region].append(file_path) elif file.endswith("Avg6190.nc"): fh = Dataset(file_path, 'r') avg6190[region] = fh.variables[index_in['cdo_name']][0, 0, 0] fh.close() plot_bar_index(index=index_in, files=files_to_plot, png_name_in=indices_graphs_out_dir+"/"+index_l+"/"+index_in['name']+'_bar.png', avg6190=avg6190)
def loop_models(): # loop all the index we want to obtain, and the required param for that index for index, req_param in zip(index_array, req_param_array): print("Calculate %s using %s" % (index, req_param)) # loop the experiment folders we have for experiment in experiment_array: # loop of all models inside the experiment folder for model, model_path in get_subdirs(experiment): # loop of all parameters inside each model for param, param_path in get_subdirs(model_path): # if the param is the required one for the index if param == req_param: for region, region_path in get_subdirs(param_path): # loop all files inside the param path for file, file_path in get_subfiles(region_path): if file.startswith("._"): pass # does nothing elif file.endswith(".nc"): # check if file is .nc output_dir = indices_output_dir + '/' + index + '/' + region + '/models/' + model if "historical" in experiment: absolute_index(index_in=index, param_in=param, nc_histo_in=file_path, out_dir_in=output_dir) elif "rcp45" in experiment: absolute_index(index_in=index, param_in=param, nc_rcp45_in=file_path, out_dir_in=output_dir) elif "rcp85" in experiment: absolute_index(index_in=index, param_in=param, nc_rcp85_in=file_path, out_dir_in=output_dir)
def merge_index(periods=False, ts=False): # loop indices folders until we find the one we want to merge for index_l, index_path in get_subdirs(indices_output_dir): if index_l != index_in['name']: continue for region, region_path in get_subdirs(index_path): # loop rcps folders until we find the one we want to merge for rcp, rcp_path in get_subdirs(region_path): if rcp != args.rcp: continue # for function in index_in['merge_functions']: # function(cdo=cdo, rcp_path=rcp_path, index=index_in) if periods: merge_periods(cdo=cdo, rcp_path=rcp_path, index=index_in) if ts: merge_ts(cdo=cdo, rcp_path=rcp_path, index=index_in)
def loop_models(): # loop the required folders (for indices that require two parameters) for experiment in experiment_set: # loop of all models inside the experiment folder for model, model_path in get_subdirs(experiment): param_path_list = {} if 'ignore' in index_in and [model, args.rcp] in index_in['ignore']: # this model/rcp should not be used for this index debug(model+' Ignored ') continue # loop of all parameters inside each model for param, param_path in get_subdirs(model_path): # if the param is the required one for the index if param in index_in['param']: for region, region_path in get_subdirs(param_path): if 'ignore' in index_in and region in index_in['ignore']: continue if region not in param_path_list.keys(): param_path_list[region] = [''] * (len(index_in['param']) + 1) # 1 more for the out_dir # loop all files inside the param path for file, file_path in get_subfiles(region_path): if file.startswith("._"): continue # does nothing elif file.endswith(".nc"): # check if file is .nc output_dir = None if "rcp45" in experiment: output_dir = indices_output_dir + '/' + index_in['name'] + '/' + region + '/rcp45/models/' + model elif "rcp85" in experiment: output_dir = indices_output_dir + '/' + index_in['name'] + '/' + region + '/rcp85/models/' + model pathlib.Path(output_dir).mkdir(parents=True, exist_ok=True) if len(index_in['param']) == 1: for function in index_in['loop_functions']: function(index=index_in, cdo=cdo, out_dir=output_dir, nc_in=file_path) else: # index with multiple param param_path_list[region][-1] = output_dir param_path_list[region][index_in['param'].index(param)] = file_path if len(index_in['param']) > 1: for region, paths in param_path_list.items(): for function in index_in['loop_functions']: if paths[-1] != '' and paths[0] != '' and paths[1] != '': function(index=index_in, cdo=cdo, out_dir=paths[-1], nc_in=paths[0], nc_in2=paths[1]) else: debug(clean("Missign param for out_dir: " + paths[-1]))
def graph_index(): bounds = [-0.7, -0.6, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7] # 11 colors colors = ['red', 'firebrick', 'tomato', 'salmon', 'lightcoral', 'lightsalmon', 'papayawhip', 'snow', 'paleturquoise', 'skyblue', 'lightskyblue', 'steelblue', 'dodgerblue', 'blue', 'darkblue'] for index, index_path in get_subdirs(indices_output_dir): param = index+"ETCCDI" for region, region_path in get_subdirs(index_path): for season, season_path in get_subdirs(region_path): for file, file_path in get_subfiles(season_path): if file.startswith("._"): pass elif file.endswith("sub.nc"): # if files ends with _sub.nc is the result of a substraction, and needs to be graphed for period in periods: if period in file: title = index + " in the " + period + " for season " + season + " in region " + region plot_basemap_regions(file_path, file_path.replace(".nc", ".png"), param, region, title, cdo, bounds, colors, colors[-1], colors[0])
def merge_index(): for index, index_path in get_subdirs(indices_output_dir): for region, region_path in get_subdirs(index_path): for season in seasons: pathlib.Path(region_path + '/' + season).mkdir(parents=True, exist_ok=True) nc_ensmean_reference = '' for period in periods: index_file_pathlib_list = pathlib.Path(region_path + '/models').rglob('*' + season + '*/*' + period + '*.nc') index_all_models = '' for index_file_pathlib in index_file_pathlib_list: index_file = str(index_file_pathlib) if (index_file.startswith("._")) or ("regrid" in index_file): continue # jump to next file index_file_regrid = index_file.replace('.nc', '_regrid.nc') target_grid_file = region_path + '/target_grid.nc' cdo.remapbil(target_grid_file, input=index_file, output=index_file_regrid, froce=False) index_all_models += index_file_regrid + ' ' if index_all_models == '': print("No files found for Season %s and period %s" % (season, period)) else: nc_ensmean_out = region_path + '/' + season + '/' + period + '.nc' try: print(nc_ensmean_out) cdo.ensmean(input=index_all_models, output=nc_ensmean_out, options='-f nc', force=True, returnCdf=False) if period == "reference": nc_ensmean_reference = nc_ensmean_out elif nc_ensmean_reference != '': # substract reference from period. Only for non reference period, if reference file has been found. nc_ensmean_sub_out = region_path + '/' + season + '/' + period + '_sub.nc' print(nc_ensmean_sub_out) cdo.sub(input=nc_ensmean_out + " " + nc_ensmean_reference, output=nc_ensmean_sub_out, options='-f nc', force=True, returnCdf=False) except CDOException: print("CDO error")
''' regionArray = ['Andes', 'Alpine'] boxAndes = [283-1, 288+1, 0, 8.5+1] # long1, long2, lat1, lat2 boxAlpine = [5-1, 14+1, 44.5-1, 48.5+1] boxesArray = [boxAndes, boxAlpine] paramArray = [] nc_files_dir = "../nc_files/" proyect_dir = "RCP8_5_converted/" max_models = 45 # loop the regionArray and boxesArray together for region, box in zip(regionArray, boxesArray): i_models = 0 # loop of all models inside the cmip5 proyect dir for model, model_path in get_subdirs(nc_files_dir+proyect_dir): i_models = i_models + 1 if(i_models > max_models): break # loop of all parameters inside each model for param, param_path in get_subdirs(model_path): # create array of params if param not in paramArray: paramArray.append(param) # create an array to plot with all the models saved # loop all files inside the param path for file, file_path in get_subfiles(param_path): if file.endswith(".nc"): # check if file is .nc_files_dir # ploth the time series of the spatial avg avg_time_series(file_path, param, region, box, model, False) break
cdo.degub = True project_dir = "/Users/danielaquintero/Downloads/tmasmin" project_dir_converted = "/Users/danielaquintero/Downloads/tmasmin_converted" # "../nc_files/rcp85_pr_daily_converted/" file_path_aux = "/Users/danielaquintero/Downloads/file_path_aux.nc" # project_dir = "/Volumes/wd_personal/rcp85_tmasmin" # project_dir_converted = "../nc_files/rcp85_tmasmin_daily_converted/" # file_path_aux = "/Volumes/SONY_FAT32/file_path_aux.nc" max_models = 45 i_models = 0 # model_array = ['EC-EARTH'] # loop of all models inside the cmip5 project dir for model, model_path in get_subdirs(project_dir): # if model not in model_array: # continue print(model) i_models = i_models + 1 if(i_models > max_models): break model_path_converted = model_path.replace(project_dir, project_dir_converted) check_and_create(model_path_converted) # loop of all parameters inside each model for param, param_path in get_subdirs(model_path): # me da todo lo que hay dentro de la carpeta modelo (osea parametros) param_path_converted = param_path.replace(project_dir, project_dir_converted)
def merge_ts(cdo, rcp_path, index): # ts= timeseries from indices_misc import logger, clean from useful_functions import get_subdirs from useful_functions import get_subfiles from indices_graph_functions import plot_time_series array_all_models = "" file_path_list = [] # Just to plot all models array_all_models_avg6190 = "" for model, model_path in get_subdirs(rcp_path + "/models/"): for file, file_path in get_subfiles(model_path): if file.startswith("._"): continue elif file.endswith( "ts.nc" ): # check if end is ts.nc which means it is a time series and needs to be ensembled if index[ 'do_anom']: # if index requires anomalie, before merging, calculate it nc_avg_61_90 = file_path.replace('_ts.nc', '_avg_61_90.nc') nc_anomal = file_path.replace('_ts.nc', '_ts_anomal.nc') year_range = "-selyear,1961/1990" avg_61_90_val = cdo.timmean( input=year_range + " " + file_path, output=nc_avg_61_90, force=False, options='-f nc', returnCdf=True).variables[index['cdo_name']][0, 0, 0] cdo.subc(avg_61_90_val, input=file_path, force=False, output=nc_anomal ) # substract the timeman of period 61-90 array_all_models += nc_anomal + ' ' file_path_list.append( nc_anomal) # only to plot individually for debbugging array_all_models_avg6190 += nc_avg_61_90 + ' ' else: # no anomaly required array_all_models += file_path + ' ' file_path_list.append( file_path) # only to plot individually for debbugging if array_all_models == '': logger.debug(clean(("No files found in %s" % rcp_path))) else: if index['do_anom']: plot_time_series(index=index, file_path_in_array=file_path_list, png_name_in=rcp_path + '/' + index['name'] + "_allModels_ts_anom.png") else: plot_time_series(index, file_path_list, png_name_in=rcp_path + '/' + index['name'] + "_allModels_ts.png") percentil_array = ["25", "50", "75"] # median = 50th percentil for percentil in percentil_array: nc_ensmean_out = rcp_path + '/' + index[ 'name'] + '_percent_' + percentil + '_ts.nc' if percentil == "mean": cdo.ensmean(input=array_all_models, output=nc_ensmean_out, options='-f nc', force=False, returnCdf=False) else: cdo.enspctl(percentil, input=array_all_models, output=nc_ensmean_out, options='-f nc', force=False, returnCdf=False) # Ensemble percentiles if index['do_anom']: nc_ensmean_out_medianOfAvg6190 = rcp_path + '/' + index[ 'name'] + '_percent_50_ts_medianOfAvg6190.nc' cdo.ensmean(input=array_all_models_avg6190, output=nc_ensmean_out_medianOfAvg6190, options='-f nc', force=False, returnCdf=False) # find anomalie (this result is not really used) nc_avg_61_90 = nc_ensmean_out.replace('_ts.nc', '_avg_61_90.nc') nc_anomal = nc_ensmean_out.replace('_ts.nc', '_ts_anomal.nc') year_range = "-selyear,1961/1990" avg_61_90_val = cdo.timmean( input=year_range + " " + nc_ensmean_out, output=nc_avg_61_90, options='-f nc', force=False, returnCdf=True).variables[index['cdo_name']][0, 0, 0] cdo.subc(avg_61_90_val, input=nc_ensmean_out, output=nc_anomal) # substract il file 61-90 logger.debug(clean((nc_anomal)))
from cdo import * from useful_functions import get_subdirs from useful_functions import get_subfiles from useful_functions import check import pathlib import argparse import sys nc_dir = "/Users/danielaquintero/Documents/tesis/cmip5_eval/nc_files/" index_dir = "indices/wsdieca" cdo = Cdo() cdo.degub = True for region, region_path in get_subdirs(nc_dir+index_dir): for model_rcp85, model_rcp85_path in get_subdirs(region_path+"/rcp85/models/"): file_rcp85_merge = "" file_rcp45_merge = "" for file_rcp85, file_rcp85_path in get_subfiles(model_rcp85_path): if file_rcp85.startswith("._"): continue # go to next file if file_rcp85.endswith("ts.nc"): # check if file is .nc file_rcp85_merge = file_rcp85_path model_rcp45_path = model_rcp85_path.replace("rcp85", "rcp45") if not check(model_rcp45_path): print(model_rcp45_path) print("NO RCP45 equivalent\n") continue else:
The nc files will be analyzed for different regions (for now two) ''' nc_files_dir = "/Volumes/wd_tesis/" proyect_dir = "tasmin_tasmax_historical_converted/" max_models = 45 i_models = 0 regionArray = [] paramArray = [] # loop of all models inside the cmip5 proyect dir for model, model_path in get_subdirs(nc_files_dir+proyect_dir): i_models = i_models + 1 if(i_models > max_models): break # loop of all parameters inside each model for param, param_path in get_subdirs(model_path): for region, region_path in get_subdirs(param_path): for subparam, subparam_path in get_subdirs(region_path): # loop all files inside the param path for file, file_path in get_subfiles(subparam_path): if file.endswith("timemax_fldmax_annual.nc"): # check if file is .nc_files_dir if region in file: # ploth the time series of the spatial avg print(file_path)
def graph_map(): import numpy as np from netCDF4 import Dataset for index_l, index_path in get_subdirs(indices_output_dir): if index_l != index_in['name']: continue for region, region_path in get_subdirs(index_path): if 'ignore' in index_in and region in index_in['ignore']: continue # first find the minumum and maximum for the index max_list = [] min_list = [] min_list_ref = [] max_list_ref = [] max_list_rel = [] min_list_rel = [] for rcp, rcp_path in get_subdirs(region_path): for season, season_path in get_subdirs(rcp_path): for file, file_path in get_subfiles(season_path): if file.startswith("._"): pass elif (file.endswith("sub.nc") and index_in['do_anom']) or (file.endswith('.nc') and 'reference' not in file and not index_in['do_anom']): if any(period in file for period in period_name_array): fh = Dataset(file_path, 'r') param = fh.variables[index_in['cdo_name']][0:, :, :] max_list.append(np.amax(param)) min_list.append(np.amin(param)) fh.close() elif file.endswith("sub_rel.nc"): if any(period in file for period in period_name_array): fh = Dataset(file_path, 'r') param = fh.variables[index_in['cdo_name']][0:, :, :] max_list_rel.append(np.amax(param)) min_list_rel.append(np.amin(param)) fh.close() elif 'reference' in file and file.endswith('.nc'): fh = Dataset(file_path, 'r') param = fh.variables[index_in['cdo_name']][0:, :, :] max_list_ref.append(np.amax(param)) min_list_ref.append(np.amin(param)) fh.close() # The min and max may be extremes that would make the graph hard to read. # Instead use percentiles. min_sub = np.percentile(np.array(min_list), index_in['min_perc']) max_sub = np.percentile(np.array(max_list), index_in['max_perc']) min_ref = min(min_list_ref) max_ref = max(max_list_ref) if not index_in['do_anom']: # for indices with implicit anomaly, it is better to use the same scale for reference period and other periods min_ref = min_sub max_ref = max_sub if (index_in['do_rel']): min_rel = np.percentile(np.array(min_list_rel), index_in['min_perc_rel']) max_rel = np.percentile(np.array(max_list_rel), index_in['max_perc_rel']) # plot for rcp, rcp_path in get_subdirs(region_path): if rcp != args.rcp: continue for season, season_path in get_subdirs(rcp_path): season_name = '' if season != 'ANN': season_name = " (" + season + ")" for file, file_path in get_subfiles(season_path): png_name_latex = indices_graphs_out_dir+"/"+index_l+"/"+region+"/"+rcp+"/"+season+"/"+file.replace('.nc', '.png') for period in period_name_array: if period in file: title = period_title_array[period_name_array.index(period)] + season_name if 'future' in period: title += ' (' + str.upper(rcp).replace('5', '.5') + ')' elif '45' in rcp: png_name_latex = None # 45 are not needed in latex expect the future one break if file.startswith("._"): pass elif (file.endswith("sub.nc") and index_in['do_anom']) or (file.endswith('.nc') and 'reference' not in file and not index_in['do_anom']): plot_basemap_regions(index_in, file_path, file_path.replace(".nc", ".png"), region, title, min_sub, max_sub, png_name_latex) elif 'reference' in file and file.endswith('.nc'): plot_basemap_regions(index_in, file_path, file_path.replace(".nc", ".png"), region, title, min_ref, max_ref, None) elif 'ignore' in index_in and region in index_in['ignore']: continue elif file.endswith("sub_rel.nc") and index_in['do_rel']: if 'future' not in file: png_name_latex = None # rel graphs only needed for future plot_basemap_regions(index_in, file_path, file_path.replace(".nc", ".png"), region, title, min_rel, max_rel, png_name_latex)
def graph_ts(): import numpy as np from netCDF4 import Dataset for index_l, index_path in get_subdirs(indices_output_dir): if index_l != index_in['name']: continue # first find the minumum and maximum for the index max_list_models = [] min_list_models = [] max_list = {} min_list = {} avg6190 = {} models_plot = {} for region, region_path in get_subdirs(index_path): if 'ignore' in index_in and region in index_in['ignore']: continue avg6190[region] = None models_plot[region] = [] max_list[region] = [] min_list[region] = [] for rcp, rcp_path in get_subdirs(region_path): for file, file_path in get_subfiles(rcp_path): if file.startswith("._"): continue elif file.endswith("ts.nc"): fh = Dataset(file_path, 'r') param = fh.variables[index_in['cdo_name']][:, 0, 0] max_list[region].append(np.amax(param)) min_list[region].append(np.amin(param)) fh.close() elif file.endswith("Avg6190.nc"): fh = Dataset(file_path, 'r') avg6190[region] = fh.variables[index_in['cdo_name']][0, 0, 0] fh.close() # all models for model, model_path in get_subdirs(rcp_path+"/models/"): for file, file_path in get_subfiles(model_path): if file.startswith("._"): continue elif (file.endswith("ts_anomal.nc") and index_in['do_anom']) or (file.endswith('ts.nc') and not index_in['do_anom']): fh = Dataset(file_path, 'r') param = fh.variables[index_in['cdo_name']][:, 0, 0] max_list_models.append(np.amax(param)) min_list_models.append(np.amin(param)) fh.close() models_plot[region].append(file_path) # plot for region, region_path in get_subdirs(index_path): if 'ignore' in index_in and region in index_in['ignore']: continue files_to_plot = [] for rcp, rcp_path in get_subdirs(region_path): for file, file_path in get_subfiles(rcp_path): if file.startswith("._"): continue elif file.endswith("ts.nc"): files_to_plot.append(file_path) debug(clean((file_path))) plot_time_series(index_in, files_to_plot, models_plot_array=models_plot[region], region=region, png_name_in=region_path+"/"+index_in['name']+'_'+region+'_Models_ts.png', png_name_latex=indices_graphs_out_dir+"/"+index_l+"/"+region+"/"+index_in['name']+'_'+region+'_Models_ts.png', min=min(min_list[region]), max=max(max_list[region]), avg6190=avg6190[region]) plot_time_series(index_in, files_to_plot, region=region, png_name_in=region_path+"/"+index_in['name']+'_'+region+'_ts.png', min=min(min_list[region]), max=max(max_list[region]), avg6190=avg6190[region])
elif args.rcp85: rcp_path = rcp85_path merge_path = histo_rcp85_merge else: parser.print_help() sys.exit() else: parser.print_help() sys.exit() cdo = Cdo() if args.verb: cdo.degub = True for model_histo, model_histo_path in get_subdirs(nc_dir + histo_path): # loop of all parameters inside each model for param_histo, param_histo_path in get_subdirs(model_histo_path): # loop all regions inside each parameter for region_histo, region_histo_path in get_subdirs(param_histo_path): # loop all files inside the param path file_histo_merge = "" file_rcp_merge = "" for file_histo, file_histo_path in get_subfiles(region_histo_path): if file_histo.startswith("._"): continue # go to next file if file_histo.endswith(".nc"): # check if file is .nc file_histo_merge = file_histo_path region_rcp_path = region_histo_path.replace(histo_path, rcp_path)