def delete_days(cdo, index, out_dir, nc_in, nc_in2=None):
'''
Delete wrongly generated days in rx5day index monthly
'''
from indices_misc import logger, clean
from useful_functions import get_subfiles
import pathlib
from os import rename
for file, file_path in get_subfiles(out_dir):
if file.startswith("._"):
continue # does nothing
elif file.endswith(
index['name'] + ".nc"
): # if file ends with index.nc, it is the pure index file
nc_out_days = out_dir + "/" + pathlib.Path(
file_path).stem + "_withDays.nc"
rename(file_path, nc_out_days)
logger.debug(clean((file_path)))
cdo.delete("day=17",
input="-delete,month=6,hour=12 " + nc_out_days,
output=file_path,
options='-f nc',
force=False,
returnCdf=False)
def generate_periods(cdo, index, out_dir, nc_in=None, nc_in2=None):
'''
Generate a map of the index, for the required time periods, starting from matrix
'''
from indices_misc import logger, clean
from indices_misc import period_range_array, period_name_array
from useful_functions import get_subfiles
import pathlib
for season in index['seasons']:
for year_range, name in zip(period_range_array, period_name_array):
cdo_year_command = "-selyear," + year_range
for file, file_path in get_subfiles(out_dir):
if file.startswith("._"):
continue # does nothing
elif file.endswith(
index['name'] + ".nc"
): # if file ends with index.nc, it is the pure idex file, with matrix values (temp and field)
pathlib.Path(out_dir + "/" + season + "/").mkdir(
parents=True, exist_ok=True)
nc_out = out_dir + "/" + season + "/" + pathlib.Path(
file_path).stem + "_" + name + ".nc"
logger.debug(clean((nc_out)))
logger.debug(clean(("")))
cdo.timmean(input="-setreftime,1850-01-01,00:00:00 " +
cdo_year_command + " " + file_path,
output=nc_out,
options='-f nc',
force=False,
returnCdf=False)
def normal_index(cdo, index, out_dir, nc_in, nc_in2=None):
'''
Calculate "normal" indices, that generate a matrix out, without needing any additional input parameters
'''
from indices_misc import logger, clean
import pathlib
index_cdo_function = getattr(
cdo, index['cdo_fun']
) # get the cdo function coressponding to the input index
nc_out_normal = (out_dir + '/' + pathlib.Path(nc_in).stem + "_" +
index['name'] + ".nc")
add_params = ''
add_command = ''
if 'add_params' in index:
add_params = index['add_params']
if 'add_command' in index:
add_command = index['add_command']
logger.debug(clean((nc_out_normal)))
index_cdo_function(add_params,
input=add_command + nc_in,
output=nc_out_normal,
options='-f nc',
force=False,
returnCdf=False)
def generate_ts(cdo, index, out_dir, nc_in=None, nc_in2=None):
'''
Generate timeseries of the index, for the whole area, starting from matrix
'''
from indices_misc import logger, clean
from useful_functions import get_subfiles
import pathlib
for file, file_path in get_subfiles(out_dir):
if file.startswith("._"):
continue # does nothing
elif file.endswith(
index['name'] + ".nc"
): # if file ends with index.nc, it is the pure idex file, with matrix values (temp and field)
nc_out = out_dir + "/" + pathlib.Path(file_path).stem + "_ts.nc"
logger.debug(clean((nc_out)))
logger.debug(clean(("")))
cdo.fldmean(input="-setreftime,1850-01-01,00:00:00 " + file_path,
output=nc_out,
options='-f nc',
force=False,
returnCdf=False)
def manual_index(cdo, index, out_dir, nc_in, nc_in2=None):
'''
Calculate indices manually
'''
from indices_misc import logger, clean
import pathlib
if nc_in2 is not None: # index with two parameters
nc_in += " " + nc_in2
nc_out = (out_dir + '/' + pathlib.Path(nc_in).stem + "_" + index['name'] +
".nc")
logger.debug(clean((nc_out)))
cdo.chname(
index['param'][0] + "," + index['cdo_name'], # change aprameter name
input="-setattribute," + index['param'][0] + "@long_name=" +
index['long_name'] # change parameter long name
+ index['cdo_fun'] + nc_in, # the actual command
output=nc_out,
options='-f nc',
force=False,
returnCdf=False)
def merge_ts(cdo, rcp_path, index): # ts= timeseries
from indices_misc import logger, clean
from useful_functions import get_subdirs
from useful_functions import get_subfiles
from indices_graph_functions import plot_time_series
array_all_models = ""
file_path_list = [] # Just to plot all models
array_all_models_avg6190 = ""
for model, model_path in get_subdirs(rcp_path + "/models/"):
for file, file_path in get_subfiles(model_path):
if file.startswith("._"):
continue
elif file.endswith(
"ts.nc"
): # check if end is ts.nc which means it is a time series and needs to be ensembled
if index[
'do_anom']: # if index requires anomalie, before merging, calculate it
nc_avg_61_90 = file_path.replace('_ts.nc', '_avg_61_90.nc')
nc_anomal = file_path.replace('_ts.nc', '_ts_anomal.nc')
year_range = "-selyear,1961/1990"
avg_61_90_val = cdo.timmean(
input=year_range + " " + file_path,
output=nc_avg_61_90,
force=False,
options='-f nc',
returnCdf=True).variables[index['cdo_name']][0, 0, 0]
cdo.subc(avg_61_90_val,
input=file_path,
force=False,
output=nc_anomal
) # substract the timeman of period 61-90
array_all_models += nc_anomal + ' '
file_path_list.append(
nc_anomal) # only to plot individually for debbugging
array_all_models_avg6190 += nc_avg_61_90 + ' '
else: # no anomaly required
array_all_models += file_path + ' '
file_path_list.append(
file_path) # only to plot individually for debbugging
if array_all_models == '':
logger.debug(clean(("No files found in %s" % rcp_path)))
else:
if index['do_anom']:
plot_time_series(index=index,
file_path_in_array=file_path_list,
png_name_in=rcp_path + '/' + index['name'] +
"_allModels_ts_anom.png")
else:
plot_time_series(index,
file_path_list,
png_name_in=rcp_path + '/' + index['name'] +
"_allModels_ts.png")
percentil_array = ["25", "50", "75"] # median = 50th percentil
for percentil in percentil_array:
nc_ensmean_out = rcp_path + '/' + index[
'name'] + '_percent_' + percentil + '_ts.nc'
if percentil == "mean":
cdo.ensmean(input=array_all_models,
output=nc_ensmean_out,
options='-f nc',
force=False,
returnCdf=False)
else:
cdo.enspctl(percentil,
input=array_all_models,
output=nc_ensmean_out,
options='-f nc',
force=False,
returnCdf=False) # Ensemble percentiles
if index['do_anom']:
nc_ensmean_out_medianOfAvg6190 = rcp_path + '/' + index[
'name'] + '_percent_50_ts_medianOfAvg6190.nc'
cdo.ensmean(input=array_all_models_avg6190,
output=nc_ensmean_out_medianOfAvg6190,
options='-f nc',
force=False,
returnCdf=False)
# find anomalie (this result is not really used)
nc_avg_61_90 = nc_ensmean_out.replace('_ts.nc',
'_avg_61_90.nc')
nc_anomal = nc_ensmean_out.replace('_ts.nc', '_ts_anomal.nc')
year_range = "-selyear,1961/1990"
avg_61_90_val = cdo.timmean(
input=year_range + " " + nc_ensmean_out,
output=nc_avg_61_90,
options='-f nc',
force=False,
returnCdf=True).variables[index['cdo_name']][0, 0, 0]
cdo.subc(avg_61_90_val, input=nc_ensmean_out,
output=nc_anomal) # substract il file 61-90
logger.debug(clean((nc_anomal)))
def merge_periods(cdo, rcp_path, index):
from indices_misc import logger, clean
from indices_misc import period_name_array
import pathlib
for season in index['seasons']:
pathlib.Path(rcp_path + '/' + season).mkdir(parents=True,
exist_ok=True)
nc_ensmean_reference = ''
for period in period_name_array:
index_file_pathlib_list = pathlib.Path(rcp_path +
'/models').rglob('*' +
season +
'*/*' +
period +
'*.nc')
index_all_models = ''
for index_file_pathlib in index_file_pathlib_list:
index_file = str(index_file_pathlib)
if (index_file.startswith("._")) or ("regrid" in index_file):
continue # jump to next file
index_file_regrid = index_file.replace('.nc', '_regrid.nc')
target_grid_file = rcp_path + '/target_grid.nc'
cdo.remapbil(target_grid_file,
input=index_file,
output=index_file_regrid,
froce=False) # regrid to target grid
index_all_models += index_file_regrid + ' '
if index_all_models == '':
logger.debug(
clean(("No files found for Season %s and period %s" %
(season, period))))
else:
nc_ensmean_out = rcp_path + '/' + season + '/' + index[
'name'] + "_" + period + '.nc'
logger.debug(clean((nc_ensmean_out)))
cdo.enspctl("50",
input=index_all_models,
output=nc_ensmean_out,
options='-f nc',
force=False,
returnCdf=False) # median = 50th percentil
if index['do_anom']:
if "reference" in period:
nc_ensmean_reference = nc_ensmean_out
elif nc_ensmean_reference != '': # substract reference from period. Only for non reference period, if reference file has been found.
nc_ensmean_sub_out = rcp_path + '/' + season + '/' + index[
'name'] + "_" + period + '_sub.nc'
nc_ensmean_sub_rel_out = rcp_path + '/' + season + '/' + index[
'name'] + "_" + period + '_sub_rel.nc'
logger.debug(clean((nc_ensmean_sub_out)))
cdo.sub(input=nc_ensmean_out + " " +
nc_ensmean_reference,
output=nc_ensmean_sub_out,
options='-f nc',
force=False,
returnCdf=False)
logger.debug(clean((nc_ensmean_sub_rel_out)))
setrtomiss = ' '
if 'setrtomiss' in index:
setrtomiss = index['setrtomiss']
cdo.mulc(100,
input="-setattribute," + index['cdo_name'] +
"@units=\"Relative Change (%)\" -div " +
nc_ensmean_sub_out + setrtomiss + ' -abs ' +
nc_ensmean_reference,
output=nc_ensmean_sub_rel_out,
options='-f nc',
force=True,
returnCdf=False)
def selyear_index(cdo, index, out_dir, nc_in, nc_in2=None):
'''
For indexes that need a selyear to generate a timeseries, like sdii
'''
import shutil
import os.path
import pathlib
from indices_misc import logger, clean
index_cdo_function = getattr(
cdo, index['cdo_fun']
) # get the cdo function coressponding to the input index
if index_cdo_function is None:
logger.debug(clean(("Error, not an index from etccdi")))
return
if nc_in2 is None:
nc_in2 = ''
first_year_file = ((nc_in.split("_"))[-2].split("-")[0])[0:4]
last_year_file = ((nc_in.split("_"))[-2].split("-")[1])[0:4]
# To avoid calculating indices again for the historical data
if False and "rcp85" in nc_in:
first_year = "2005"
else:
first_year = first_year_file
nc_out_array = ""
if first_year_file != "1861" or last_year_file != "2090":
logger.debug(
clean(("Error in first or last year " + first_year_file + "-" +
last_year_file)))
return
nc_out_fldmean = (out_dir + '/' + pathlib.Path(nc_in).stem + "_" +
index['name'] + "_ts.nc").replace(
first_year_file, first_year)
# Create nc files for field mean and year mean.
if os.path.exists(nc_out_fldmean): # False:
logger.debug(clean(nc_out_fldmean + " already exists"))
return
pathlib.Path(out_dir + '/years/').mkdir(parents=True, exist_ok=True)
add_params = ''
if 'add_params' in index:
add_params = index['add_params']
# loop all years in the range and calculate the index
for year in range(int(first_year), int(last_year_file) + 1):
cdo_selyear_command = "-selyear," + str(year)
nc_out = out_dir + '/years/' + pathlib.Path(
nc_in).stem + "_" + index['name'] + str(year) + ".nc"
logger.debug(clean((nc_out)))
index_cdo_function(add_params,
input=cdo_selyear_command + " " + nc_in + " " +
nc_in2,
output=nc_out,
options='-f nc',
force=False,
returnCdf=False)
# add the out nc to the array
nc_out_array = nc_out_array + " " + nc_out
# after calculating index for each year, merge them.
nc_out_merge = (
out_dir + '/' +
pathlib.Path(nc_in).stem + "_" + index['name'] + ".nc").replace(
first_year_file,
first_year) # adapt the name to the actual first year used
cdo.mergetime(input=nc_out_array,
output=nc_out_merge,
options='-f nc',
force=False,
returnCdf=False)
# fldmean to obtain a timeseries
cdo.fldmean(input="-setreftime,1850-01-01,00:00:00 " + nc_out_merge,
output=nc_out_fldmean,
options="-f nc",
force=False,
returnCdf=False)
logger.debug(clean(("")))
logger.debug(clean((nc_out_fldmean)))
logger.debug(clean(("")))
try:
shutil.rmtree(out_dir + '/years/')
except OSError as e:
logger.debug(clean(("Error: %s" % (e.strerror))))
def percentile_index(cdo, index, out_dir, nc_in, nc_in2=None):
'''
Calculate percentile indices.
For temperature indices that require bootstrapping first calculates ydrunmin and ydrunmax (if not already generated)
For precipitation indices just uses the timmin and timmax
'''
from indices_misc import logger, clean
import pathlib
index_cdo_function = getattr(
cdo, index['cdo_fun']
) # get the cdo function coressponding to the input index
if index_cdo_function is None:
logger.debug(clean(("Error, not an index from etccdi")))
return -1
nc_out_runmin = (out_dir + '/' + pathlib.Path(nc_in).stem + "_runmin.nc")
nc_out_runmax = (out_dir + '/' + pathlib.Path(nc_in).stem + "_runmax.nc")
nc_out_percentile = (out_dir + '/' + pathlib.Path(nc_in).stem + "_" +
index['name'] + ".nc")
windowDays = ""
bootstrapping = "1961,1990"
if index['isTemp']:
windowDays = "5," # Number of timestamps required for Temp percentiles
# Genreate runmin and runmax (if not already generated)
cdo.ydrunmin(windowDays,
input=nc_in,
output=nc_out_runmin,
options='-f nc',
force=False,
returnCdf=False)
cdo.ydrunmax(windowDays,
input=nc_in,
output=nc_out_runmax,
options='-f nc',
force=False,
returnCdf=False)
else:
# Genreate runmin and runmax (if not already generated)
# timmin, timmax calculate the min and max in time (so the result is a point for each grid)
# setrtomiss sets the values in the range to missing value. Because we are interested in days with precipitation larger than 1, wet-day precipitation (PR > 1 mm)
# then set precipitation in range -50,1 to the missing value (to ignore it)
cdo.timmin(input="-setrtomiss,-50,1 " + nc_in,
output=nc_out_runmin,
options='-f nc',
force=False,
returnCdf=False)
cdo.timmax(input="-setrtomiss,-50,1 " + nc_in,
output=nc_out_runmax,
options='-f nc',
force=False,
returnCdf=False)
logger.debug(clean((nc_out_runmin)))
logger.debug(clean((nc_out_runmax)))
logger.debug(clean((nc_out_percentile)))
index_cdo_function(windowDays + bootstrapping,
input=nc_in + " " + nc_out_runmin + " " + nc_out_runmax,
output=nc_out_percentile,
options='-f nc',
force=False,
returnCdf=False)
logger.debug(clean(("")))
def direct_periods_index(cdo, index, out_dir, nc_in, nc_in2=None):
'''
For indices that give a single value for the timeperiod, like sdii
Generate given index for the following 4 periods:
Past : 1861 1890
Reference : 1961 1990
Present : 1991 2020
Future : 2061 2090
The index is calculated for seasons ANN DJF and JJA
'''
import pathlib
from indices_misc import logger, clean
from indices_misc import period_range_array, period_name_array
from os import path
index_cdo_function = getattr(
cdo, index['cdo_fun']
) # get the cdo function coressponding to the input index
if index_cdo_function is None:
logger.debug(clean(("Error, not an index from etccdi")))
return -1
print(nc_in2)
if nc_in2 is None:
nc_in2 = ''
if 'add_fun' in index:
index_add_function = getattr(
cdo, index['add_fun']
) # get the cdo function coressponding to the input index
if index_add_function is None:
logger.debug(clean(("Error, not an function from cdo")))
return -1
add_params = ''
if 'add_params' in index:
add_params = index['add_params']
for season in index['seasons']:
pathlib.Path(out_dir + '/' + season).mkdir(parents=True, exist_ok=True)
cdo_season_command = "-select,season=" + season
for year_range, name in zip(period_range_array, period_name_array):
cdo_year_command = "-selyear," + year_range
nc_out = out_dir + "/" + season + "/" + pathlib.Path(
nc_in).stem + "_" + index['name'] + "_" + name + ".nc"
if path.exists(nc_out): # False:
logger.debug(clean(("%s already exists", nc_out)))
return
logger.debug(clean((nc_out)))
if 'add_fun' not in index:
index_cdo_function(add_params,
input=cdo_year_command + " " +
cdo_season_command + " " + nc_in + " " +
nc_in2,
output=nc_out,
options='-f nc',
force=False,
returnCdf=False)
else:
cdo_function_command = '-' + index['cdo_fun'] + ',' + add_params
index_add_function(
index['add_fun_params'],
input=cdo_function_command + ' ' + cdo_year_command + " " +
cdo_season_command + " " + nc_in + " " + nc_in2,
output=nc_out,
options='-f nc',
force=False,
returnCdf=False)