def test_get_time():
timestamps = utils.get_time(local_path(TESTDATA["cmip5_tasmax_2007_nc"]))
assert 12 == len(timestamps)
timestamps = utils.get_time(local_path(TESTDATA["cordex_tasmax_2007_nc"]))
assert 12 == len(timestamps)
values = utils.get_values(
[local_path(TESTDATA["cordex_tasmax_2006_nc"]), local_path(TESTDATA["cordex_tasmax_2007_nc"])]
)
assert 23 == len(values)
def eval_timerange(resource, time_range):
"""
quality checker if given time_range is covered by timesteps in resource files
:param resource: input netCDF files
:param time_range: start and end date of time range [datetime,datetime]
:returns [datetime,datetime]: time_range
"""
from flyingpigeon.utils import get_time
logger.info('time_range: %s' % time_range)
if type(resource) != str:
resource.sort()
time = get_time(resource)
start = time[0]
end = time[-1]
if (time_range[0] > start or time_range[0] < end ):
logger.debug('time range start %s not in input dataset covering: %s to %s' % (time_range[0] , start, end))
time_range[0] = start
logger.debug('time_range start changed to first timestep of dataset')
if (time_range[1] > end or time_range[1] < start ):
logger.debug('time range end %s not in input dataset covering: %s to %s' % (time_range[0] , start, end))
time_range[1] = end
logger.debug('time_range end changed to last timestep of dataset')
if (time_range[0] > time_range[1]):
time_range = reversed(time_range)
logger.debug('time range reversed! start was later than end ')
logger.info('time range start and end set')
return time_range
def execute(self):
from flyingpigeon.ocgis_module import call
from flyingpigeon.utils import sort_by_filename, archive, get_values, get_time
ncs = self.getInputValues(identifier='netcdf_file')
logger.info("ncs: %s " % ncs)
coords = self.getInputValues(identifier='coords')
logger.info("coords %s", coords)
filenames = []
nc_exp = sort_by_filename(ncs, historical_concatination=True)
#(fp_tar, tarout_file) = tempfile.mkstemp(dir=".", suffix='.tar')
#tar = tarfile.open(tarout_file, "w")
from numpy import savetxt, column_stack
from shapely.geometry import Point
for key in nc_exp.keys():
try:
logger.info('start calculation for %s ' % key )
ncs = nc_exp[key]
times = get_time(ncs)
concat_vals = ['%s-%02d-%02d_%02d:%02d:%02d' %
(t.year, t.month, t.day, t.hour, t.minute, t.second) for t in times]
header = 'date_time'
filename = '%s.csv' % key
filenames.append(filename)
for p in coords:
try:
self.status.set('processing point : {0}'.format(p), 20)
# define the point:
p = p.split(',')
point = Point(float(p[0]), float(p[1]))
# get the values
timeseries = call(resource=ncs, geom=point, select_nearest=True)
vals = get_values(timeseries)
# concatination of values
header = header + ',%s-%s' % (p[0], p[1])
concat_vals = column_stack([concat_vals, vals])
except Exception as e:
logger.debug('failed for point %s %s' % (p , e))
self.status.set('*** all points processed for {0} ****'.format(key), 50)
savetxt(filename, concat_vals, fmt='%s', delimiter=',', header=header)
except Exception as e:
logger.debug('failed for %s %s' % (key, e))
### set the outputs
self.status.set('*** creating output tar archive ****',90)
tarout_file = archive(filenames)
self.tarout.setValue( tarout_file )
def execute(self):
from flyingpigeon.ocgis_module import call
from flyingpigeon.utils import get_time, get_variable, sort_by_filename
from datetime import datetime as dt
from netCDF4 import Dataset
from numpy import savetxt, column_stack, squeeze
ncs = self.getInputValues(identifier='netcdf_file')
logging.info("ncs: %s " % ncs)
coords = self.getInputValues(identifier='coords')
logging.info("coords %s", coords)
nc_exp = sort_by_filename(ncs) # dictionary {experiment:[files]}
filenames = []
(fp_tar, tarout_file) = tempfile.mkstemp(dir=".", suffix='.tar')
tar = tarfile.open(tarout_file, "w")
for key in nc_exp.keys():
logging.info('start calculation for %s ' % key )
ncs = nc_exp[key]
nc = ncs[0]
times = get_time(nc)
var = get_variable(nc)
concat_vals = [dt.strftime(t, format='%Y-%d-%m_%H:%M:%S') for t in times]
header = 'date_time'
filename = '%s.csv' % key
filenames.append(filename)
for ugid, p in enumerate(coords, start=1):
self.status.set('processing point : {0}'.format(p), 20)
p = p.split(',')
self.status.set('splited x and y coord : {0}'.format(p), 20)
point = Point(float(p[0]), float(p[1]))
#get the timeseries at gridpoint
timeseries = call(resource=ncs, geom=point, select_nearest=True)
ds = Dataset(timeseries)
vals = squeeze(ds.variables[var])
header = header + ',%s_%s' % (p[0], p[1])
concat_vals = column_stack([concat_vals, vals])
savetxt(filename, concat_vals, fmt='%s', delimiter=',', header=header)
tar.add( filename )
tar.close()
self.tarout.setValue( tarout_file )
def spaghetti(resouces, variable=None, title=None, dir_out=None):
"""
retunes a png file containing the appropriate spaghetti plot.
:param resouces: list of files containing the same variable
:param variable: variable to be visualised, if None (default) variable will be detected
:param title: sting to be used as title
:param dir_out: directory for output files
"""
fig = plt.figure(figsize=(20,10), dpi=600, facecolor='w', edgecolor='k')
logger.debug('Start visualisation spagetti plot')
# === prepare invironment
if type(resouces) == str:
resouces = list([resouces])
if variable == None:
variable = utils.get_variable(resouces[0])
if title == None:
title = "Field mean of %s " % variable
if dir_out == None:
dir_out = os.curdir
try:
o1 , output_png = mkstemp(dir=dir_out, suffix='.png')
for c , nc in enumerate(resouces):
# get timestapms
try:
d = utils.get_time(nc) # [datetime.strptime(elem, '%Y-%m-%d') for elem in strDate[0]]
dt = [datetime.strptime(str(i), '%Y-%m-%d %H:%M:%S') for i in d ]
ds=Dataset(nc)
data = np.squeeze(ds.variables[variable][:])
if len(data.shape) == 3:
meanData = np.mean(data,axis=1)
ts = np.mean(meanData,axis=1)
else:
ts = data
plt.plot( dt,ts )
#fig.line( dt,ts )
except Exception as e:
logger.debug('lineplot failed for %s: %s\n' % (nc, e))
# plot into current figure
# , legend= nc
#fig.legend()[0].orientation = "bottom_left"
# fig.legend().orientation = "bottom_left"
plt.title(title, fontsize=20)
plt.grid()# .grid_line_alpha=0.3
#lt.rcParams.update({'font.size': 22})
#window_size = 30
#window = np.ones(window_size)/float(window_size)
fig.savefig(output_png)
#bplt.hold('off')
plt.close()
logger.debug('timesseries spagetti plot done for %s with %s lines.'% (variable, c))
except Exception as e:
msg = 'matplotlib spagetti plot failed for %s' % variable
logger.debug(msg)
#raise Exception(msg)
return output_png
def execute(self):
logger.info('Start process')
from datetime import datetime as dt
from flyingpigeon import weatherregimes as wr
from tempfile import mkstemp
################################
# reading in the input arguments
################################
try:
resource = self.getInputValues(identifier='resource')
url_Rdat = self.getInputValues(identifier='Rdat')[0]
url_dat = self.getInputValues(identifier='dat')[0]
url_ref_file = self.getInputValues(identifier='netCDF') # can be None
season = self.getInputValues(identifier='season')[0]
period = self.getInputValues(identifier='period')[0]
anualcycle = self.getInputValues(identifier='anualcycle')[0]
except Exception as e:
logger.debug('failed to read in the arguments %s ' % e)
try:
start = dt.strptime(period.split('-')[0] , '%Y%m%d')
end = dt.strptime(period.split('-')[1] , '%Y%m%d')
# kappa = int(self.getInputValues(identifier='kappa')[0])
logger.info('period %s' % str(period))
logger.info('season %s' % str(season))
logger.info('read in the arguments')
logger.info('url_ref_file: %s' % url_ref_file)
logger.info('url_Rdat: %s' % url_Rdat)
logger.info('url_dat: %s' % url_dat)
except Exception as e:
logger.debug('failed to convert arguments %s ' % e)
############################
# fetching trainging data
############################
from flyingpigeon.utils import download, get_time
from os.path import abspath
try:
dat = abspath(download(url_dat))
Rdat = abspath(download(url_Rdat))
logger.info('training data fetched')
except Exception as e:
logger.error('failed to fetch training data %s' % e)
############################################################
### get the required bbox and time region from resource data
############################################################
# from flyingpigeon.weatherregimes import get_level
from flyingpigeon.ocgis_module import call
from flyingpigeon.utils import get_variable
time_range = [start, end]
variable = get_variable(resource)
if len(url_ref_file) > 0:
ref_file = download(url_ref_file[0])
model_subset = call(resource=resource, variable=variable,
time_range=time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap',
regrid_destination=ref_file, regrid_options='bil')
logger.info('Dataset subset with regridding done: %s ' % model_subset)
else:
model_subset = call(resource=resource, variable=variable,
time_range=time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap',
)
logger.info('Dataset time period extracted: %s ' % model_subset)
##############################################
### computing anomalies
##############################################
cycst = anualcycle.split('-')[0]
cycen = anualcycle.split('-')[0]
reference = [dt.strptime(cycst,'%Y%m%d'), dt.strptime(cycen,'%Y%m%d')]
model_anomal = wr.get_anomalies(model_subset, reference=reference)
#####################
### extracting season
#####################
model_season = wr.get_season(model_anomal, season=season)
#######################
### call the R scripts
#######################
import shlex
import subprocess
from flyingpigeon import config
from os.path import curdir, exists, join
try:
rworkspace = curdir
Rsrc = config.Rsrc_dir()
Rfile = 'weatherregimes_projection.R'
yr1 = start.year
yr2 = end.year
time = get_time(model_season, format='%Y%m%d')
#ip, output_graphics = mkstemp(dir=curdir ,suffix='.pdf')
ip, file_pca = mkstemp(dir=curdir ,suffix='.txt')
ip, file_class = mkstemp(dir=curdir ,suffix='.Rdat')
ip, output_frec = mkstemp(dir=curdir ,suffix='.txt')
args = ['Rscript', join(Rsrc,Rfile), '%s/' % curdir,
'%s/' % Rsrc,
'%s' % model_season,
'%s' % variable,
'%s' % str(time).strip("[]").replace("'","").replace(" ",""),
# '%s' % output_graphics,
'%s' % dat,
'%s' % Rdat,
'%s' % file_pca,
'%s' % file_class,
'%s' % output_frec,
'%s' % season,
'%s' % start.year,
'%s' % end.year,
'%s' % 'MODEL']
logger.info('Rcall builded')
except Exception as e:
msg = 'failed to build the R command %s' % e
logger.error(msg)
raise Exception(msg)
try:
output,error = subprocess.Popen(args, stdout = subprocess.PIPE, stderr= subprocess.PIPE).communicate() #, shell=True
logger.info('R outlog info:\n %s ' % output)
logger.debug('R outlog errors:\n %s ' % error)
if len(output) > 0:
self.status.set('**** weatherregime in R suceeded', 90)
else:
logger.error('NO! output returned from R call')
except Exception as e:
msg = 'weatherregime in R %s ' % e
logger.error(msg)
raise Exception(msg)
############################################
### set the outputs
############################################
#self.Routput_graphic.setValue( output_graphics )
self.output_pca.setValue( file_pca )
self.output_classification.setValue( file_class )
self.output_netcdf.setValue( model_season )
self.output_frequency.setValue( output_frec )
def execute(self):
logger.info('Start process')
try:
logger.info('read in the arguments')
resources = self.getInputValues(identifier='resources')
method = self.getInputValues(identifier='method')
time_region = self.getInputValues(identifier='time_region')[0]
bbox = self.getInputValues(identifier='BBox')[0]
logger.info('bbox %s' % str(bbox))
logger.info('time_region %s' % str(time_region))
logger.info('method: %s' % str(method))
except Exception as e:
logger.error('failed to read in the arguments %s ' % e)
#bbox = '-80,22.5,50,70'
logger.info('bbox is set to %s' % bbox)
#####################
### get the required bbox from resource
#####################
# from flyingpigeon.ocgis_module import call
from flyingpigeon.utils import sort_by_filename, get_time # , calc_grouping
from flyingpigeon import weatherclass as wc
from flyingpigeon.visualisation import plot_tSNE, plot_kMEAN, concat_images, plot_pressuremap
from datetime import datetime as dt
from numpy import savetxt, column_stack
import tarfile
from cdo import *
cdo = Cdo()
# grouping = calc_grouping(time_region)
ncs = sort_by_filename(resources, historical_concatination=True)
png_clusters = []
txt_info = []
png_pressuremaps = []
try:
# open tar files
tar_info = tarfile.open('info.tar', "w")
logger.info('tar files prepared')
except:
msg = 'tar file preparation failed'
logger.exception(msg)
raise Exception(msg)
for key in ncs.keys():
if len(ncs[key])>1:
input = cdo.timmerge(input=ncs[key], output='merge.nc' )
elif len(ncs[key])==1:
input = ncs[key]
else:
logger.debug('invalid number of input files for dataset %s' % key)
#for tr in time_region:
if not time_region == 'None':
nc_grouped = cdo.selmon(time_region, input=input, output='grouped.nc')
else:
nc_grouped = input
# for bb in bbox:
nc = cdo.sellonlatbox('%s' % bbox, input=nc_grouped, output='subset.nc')
logger.info('nc subset: %s ' % nc)
try:
vals, pca = wc.get_pca(nc)
logger.info('PCa calculated')
except:
logger.debug('failed to calculate PCs')
raise
for md in method:
try:
if md == 'tSNE':
data = wc.calc_tSNE(pca)
png_clusters.append(plot_tSNE(data,title='tSNE month: %s [lonlat: %s]' % (time_region,bbox), sub_title='file: %s' % key))
logger.info('tSNE calculated for %s ' % key)
if md == 'kMEAN':
kmeans = wc.calc_kMEAN(pca)
c = kmeans.predict(pca)
times = get_time(nc)
timestr = [dt.strftime(t, format='%Y-%d-%m_%H:%M:%S') for t in times]
tc = column_stack([timestr, c])
fn = '%s.txt' % key
savetxt(fn, tc, fmt='%s', header='Date_Time WeatherRegime')
tar_info.add(fn) #, arcname = basename(nc)
png_clusters.append(plot_kMEAN(kmeans, pca, title='kMEAN month: %s [lonlat: %s]' % (time_region,bbox), sub_title='file: %s' % key))
logger.info('kMEAN calculated for %s ' % key)
subplots = []
for i in range(4):
subplots.append(plot_pressuremap((vals[c==i]/100), title='Weather Regime %s: Month %s ' % (i, time_region), sub_title='file: %s' % key))
from PIL import Image
import sys
from tempfile import mkstemp
open_subplots = map(Image.open, subplots)
w = max(i.size[0] for i in open_subplots)
h = max(i.size[1] for i in open_subplots)
result = Image.new("RGB", (w*2, h*2))
# p = h / len(open_subplots)
c = 0
for i ,iw in enumerate([0,w]):
for j, jh in enumerate([0,h]):
oi = open_subplots[c]
c = c +1
cw = oi.size[0]
ch = oi.size[1]
box = [iw,jh,iw+cw,jh+ch]
result.paste(oi, box=box)
ip, pressuremap = mkstemp(dir='.',suffix='.png')
result.save(pressuremap)
png_pressuremaps.append(pressuremap)
except:
logger.debug('faild to calculate cluster for %s' % key )
raise
c_clusters = concat_images(png_clusters)
c_maps = concat_images(png_pressuremaps)
try:
tar_info.close()
logger.info('tar files closed')
except Exception as e:
logger.exception('tar file closing failed')
# call
# self.output_nc.setValue( nc )
self.output_clusters.setValue( c_clusters )
self.output_maps.setValue( c_maps )
self.output_info.setValue('info.tar')
def spaghetti(resouces, variable=None, title=None, dir_out=None):
"""
creates a png file containing the appropriate spaghetti plot as a field mean of the values.
:param resouces: list of files containing the same variable
:param variable: variable to be visualised. If None (default), variable will be detected
:param title: string to be used as title
:param dir_out: directory for output files
:retruns str: path to png file
"""
try:
fig = plt.figure(figsize=(20,10), dpi=600, facecolor='w', edgecolor='k')
logger.debug('Start visualisation spaghetti plot')
# === prepare invironment
if type(resouces) != list:
resouces = [resouces]
if variable == None:
variable = utils.get_variable(resouces[0])
if title == None:
title = "Field mean of %s " % variable
if dir_out == None:
dir_out = os.curdir
logger.info('plot values preparation done')
except Exception as e:
msg = "plot values preparation failed: %s" % (e)
logger.exception(msg)
raise Exception(msg)
try:
o1 , output_png = mkstemp(dir=dir_out, suffix='.png')
for c , nc in enumerate(resouces):
# get timestapms
try:
d = utils.get_time(nc) # [datetime.strptime(elem, '%Y-%m-%d') for elem in strDate[0]]
dt = [datetime.strptime(str(i), '%Y-%m-%d %H:%M:%S') for i in d ]
ds=Dataset(nc)
data = np.squeeze(ds.variables[variable][:])
if len(data.shape) == 3:
meanData = np.mean(data,axis=1)
ts = np.mean(meanData,axis=1)
else:
ts = data[:]
plt.plot( dt,ts )
#fig.line( dt,ts )
except Exception as e:
msg = "lineplot failed for %s" % (nc)
logger.exception(msg)
raise Exception(msg)
plt.title(title, fontsize=20)
plt.grid()
fig.savefig(output_png)
plt.close()
logger.info('timeseries spaghetti plot done for %s with %s lines.'% (variable, c))
except Exception as e:
msg = 'matplotlib spaghetti plot failed: %s' % e
logger.exception(msg)
raise Exception(msg)
return output_png
def calc_indice_percentile(resources=[], variable=None,
prefix=None, indices='TG90p', refperiod=None,
groupings='yr', polygons=None, percentile=90, mosaik = False,
dir_output=None, dimension_map = None):
"""
Calculates given indices for suitable files in the appopriate time grouping and polygon.
:param resource: list of filenames in drs convention (netcdf)
:param variable: variable name to be selected in the in netcdf file (default=None)
:param indices: list of indices (default ='TG90p')
:param prefix: filename prefix
:param refperiod: reference refperiod touple = (start,end)
:param grouping: indices time aggregation (default='yr')
:param dir_output: output directory for result file (netcdf)
:param dimension_map: optional dimension map if different to standard (default=None)
:return: list of netcdf files with calculated indices. Files are saved into out_dir
"""
from os.path import join, dirname, exists
from os import remove
import uuid
from numpy import ma
from datetime import datetime as dt
from flyingpigeon.ocgis_module import call
from flyingpigeon.subset import clipping
from flyingpigeon.utils import get_values, get_time
if type(resources) != list:
resources = list([resources])
if type(indices) != list:
indices = list([indices])
if type(groupings) != list:
groupings = list([groupings])
if type(refperiod) == list:
refperiod = refperiod[0]
if refperiod != None:
start = dt.strptime(refperiod.split('-')[0] , '%Y%m%d')
end = dt.strptime(refperiod.split('-')[1] , '%Y%m%d')
time_range = [start, end]
else:
time_range = None
if dir_output != None:
if not exists(dir_output):
makedirs(dir_output)
########################################################################################################################
# Compute a custom percentile basis using ICCLIM. ######################################################################
########################################################################################################################
from ocgis.contrib import library_icclim as lic
nc_indices = []
nc_dic = sort_by_filename(resources)
for grouping in groupings:
calc_group = calc_grouping(grouping)
for key in nc_dic.keys():
resource = nc_dic[key]
if variable == None:
variable = get_variable(resource)
if polygons == None:
nc_reference = call(resource=resource,
prefix=str(uuid.uuid4()),
time_range=time_range,
output_format='nc',
dir_output=dir_output)
else:
nc_reference = clipping(resource=resource,
prefix=str(uuid.uuid4()),
time_range=time_range,
output_format='nc',
polygons=polygons,
dir_output=dir_output,
mosaik = mosaik)
arr = get_values(nc_files=nc_reference)
dt_arr = get_time(nc_files=nc_reference)
arr = ma.masked_array(arr)
dt_arr = ma.masked_array(dt_arr)
percentile = percentile
window_width = 5
for indice in indices:
name = indice.replace('_', str(percentile))
var = indice.split('_')[0]
operation = None
if 'T' in var:
if percentile >= 50:
operation = 'Icclim%s90p' % var
func = 'icclim_%s90p' % var # icclim_TG90p
else:
operation = 'Icclim%s10p' % var
func = 'icclim_%s10p' % var
################################
# load the appropriate operation
################################
ops = [op for op in dir(lic) if operation in op]
if len(ops) == 0:
raise Exception("operator does not exist %s", operation)
exec "percentile_dict = lic.%s.get_percentile_dict(arr, dt_arr, percentile, window_width)" % ops[0]
calc = [{'func': func, 'name': name, 'kwds': {'percentile_dict': percentile_dict}}]
if polygons == None:
nc_indices.append(call(resource=resource,
prefix=key.replace(variable,name).replace('_day_', '_%s_' % grouping),
calc=calc,
calc_grouping=calc_group,
output_format='nc',
dir_output=dir_output))
else:
nc_indices.extend(clipping(resource=resource,
prefix=key.replace(variable,name).replace('_day_', '_%s_' % grouping),
calc=calc,
calc_grouping=calc_group,
output_format='nc',
dir_output=dir_output,
polygons=polygons,
mosaik = mosaik,
))
return nc_indices
def execute(self):
logger.info('Start process')
from datetime import datetime as dt
from flyingpigeon import weatherregimes as wr
from tempfile import mkstemp
################################
# reading in the input arguments
################################
try:
resource = self.getInputValues(identifier='resource')
url_Rdat = self.getInputValues(identifier='Rdat')[0]
url_dat = self.getInputValues(identifier='dat')[0]
url_ref_file = self.getInputValues(
identifier='netCDF') # can be None
season = self.getInputValues(identifier='season')[0]
period = self.getInputValues(identifier='period')[0]
anualcycle = self.getInputValues(identifier='anualcycle')[0]
except Exception as e:
logger.debug('failed to read in the arguments %s ' % e)
try:
start = dt.strptime(period.split('-')[0], '%Y%m%d')
end = dt.strptime(period.split('-')[1], '%Y%m%d')
# kappa = int(self.getInputValues(identifier='kappa')[0])
logger.info('period %s' % str(period))
logger.info('season %s' % str(season))
logger.info('read in the arguments')
logger.info('url_ref_file: %s' % url_ref_file)
logger.info('url_Rdat: %s' % url_Rdat)
logger.info('url_dat: %s' % url_dat)
except Exception as e:
logger.debug('failed to convert arguments %s ' % e)
############################
# fetching trainging data
############################
from flyingpigeon.utils import download, get_time
from os.path import abspath
try:
dat = abspath(download(url_dat))
Rdat = abspath(download(url_Rdat))
logger.info('training data fetched')
except Exception as e:
logger.error('failed to fetch training data %s' % e)
############################################################
### get the required bbox and time region from resource data
############################################################
# from flyingpigeon.weatherregimes import get_level
from flyingpigeon.ocgis_module import call
from flyingpigeon.utils import get_variable
time_range = [start, end]
variable = get_variable(resource)
if len(url_ref_file) > 0:
ref_file = download(url_ref_file[0])
model_subset = call(
resource=resource,
variable=variable,
time_range=
time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap',
regrid_destination=ref_file,
regrid_options='bil')
logger.info('Dataset subset with regridding done: %s ' %
model_subset)
else:
model_subset = call(
resource=resource,
variable=variable,
time_range=
time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap',
)
logger.info('Dataset time period extracted: %s ' % model_subset)
##############################################
### computing anomalies
##############################################
cycst = anualcycle.split('-')[0]
cycen = anualcycle.split('-')[0]
reference = [
dt.strptime(cycst, '%Y%m%d'),
dt.strptime(cycen, '%Y%m%d')
]
model_anomal = wr.get_anomalies(model_subset, reference=reference)
#####################
### extracting season
#####################
model_season = wr.get_season(model_anomal, season=season)
#######################
### call the R scripts
#######################
import shlex
import subprocess
from flyingpigeon import config
from os.path import curdir, exists, join
try:
rworkspace = curdir
Rsrc = config.Rsrc_dir()
Rfile = 'weatherregimes_projection.R'
yr1 = start.year
yr2 = end.year
time = get_time(model_season, format='%Y%m%d')
#ip, output_graphics = mkstemp(dir=curdir ,suffix='.pdf')
ip, file_pca = mkstemp(dir=curdir, suffix='.txt')
ip, file_class = mkstemp(dir=curdir, suffix='.Rdat')
ip, output_frec = mkstemp(dir=curdir, suffix='.txt')
args = [
'Rscript',
join(Rsrc, Rfile),
'%s/' % curdir,
'%s/' % Rsrc,
'%s' % model_season,
'%s' % variable,
'%s' % str(time).strip("[]").replace("'", "").replace(" ", ""),
# '%s' % output_graphics,
'%s' % dat,
'%s' % Rdat,
'%s' % file_pca,
'%s' % file_class,
'%s' % output_frec,
'%s' % season,
'%s' % start.year,
'%s' % end.year,
'%s' % 'MODEL'
]
logger.info('Rcall builded')
except Exception as e:
msg = 'failed to build the R command %s' % e
logger.error(msg)
raise Exception(msg)
try:
output, error = subprocess.Popen(
args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate() #, shell=True
logger.info('R outlog info:\n %s ' % output)
logger.debug('R outlog errors:\n %s ' % error)
if len(output) > 0:
self.status.set('**** weatherregime in R suceeded', 90)
else:
logger.error('NO! output returned from R call')
except Exception as e:
msg = 'weatherregime in R %s ' % e
logger.error(msg)
raise Exception(msg)
############################################
### set the outputs
############################################
#self.Routput_graphic.setValue( output_graphics )
self.output_pca.setValue(file_pca)
self.output_classification.setValue(file_class)
self.output_netcdf.setValue(model_season)
self.output_frequency.setValue(output_frec)
def spaghetti(resouces, variable=None, title=None, dir_out=None):
"""
creates a png file containing the appropriate spaghetti plot as a field mean of the values.
:param resouces: list of files containing the same variable
:param variable: variable to be visualised. If None (default), variable will be detected
:param title: string to be used as title
:param dir_out: directory for output files
:retruns str: path to png file
"""
try:
fig = plt.figure(figsize=(20, 10),
dpi=600,
facecolor='w',
edgecolor='k')
logger.debug('Start visualisation spaghetti plot')
# === prepare invironment
if type(resouces) != list:
resouces = [resouces]
if variable is None:
variable = utils.get_variable(resouces[0])
if title is None:
title = "Field mean of %s " % variable
if dir_out is None:
dir_out = os.curdir
logger.info('plot values preparation done')
except:
msg = "plot values preparation failed"
logger.exception(msg)
raise Exception(msg)
try:
o1, output_png = mkstemp(dir=dir_out, suffix='.png')
for c, nc in enumerate(resouces):
# get timestapms
try:
d = utils.get_time(
nc
) # [datetime.strptime(elem, '%Y-%m-%d') for elem in strDate[0]]
dt = [
datetime.strptime(str(i), '%Y-%m-%d %H:%M:%S') for i in d
]
ds = Dataset(nc)
data = np.squeeze(ds.variables[variable][:])
if len(data.shape) == 3:
meanData = np.mean(data, axis=1)
ts = np.mean(meanData, axis=1)
else:
ts = data[:]
plt.plot(dt, ts)
# fig.line( dt,ts )
except:
msg = "spaghetti plot failed for"
logger.exception(msg)
raise Exception(msg)
plt.title(title, fontsize=20)
plt.grid()
fig.savefig(output_png)
plt.close()
logger.info('timeseries spaghetti plot done for %s with %s lines.' %
(variable, c))
except:
msg = 'matplotlib spaghetti plot failed'
logger.exception(msg)
raise Exception(msg)
return output_png
from os import listdir
from os.path import join
from flyingpigeon import utils
from flyingpigeon import metadata as md
from pandas import DataFrame
from flyingpigeon import calculation as cal
p = '/home/nils/data/AFR-44/tas/'
ncs = [
join(p, nc) for nc in listdir(p)
if not 'tas_AFR-44_MOHC-HadGEM2-ES_historical_r1i1p1_KNMI-RACMO22T_v2_day'
in nc
]
ncs_dic = utils.sort_by_filename(ncs)
ts = utils.get_time(ncs_dic[ncs_dic.keys()[0]])
data = cal.fieldmean(ncs_dic[ncs_dic.keys()[0]])
def method_A(resource=[], start=None, end=None, timeslice=20,
variable=None, title=None, cmap='seismic' ):
"""returns the result
:param resource: list of paths to netCDF files
:param start: beginning of reference period (if None (default), the first year of the consistent ensemble will be detected)
:param end: end of comparison period (if None (default), the last year of the consistent ensemble will be detected)
:param timeslice: period length for mean calculation of reference and comparison period
:param variable: variable name to be detected in the netCDF file. If not set (not recommended), the variable name will be detected
:param title: str to be used as title for the signal mal
:param cmap: define the color scheme for signal map plotting
:return: signal.nc, low_agreement_mask.nc, high_agreement_mask.nc, graphic.png, text.txt
"""
from os.path import split
from cdo import Cdo
cdo = Cdo()
cdo.forceOutput = True
try:
# preparing the resource
# from flyingpigeon.ocgis_module import call
file_dic = sort_by_filename(resource, historical_concatination = True)
#print file_dic
logger.info('file names sorted experimets: %s' % len(file_dic.keys()))
except Exception as e:
msg = 'failed to sort the input files'
logger.exception(msg)
raise Exception(msg)
try:
mergefiles = []
for key in file_dic.keys():
if type(file_dic[key]) == list and len(file_dic[key]) > 1:
input = []
for i in file_dic[key]:
print i
input.extend([i.replace(' ','\\\ ')])
mergefiles.append(cdo.mergetime(input=input, output=key+'_mergetime.nc'))
else:
mergefiles.extend(file_dic[key])
# files.append(cdo.selyear('%s/%s' % (start1,end2), input = tmpfile , output = key+'.nc' )) #python version
logger.info('datasets merged %s ' % mergefiles)
except Exception as e:
msg = 'seltime and mergetime failed %s' % e
logger.exception(msg)
raise Exception(e)
try:
text_src = open('infiles.txt', 'a')
for key in file_dic.keys():
text_src.write(key + '\n')
text_src.close()
except Exception as e:
msg = 'failed to write source textfile'
logger.exception(msg)
raise Exception(msg)
# configure reference and compare period
try:
if start == None:
st_set = set()
en_set = set()
for f in mergefiles:
print f
times = get_time(f)
st_set.update([times[0].year])
if end == None:
en_set.update([times[-1].year])
start = max(st_set)
if end == None:
end = min(en_set)
logger.info('Start and End: %s - %s ' % (start, end))
if start >= end:
logger.error('ensemble is inconsistent!!! start year is later than end year')
except Exception as e:
msg = 'failed to detect start and end times of the ensemble'
logger.exception(msg)
raise Exception(msg)
# set the periodes:
try:
start = int(start)
end = int(end)
if timeslice == None:
timeslice = int((end - start) / 3)
if timeslice == 0:
timeslice = 1
else:
timeslice = int(timeslice)
start1 = start
start2 = start1 + timeslice - 1
end1 = end - timeslice + 1
end2 = end
logger.info('timeslice and periodes set')
except Exception as e:
msg = 'failed to set the periodes'
logger.exception(msg)
raise Exception(msg)
try:
files = []
for i, mf in enumerate(mergefiles):
files.append(cdo.selyear('{0}/{1}'.format(start1,end2), input=[mf.replace(' ','\ ')] , output='file_{0}_.nc'.format(i) )) #python version
logger.info('timeseries selected from defined start to end year')
except Exception as e:
msg = 'seltime and mergetime failed'
logger.exception(msg)
raise Exception(msg)
try:
# ensemble mean
nc_ensmean = cdo.ensmean(input=files , output='nc_ensmean.nc')
logger.info('ensemble mean calculation done')
except Exception as e:
msg = 'ensemble mean failed'
logger.exception(msg)
raise Exception(msg)
try:
# ensemble std
nc_ensstd = cdo.ensstd(input=files , output='nc_ensstd.nc')
logger.info('ensemble std and calculation done')
except Exception as e:
msg = 'ensemble std or failed'
logger.exception(msg)
raise Exception(msg)
# get the get the signal as difference from the beginning (first years) and end period (last years), :
try:
selyearstart = cdo.selyear('%s/%s' % (start1,start2), input = nc_ensmean, output = 'selyearstart.nc' )
selyearend = cdo.selyear('%s/%s' % (end1,end2), input = nc_ensmean, output = 'selyearend.nc' )
meanyearst = cdo.timmean(input = selyearstart, output= 'meanyearst.nc')
meanyearend = cdo.timmean(input = selyearend, output= 'meanyearend.nc')
signal = cdo.sub(input=[meanyearend, meanyearst], output = 'signal.nc')
logger.info('Signal calculation done')
except Exception as e:
msg = 'calculation of signal failed'
logger.exception(msg)
raise Exception(msg)
# get the intermodel standard deviation (mean over whole period)
try:
#std_selyear = cdo.selyear('%s/%s' % (end1,end2), input=nc_ensstd, output='std_selyear.nc')
#std = cdo.timmean(input = std_selyear, output = 'std.nc')
std = cdo.timmean(input = nc_ensstd, output = 'std.nc')
std2 = cdo.mulc('2', input = std, output = 'std2.nc')
logger.info('calculation of internal model std for time period done')
except Exception as e:
msg = 'calculation of internal model std failed'
logger.exception(msg)
raise Exception(msg)
try:
absolut = cdo.abs(input=signal, output='absolut_signal.nc')
high_agreement_mask = cdo.gt(input=[absolut,std2], output= 'large_change_with_high_model_agreement.nc')
low_agreement_mask = cdo.lt(input=[absolut,std], output= 'small_signal_or_low_agreement_of_models.nc')
logger.info('high and low mask done')
except Exception as e:
msg = 'calculation of robustness mask failed'
logger.exception(msg)
raise Exception(msg)
try:
if variable == None:
variable = get_variable(signal)
logger.info('variable to be plotted: %s' % variable)
if title == None:
title='Change of %s (difference of mean %s-%s to %s-%s)' % (variable, end1, end2, start1, start2)
graphic = None
graphic = map_ensembleRobustness(signal, high_agreement_mask, low_agreement_mask,
variable=variable,
cmap=cmap,
title = title)
logger.info('graphic generated')
except Exception as e:
msg('graphic generation failed: %s' % e)
logger.debug(msg)
raise Exception(msg)
return signal, low_agreement_mask, high_agreement_mask, graphic, text_src #
def set_dynamic_md(resource):
"""
Dynamic meta data like time frequency, spatial extent, start/end time, etc.
:param resource: netCDF file where basic meta data should be set
"""
from flyingpigeon.utils import get_timerange, get_time
frequency = get_frequency(resource)
time_coverage_start, time_coverage_end = get_timerange(resource)
time_number_steps = len(get_time(resource))
# max_lat, min_lat, max_lon, min_lat = get_extent(resource)
ds = Dataset(resource, mode='a')
try:
driving_experiment = ds.driving_experiment
ds.delncattr('driving_experiment')
except Exception as e:
LOGGER.error(e)
driving_experiment = ''
try:
driving_experiment_name = ds.driving_experiment_name
ds.delncattr('driving_experiment_name')
except Exception as e:
LOGGER.error(e)
driving_experiment_name = ''
try:
driving_model_ensemble_member = ds.driving_model_ensemble_member
ds.delncattr('driving_model_ensemble_member')
except Exception as e:
LOGGER.error(e)
driving_model_ensemble_member = ''
try:
experiment = ds.experiment
ds.delncattr('experiment')
except Exception as e:
LOGGER.error(e)
experiment = ''
try:
tracking_id = ds.tracking_id
ds.delncattr('tracking_id')
except Exception as e:
LOGGER.error(e)
tracking_id = ''
try:
experiment_id = ds.experiment_id
ds.delncattr('experiment_id')
except Exception as e:
LOGGER.error(e)
experiment_id = ''
try:
project_id = ds.project_id
ds.delncattr('project_id')
except Exception as e:
LOGGER.error(e)
project_id = ''
try:
institution_id = ds.institution_id
ds.delncattr('institution_id')
except Exception as e:
LOGGER.error(e)
institution_id = ''
try:
model_version_id = ds.model_version_id
ds.delncattr('model_version_id')
except Exception as e:
LOGGER.error(e)
model_version_id = ''
try:
driving_model_id = ds.driving_model_id
ds.delncattr('driving_model_id')
except Exception as e:
LOGGER.error(e)
driving_model_id = ''
try:
driving_ensemble_member = ds.driving_ensemble_member
ds.delncattr('driving_ensemble_member')
except Exception as e:
LOGGER.error(e)
driving_ensemble_member = ''
try:
driving_model_id = ds.driving_model_id
ds.delncattr('driving_model_id')
except Exception as e:
LOGGER.error(e)
driving_model_id = ''
try:
model_id = ds.model_id
ds.delncattr('model_id')
except Exception as e:
LOGGER.error(e)
driving_model_id = ''
try:
contact = ds.contact
ds.delncattr('contact')
except Exception as e:
LOGGER.error(e)
contact = ''
try:
driving_experiment_id = ds.driving_experiment_id
ds.delncattr('driving_experiment_id')
except Exception as e:
LOGGER.error(e)
driving_experiment_id = ''
try:
domain = ds.CORDEX_domain
except Exception as e:
LOGGER.error(e)
domain = ''
ds.close()
min_lat, max_lat, min_lon, max_lon = get_extent(resource)
geospatial_increment = get_geospatial_increment(resource)
try:
md_dynamic = {
'in_var_driving_experiment': driving_experiment,
'in_var_driving_experiment_name': driving_experiment_name,
'in_var_driving_model_ensemble_member':
driving_model_ensemble_member,
'in_var_experiment': experiment,
'in_var_experiment_id': experiment_id,
'in_var_project_id': project_id,
'in_var_contact': contact,
'in_var_institution_id': institution_id,
'in_var_model_version_id': model_version_id,
'in_var_driving_model_id': driving_model_id,
'in_var_model_id': model_id,
'in_var_driving_ensemble_member': driving_ensemble_member,
'in_var_driving_experiment_id': driving_experiment_id,
'in_var_domain': domain,
'in_var_tracking_id': tracking_id,
'frequency': frequency,
'time_coverage_start': time_coverage_start,
'time_coverage_end': time_coverage_end,
'time_number_steps': time_number_steps,
#'time_number_gaps': '',
#'cdm_datatype':'' ,
'domain': '%s_subset' % domain,
'geospatial_increment': geospatial_increment,
'geospatial_lat_min': min_lat,
'geospatial_lat_max': max_lat,
'geospatial_lon_min': min_lon,
'geospatial_lon_max': max_lon,
}
#:product = "output" ;
#:rcm_version_id = "v1" ;
#:references = "http://www.smhi.se/en/Research/Research-departments/climate-research-rossby-centre" ;
except Exception as e:
LOGGER.error('failed to populate dynamic metadata dictionary')
try:
ds = Dataset(resource, mode='a')
ds.setncatts(md_dynamic)
ds.close()
except Exception as e:
LOGGER.error(e)
return (resource)
def calc_indice_percentile(resources=[], variable=None,
prefix=None, indices='TG90p', refperiod=None,
groupings='yr', polygons=None, percentile=90, mosaic = False,
dir_output=None, dimension_map = None):
"""
Calculates given indices for suitable files in the appropriate time grouping and polygon.
:param resource: list of filenames in data reference syntax (DRS) convention (netcdf)
:param variable: variable name to be selected in the in netcdf file (default=None)
:param indices: list of indices (default ='TG90p')
:param prefix: filename prefix
:param refperiod: reference period tuple = (start,end)
:param grouping: indices time aggregation (default='yr')
:param dir_output: output directory for result file (netcdf)
:param dimension_map: optional dimension map if different to standard (default=None)
:return: list of netcdf files with calculated indices. Files are saved into out_dir.
"""
from os.path import join, dirname, exists
from os import remove
import uuid
from numpy import ma
from datetime import datetime as dt
from flyingpigeon.ocgis_module import call
from flyingpigeon.subset import clipping
from flyingpigeon.utils import get_values, get_time
if type(resources) != list:
resources = list([resources])
if type(indices) != list:
indices = list([indices])
if type(groupings) != list:
groupings = list([groupings])
if type(refperiod) == list:
refperiod = refperiod[0]
if refperiod != None:
start = dt.strptime(refperiod.split('-')[0] , '%Y%m%d')
end = dt.strptime(refperiod.split('-')[1] , '%Y%m%d')
time_range = [start, end]
else:
time_range = None
if dir_output != None:
if not exists(dir_output):
makedirs(dir_output)
########################################################################################################################
# Compute a custom percentile basis using ICCLIM. ######################################################################
########################################################################################################################
from ocgis.contrib import library_icclim as lic
nc_indices = []
nc_dic = sort_by_filename(resources)
for grouping in groupings:
calc_group = calc_grouping(grouping)
for key in nc_dic.keys():
resource = nc_dic[key]
if variable == None:
variable = get_variable(resource)
if polygons == None:
nc_reference = call(resource=resource,
prefix=str(uuid.uuid4()),
time_range=time_range,
output_format='nc',
dir_output=dir_output)
else:
nc_reference = clipping(resource=resource,
prefix=str(uuid.uuid4()),
time_range=time_range,
output_format='nc',
polygons=polygons,
dir_output=dir_output,
mosaic = mosaic)
arr = get_values(resource=nc_reference)
dt_arr = get_time(resource=nc_reference)
arr = ma.masked_array(arr)
dt_arr = ma.masked_array(dt_arr)
percentile = percentile
window_width = 5
for indice in indices:
name = indice.replace('_', str(percentile))
var = indice.split('_')[0]
operation = None
if 'T' in var:
if percentile >= 50:
operation = 'Icclim%s90p' % var
func = 'icclim_%s90p' % var # icclim_TG90p
else:
operation = 'Icclim%s10p' % var
func = 'icclim_%s10p' % var
################################
# load the appropriate operation
################################
ops = [op for op in dir(lic) if operation in op]
if len(ops) == 0:
raise Exception("operator does not exist %s", operation)
exec "percentile_dict = lic.%s.get_percentile_dict(arr, dt_arr, percentile, window_width)" % ops[0]
calc = [{'func': func, 'name': name, 'kwds': {'percentile_dict': percentile_dict}}]
if polygons == None:
nc_indices.append(call(resource=resource,
prefix=key.replace(variable,name).replace('_day_', '_%s_' % grouping),
calc=calc,
calc_grouping=calc_group,
output_format='nc',
dir_output=dir_output))
else:
nc_indices.extend(clipping(resource=resource,
prefix=key.replace(variable,name).replace('_day_', '_%s_' % grouping),
calc=calc,
calc_grouping=calc_group,
output_format='nc',
dir_output=dir_output,
polygons=polygons,
mosaic = mosaic,
))
return nc_indices
#def calc_indice_unconventional(resource=[], variable=None, prefix=None,
#indices=None, polygons=None, groupings=None,
#dir_output=None, dimension_map = None):
#"""
#Calculates given indices for suitable files in the appropriate time grouping and polygon.
#:param resource: list of filenames in data reference syntax (DRS) convention (netcdf)
#:param variable: variable name to be selected in the in netcdf file (default=None)
#:param indices: list of indices (default ='TGx')
#:param polygons: list of polygons (default =None)
#:param grouping: indices time aggregation (default='yr')
#:param out_dir: output directory for result file (netcdf)
#:param dimension_map: optional dimension map if different to standard (default=None)
#:return: list of netcdf files with calculated indices. Files are saved into dir_output
#"""
#from os.path import join, dirname, exists
#from os import remove
#import uuid
#from flyingpigeon import ocgis_module
#from flyingpigeon.subset import get_ugid, get_geom
#if type(resource) != list:
#resource = list([resource])
#if type(indices) != list:
#indices = list([indices])
#if type(polygons) != list and polygons != None:
#polygons = list([polygons])
#elif polygons == None:
#polygons = [None]
#else:
#logger.error('Polygons not found')
#if type(groupings) != list:
#groupings = list([groupings])
#if dir_output != None:
#if not exists(dir_output):
#makedirs(dir_output)
#experiments = sort_by_filename(resource)
#outputs = []
#print('environment for calc_indice_unconventional set')
#logger.info('environment for calc_indice_unconventional set')
#for key in experiments:
#if variable == None:
#variable = get_variable(experiments[key][0])
#try:
#ncs = experiments[key]
#for indice in indices:
#logger.info('indice: %s' % indice)
#try:
#for grouping in groupings:
#logger.info('grouping: %s' % grouping)
#try:
#calc_group = calc_grouping(grouping)
#logger.info('calc_group: %s' % calc_group)
#for polygon in polygons:
#try:
#domain = key.split('_')[1].split('-')[0]
#if polygon == None:
#if prefix == None:
#prefix = key.replace(variable, indice).replace('_day_','_%s_' % grouping )
#geom = None
#ugid = None
#else:
#if prefix == None:
#prefix = key.replace(variable, indice).replace('_day_','_%s_' % grouping ).replace(domain,polygon)
#geom = get_geom(polygon=polygon)
#ugid = get_ugid(polygons=polygon, geom=geom)
#if indice == 'TGx':
#calc=[{'func': 'max', 'name': 'TGx'}]
#tmp = ocgis_module.call(resource=ncs,# conform_units_to='celcius',
#variable=variable, dimension_map=dimension_map,
#calc=calc, calc_grouping=calc_group, prefix=prefix,
#dir_output=dir_output, geom=geom, select_ugid=ugid)
#elif indice == 'TGn':
#calc=[{'func': 'min', 'name': 'TGn'}]
#tmp = ocgis_module.call(resource=ncs, #conform_units_to='celcius',
#variable=variable, dimension_map=dimension_map,
#calc=calc, calc_grouping= calc_group, prefix=prefix,
#dir_output=dir_output, geom=geom, select_ugid = ugid)
#elif indice == 'TGx5day':
#calc = [{'func': 'moving_window', 'name': 'TGx5day', 'kwds': {'k': 5, 'operation': 'mean', 'mode': 'same' }}]
#tmp2 = ocgis_module.call(resource=ncs, #conform_units_to='celcius',
#variable=variable, dimension_map=dimension_map,
#calc=calc, prefix=str(uuid.uuid4()),
#geom=geom, select_ugid = ugid)
#calc=[{'func': 'max', 'name': 'TGx5day'}]
#logger.info('moving window calculated : %s' % tmp2)
#tmp = ocgis_module.call(resource=tmp2,
#variable=indice, dimension_map=dimension_map,
#calc=calc, calc_grouping=calc_group, prefix=prefix,
#dir_output=dir_output)
#remove(tmp2)
#elif indice == 'TGn5day':
#calc = [{'func': 'moving_window', 'name': 'TGn5day', 'kwds': {'k': 5, 'operation': 'mean', 'mode': 'same' }}]
#tmp2 = ocgis_module.call(resource=ncs, #conform_units_to='celcius',
#variable=variable, dimension_map=dimension_map,
#calc=calc, prefix=str(uuid.uuid4()),
#geom=geom, select_ugid = ugid)
#calc=[{'func': 'min', 'name': 'TGn5day'}]
#logger.info('moving window calculated : %s' % tmp2)
#tmp = ocgis_module.call(resource=tmp2,
#variable=indice, dimension_map=dimension_map,
#calc=calc, calc_grouping=calc_group, prefix=prefix,
#dir_output=dir_output)
#remove(tmp2)
#else:
#logger.error('Indice %s is not a known inidce' % (indice))
#outputs.append(tmp)
#logger.info('indice file calcualted %s ' % (tmp))
#except Exception as e:
#logger.debug('could not calc indice %s for key %s, polygon %s and calc_grouping %s : %s' % (indice, key, polygon, grouping, e ))
#except Exception as e:
#logger.debug('could not calc indice %s for key %s and calc_grouping %s : %s' % ( indice, key, polygon, e ))
#except Exception as e:
#logger.debug('could not calc indice %s for key %s: %s'% (indice, key, e ))
#except Exception as e:
#logger.debug('could not calc key %s: %s' % (key, e))
#return outputs
def calc_indice_percentile(resources=[],
variable=None,
prefix=None,
indices='TG90p',
refperiod=None,
groupings='yr',
polygons=None,
percentile=90,
mosaic=False,
dir_output=None,
dimension_map=None):
"""
Calculates given indices for suitable files in the appropriate time grouping and polygon.
:param resource: list of filenames in data reference syntax (DRS) convention (netcdf)
:param variable: variable name to be selected in the in netcdf file (default=None)
:param indices: list of indices (default ='TG90p')
:param prefix: filename prefix
:param refperiod: reference period tuple = (start,end)
:param grouping: indices time aggregation (default='yr')
:param dir_output: output directory for result file (netcdf)
:param dimension_map: optional dimension map if different to standard (default=None)
:return: list of netcdf files with calculated indices. Files are saved into out_dir.
"""
from os.path import join, dirname, exists
from os import remove
import uuid
from numpy import ma
from datetime import datetime as dt
from flyingpigeon.ocgis_module import call
from flyingpigeon.subset import clipping
from flyingpigeon.utils import get_values, get_time
if type(resources) != list:
resources = list([resources])
if type(indices) != list:
indices = list([indices])
if type(groupings) != list:
groupings = list([groupings])
if type(refperiod) == list:
refperiod = refperiod[0]
if refperiod != None:
start = dt.strptime(refperiod.split('-')[0], '%Y%m%d')
end = dt.strptime(refperiod.split('-')[1], '%Y%m%d')
time_range = [start, end]
else:
time_range = None
if dir_output != None:
if not exists(dir_output):
makedirs(dir_output)
########################################################################################################################
# Compute a custom percentile basis using ICCLIM. ######################################################################
########################################################################################################################
from ocgis.contrib import library_icclim as lic
nc_indices = []
nc_dic = sort_by_filename(resources)
for grouping in groupings:
calc_group = calc_grouping(grouping)
for key in nc_dic.keys():
resource = nc_dic[key]
if variable == None:
variable = get_variable(resource)
if polygons == None:
nc_reference = call(resource=resource,
prefix=str(uuid.uuid4()),
time_range=time_range,
output_format='nc',
dir_output=dir_output)
else:
nc_reference = clipping(resource=resource,
prefix=str(uuid.uuid4()),
time_range=time_range,
output_format='nc',
polygons=polygons,
dir_output=dir_output,
mosaic=mosaic)
arr = get_values(resource=nc_reference)
dt_arr = get_time(resource=nc_reference)
arr = ma.masked_array(arr)
dt_arr = ma.masked_array(dt_arr)
percentile = percentile
window_width = 5
for indice in indices:
name = indice.replace('_', str(percentile))
var = indice.split('_')[0]
operation = None
if 'T' in var:
if percentile >= 50:
operation = 'Icclim%s90p' % var
func = 'icclim_%s90p' % var # icclim_TG90p
else:
operation = 'Icclim%s10p' % var
func = 'icclim_%s10p' % var
################################
# load the appropriate operation
################################
ops = [op for op in dir(lic) if operation in op]
if len(ops) == 0:
raise Exception("operator does not exist %s", operation)
exec "percentile_dict = lic.%s.get_percentile_dict(arr, dt_arr, percentile, window_width)" % ops[
0]
calc = [{
'func': func,
'name': name,
'kwds': {
'percentile_dict': percentile_dict
}
}]
if polygons == None:
nc_indices.append(
call(resource=resource,
prefix=key.replace(variable, name).replace(
'_day_', '_%s_' % grouping),
calc=calc,
calc_grouping=calc_group,
output_format='nc',
dir_output=dir_output))
else:
nc_indices.extend(
clipping(
resource=resource,
prefix=key.replace(variable, name).replace(
'_day_', '_%s_' % grouping),
calc=calc,
calc_grouping=calc_group,
output_format='nc',
dir_output=dir_output,
polygons=polygons,
mosaic=mosaic,
))
return nc_indices
#def calc_indice_unconventional(resource=[], variable=None, prefix=None,
#indices=None, polygons=None, groupings=None,
#dir_output=None, dimension_map = None):
#"""
#Calculates given indices for suitable files in the appropriate time grouping and polygon.
#:param resource: list of filenames in data reference syntax (DRS) convention (netcdf)
#:param variable: variable name to be selected in the in netcdf file (default=None)
#:param indices: list of indices (default ='TGx')
#:param polygons: list of polygons (default =None)
#:param grouping: indices time aggregation (default='yr')
#:param out_dir: output directory for result file (netcdf)
#:param dimension_map: optional dimension map if different to standard (default=None)
#:return: list of netcdf files with calculated indices. Files are saved into dir_output
#"""
#from os.path import join, dirname, exists
#from os import remove
#import uuid
#from flyingpigeon import ocgis_module
#from flyingpigeon.subset import get_ugid, get_geom
#if type(resource) != list:
#resource = list([resource])
#if type(indices) != list:
#indices = list([indices])
#if type(polygons) != list and polygons != None:
#polygons = list([polygons])
#elif polygons == None:
#polygons = [None]
#else:
#logger.error('Polygons not found')
#if type(groupings) != list:
#groupings = list([groupings])
#if dir_output != None:
#if not exists(dir_output):
#makedirs(dir_output)
#experiments = sort_by_filename(resource)
#outputs = []
#print('environment for calc_indice_unconventional set')
#logger.info('environment for calc_indice_unconventional set')
#for key in experiments:
#if variable == None:
#variable = get_variable(experiments[key][0])
#try:
#ncs = experiments[key]
#for indice in indices:
#logger.info('indice: %s' % indice)
#try:
#for grouping in groupings:
#logger.info('grouping: %s' % grouping)
#try:
#calc_group = calc_grouping(grouping)
#logger.info('calc_group: %s' % calc_group)
#for polygon in polygons:
#try:
#domain = key.split('_')[1].split('-')[0]
#if polygon == None:
#if prefix == None:
#prefix = key.replace(variable, indice).replace('_day_','_%s_' % grouping )
#geom = None
#ugid = None
#else:
#if prefix == None:
#prefix = key.replace(variable, indice).replace('_day_','_%s_' % grouping ).replace(domain,polygon)
#geom = get_geom(polygon=polygon)
#ugid = get_ugid(polygons=polygon, geom=geom)
#if indice == 'TGx':
#calc=[{'func': 'max', 'name': 'TGx'}]
#tmp = ocgis_module.call(resource=ncs,# conform_units_to='celcius',
#variable=variable, dimension_map=dimension_map,
#calc=calc, calc_grouping=calc_group, prefix=prefix,
#dir_output=dir_output, geom=geom, select_ugid=ugid)
#elif indice == 'TGn':
#calc=[{'func': 'min', 'name': 'TGn'}]
#tmp = ocgis_module.call(resource=ncs, #conform_units_to='celcius',
#variable=variable, dimension_map=dimension_map,
#calc=calc, calc_grouping= calc_group, prefix=prefix,
#dir_output=dir_output, geom=geom, select_ugid = ugid)
#elif indice == 'TGx5day':
#calc = [{'func': 'moving_window', 'name': 'TGx5day', 'kwds': {'k': 5, 'operation': 'mean', 'mode': 'same' }}]
#tmp2 = ocgis_module.call(resource=ncs, #conform_units_to='celcius',
#variable=variable, dimension_map=dimension_map,
#calc=calc, prefix=str(uuid.uuid4()),
#geom=geom, select_ugid = ugid)
#calc=[{'func': 'max', 'name': 'TGx5day'}]
#logger.info('moving window calculated : %s' % tmp2)
#tmp = ocgis_module.call(resource=tmp2,
#variable=indice, dimension_map=dimension_map,
#calc=calc, calc_grouping=calc_group, prefix=prefix,
#dir_output=dir_output)
#remove(tmp2)
#elif indice == 'TGn5day':
#calc = [{'func': 'moving_window', 'name': 'TGn5day', 'kwds': {'k': 5, 'operation': 'mean', 'mode': 'same' }}]
#tmp2 = ocgis_module.call(resource=ncs, #conform_units_to='celcius',
#variable=variable, dimension_map=dimension_map,
#calc=calc, prefix=str(uuid.uuid4()),
#geom=geom, select_ugid = ugid)
#calc=[{'func': 'min', 'name': 'TGn5day'}]
#logger.info('moving window calculated : %s' % tmp2)
#tmp = ocgis_module.call(resource=tmp2,
#variable=indice, dimension_map=dimension_map,
#calc=calc, calc_grouping=calc_group, prefix=prefix,
#dir_output=dir_output)
#remove(tmp2)
#else:
#logger.error('Indice %s is not a known inidce' % (indice))
#outputs.append(tmp)
#logger.info('indice file calcualted %s ' % (tmp))
#except Exception as e:
#logger.debug('could not calc indice %s for key %s, polygon %s and calc_grouping %s : %s' % (indice, key, polygon, grouping, e ))
#except Exception as e:
#logger.debug('could not calc indice %s for key %s and calc_grouping %s : %s' % ( indice, key, polygon, e ))
#except Exception as e:
#logger.debug('could not calc indice %s for key %s: %s'% (indice, key, e ))
#except Exception as e:
#logger.debug('could not calc key %s: %s' % (key, e))
#return outputs
def set_dynamic_md(resource):
"""
Dynamic meta data like time frequency, spatial extent, start/end time, etc.
:param resource: netCDF file where basic meta data should be set
"""
from flyingpigeon.utils import get_timerange, get_time
frequency = get_frequency(resource)
time_coverage_start, time_coverage_end = get_timerange(resource)
time_number_steps = len(get_time(resource))
# max_lat, min_lat, max_lon, min_lat = get_extent(resource)
ds = Dataset(resource, mode='a')
try:
driving_experiment = ds.driving_experiment
ds.delncattr('driving_experiment')
except Exception as e:
logger.error(e)
driving_experiment = ''
try:
driving_experiment_name = ds.driving_experiment_name
ds.delncattr('driving_experiment_name')
except Exception as e:
logger.error(e)
driving_experiment_name = ''
try:
driving_model_ensemble_member = ds.driving_model_ensemble_member
ds.delncattr('driving_model_ensemble_member')
except Exception as e:
logger.error(e)
driving_model_ensemble_member = ''
try:
experiment = ds.experiment
ds.delncattr('experiment')
except Exception as e:
logger.error(e)
experiment = ''
try:
tracking_id = ds.tracking_id
ds.delncattr('tracking_id')
except Exception as e:
logger.error(e)
tracking_id = ''
try:
experiment_id = ds.experiment_id
ds.delncattr('experiment_id')
except Exception as e:
logger.error(e)
experiment_id = ''
try:
project_id = ds.project_id
ds.delncattr('project_id')
except Exception as e:
logger.error(e)
project_id = ''
try:
institution_id = ds.institution_id
ds.delncattr('institution_id')
except Exception as e:
logger.error(e)
institution_id = ''
try:
model_version_id = ds.model_version_id
ds.delncattr('model_version_id')
except Exception as e:
logger.error(e)
model_version_id = ''
try:
driving_model_id = ds.driving_model_id
ds.delncattr('driving_model_id')
except Exception as e:
logger.error(e)
driving_model_id = ''
try:
driving_ensemble_member = ds.driving_ensemble_member
ds.delncattr('driving_ensemble_member')
except Exception as e:
logger.error(e)
driving_ensemble_member = ''
try:
driving_model_id = ds.driving_model_id
ds.delncattr('driving_model_id')
except Exception as e:
logger.error(e)
driving_model_id = ''
try:
model_id = ds.model_id
ds.delncattr('model_id')
except Exception as e:
logger.error(e)
driving_model_id =''
try:
contact = ds.contact
ds.delncattr('contact')
except Exception as e:
logger.error(e)
contact = ''
try:
driving_experiment_id = ds.driving_experiment_id
ds.delncattr('driving_experiment_id')
except Exception as e:
logger.error(e)
driving_experiment_id = ''
try:
domain = ds.CORDEX_domain
except Exception as e:
logger.error(e)
domain = ''
ds.close()
min_lat, max_lat, min_lon, max_lon = get_extent(resource)
geospatial_increment = get_geospatial_increment(resource)
try:
md_dynamic = {
'in_var_driving_experiment' :driving_experiment,
'in_var_driving_experiment_name': driving_experiment_name,
'in_var_driving_model_ensemble_member' : driving_model_ensemble_member,
'in_var_experiment': experiment,
'in_var_experiment_id': experiment_id,
'in_var_project_id': project_id,
'in_var_contact': contact,
'in_var_institution_id':institution_id,
'in_var_model_version_id': model_version_id,
'in_var_driving_model_id': driving_model_id,
'in_var_model_id': model_id,
'in_var_driving_ensemble_member':driving_ensemble_member,
'in_var_driving_experiment_id': driving_experiment_id,
'in_var_domain': domain,
'in_var_tracking_id' : tracking_id,
'frequency': frequency,
'time_coverage_start': time_coverage_start,
'time_coverage_end':time_coverage_end,
'time_number_steps':time_number_steps,
#'time_number_gaps': '',
#'cdm_datatype':'' ,
'domain':'%s_subset' % domain ,
'geospatial_increment': geospatial_increment,
'geospatial_lat_min':min_lat ,
'geospatial_lat_max':max_lat ,
'geospatial_lon_min':min_lon ,
'geospatial_lon_max':max_lon ,
}
#:product = "output" ;
#:rcm_version_id = "v1" ;
#:references = "http://www.smhi.se/en/Research/Research-departments/climate-research-rossby-centre" ;
except Exception as e:
logger.error('failed to populate dynamic metadata dictionary')
try:
ds = Dataset(resource, mode='a')
ds.setncatts(md_dynamic)
ds.close()
except Exception as e:
logger.error(e)
return(resource)
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
response.update_status('execution started at : {}'.format(dt.now()), 5)
################################
# reading in the input arguments
################################
try:
LOGGER.info('read in the arguments')
# resources = self.getInputValues(identifier='resources')
season = request.inputs['season'][0].data
LOGGER.info('season %s', season)
period = request.inputs['period'][0].data
LOGGER.info('period %s', period)
anualcycle = request.inputs['anualcycle'][0].data
start = dt.strptime(period.split('-')[0], '%Y%m%d')
end = dt.strptime(period.split('-')[1], '%Y%m%d')
LOGGER.debug('start: %s , end: %s ' % (start, end))
resource = archiveextract(resource=rename_complexinputs(request.inputs['resource']))
# resource = archiveextract(resource=[res.file for res in request.inputs['resource']])
url_Rdat = request.inputs['Rdat'][0].data
url_dat = request.inputs['dat'][0].data
url_ref_file = request.inputs['netCDF'][0].data # can be None
# season = self.getInputValues(identifier='season')[0]
# period = self.getInputValues(identifier='period')[0]
# anualcycle = self.getInputValues(identifier='anualcycle')[0]
LOGGER.info('period %s' % str(period))
LOGGER.info('season %s' % str(season))
LOGGER.info('read in the arguments')
LOGGER.info('url_ref_file: %s' % url_ref_file)
LOGGER.info('url_Rdat: %s' % url_Rdat)
LOGGER.info('url_dat: %s' % url_dat)
except Exception as e:
LOGGER.debug('failed to convert arguments %s ' % e)
############################
# fetching trainging data
############################
try:
dat = abspath(download(url_dat))
Rdat = abspath(download(url_Rdat))
LOGGER.info('training data fetched')
except Exception as e:
LOGGER.error('failed to fetch training data %s' % e)
##########################################################
# get the required bbox and time region from resource data
##########################################################
# from flyingpigeon.weatherregimes import get_level
try:
from flyingpigeon.ocgis_module import call
from flyingpigeon.utils import get_variable
time_range = [start, end]
variable = get_variable(resource)
if len(url_ref_file) > 0:
ref_file = download(url_ref_file)
model_subset = call(
resource=resource, variable=variable,
time_range=time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap',
regrid_destination=ref_file, regrid_options='bil')
LOGGER.info('Dataset subset with regridding done: %s ' % model_subset)
else:
model_subset = call(
resource=resource, variable=variable,
time_range=time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap',
)
LOGGER.info('Dataset time period extracted: %s ' % model_subset)
except:
LOGGER.exception('failed to make a data subset ')
#######################
# computing anomalies
#######################
try:
cycst = anualcycle.split('-')[0]
cycen = anualcycle.split('-')[0]
reference = [dt.strptime(cycst, '%Y%m%d'), dt.strptime(cycen, '%Y%m%d')]
model_anomal = wr.get_anomalies(model_subset, reference=reference)
#####################
# extracting season
#####################
model_season = wr.get_season(model_anomal, season=season)
except:
LOGGER.exception('failed to compute anualcycle or seasons')
#######################
# call the R scripts
#######################
import shlex
import subprocess
from flyingpigeon import config
from os.path import curdir, exists, join
try:
rworkspace = curdir
Rsrc = config.Rsrc_dir()
Rfile = 'weatherregimes_projection.R'
yr1 = start.year
yr2 = end.year
time = get_time(model_season, format='%Y%m%d')
# ip, output_graphics = mkstemp(dir=curdir ,suffix='.pdf')
ip, file_pca = mkstemp(dir=curdir, suffix='.txt')
ip, file_class = mkstemp(dir=curdir, suffix='.Rdat')
ip, output_frec = mkstemp(dir=curdir, suffix='.txt')
args = ['Rscript', join(Rsrc, Rfile), '%s/' % curdir,
'%s/' % Rsrc,
'%s' % model_season,
'%s' % variable,
'%s' % str(time).strip("[]").replace("'", "").replace(" ", ""),
# '%s' % output_graphics,
'%s' % dat,
'%s' % Rdat,
'%s' % file_pca,
'%s' % file_class,
'%s' % output_frec,
'%s' % season,
'%s' % start.year,
'%s' % end.year,
'%s' % 'MODEL']
LOGGER.info('Rcall builded')
except Exception as e:
msg = 'failed to build the R command %s' % e
LOGGER.error(msg)
raise Exception(msg)
try:
output, error = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
# , shell=True
LOGGER.info('R outlog info:\n %s ' % output)
LOGGER.debug('R outlog errors:\n %s ' % error)
if len(output) > 0:
response.update_status('**** weatherregime in R suceeded', 90)
else:
LOGGER.error('NO! output returned from R call')
except Exception as e:
msg = 'weatherregime in R %s ' % e
LOGGER.error(msg)
raise Exception(msg)
#################
# set the outputs
#################
response.update_status('Set the process outputs ', 95)
response.outputs['output_pca'].file = file_pca
response.outputs['output_classification'].file = file_class
response.outputs['output_netcdf'].file = model_season
response.outputs['output_frequency'].file = output_frec
response.update_status('done', 100)
return response
