def test_get_variable(): variable = utils.get_variable(local_path(TESTDATA['cmip5_tasmax_2007_nc'])) assert 'tasmax' == variable variable = utils.get_variable(local_path( TESTDATA['cordex_tasmax_2007_nc'])) assert 'tasmax' == variable
def write_to_file(nc_indice, data): from netCDF4 import Dataset from shutil import copy from os.path import split, join from flyingpigeon.utils import get_variable from flyingpigeon.metadata import get_frequency #path, nc_indice = split(indice_file) var = get_variable(nc_indice) fq = get_frequency(nc_indice) agg = nc_indice.split('_')[-2] nc = nc_indice.replace(var, 'tree').replace(agg, fq) copy(nc_indice, nc) ds = Dataset(nc, mode='a') vals = ds.variables[var] ds.renameVariable(var, 'tree') vals[:, :, :] = data[:, :, :] vals.long_name = 'Favourabilliy for tree species' vals.standard_name = 'tree' vals.units = '0-1' ds.close() return nc
def get_reference(ncs_indices, period='all'): """ calculates the netCDF files containing the mean climatology for statistical GAM training :param ncs_indices: list of climate indices defining the growing conditions of tree species :param refperiod: time period for statistic training :return present: present conditions """ from datetime import datetime as dt from flyingpigeon.ocgis_module import call from flyingpigeon.utils import get_variable from os.path import basename if not period == 'all': s, e = period.split('-') start = dt.strptime(s+'-01-01', '%Y-%m-%d') end = dt.strptime(e+'-12-31', '%Y-%m-%d') time_range=[start, end] else: time_range=None ref_indices = [] for nc_indice in ncs_indices: variable = get_variable(nc_indice) f = basename(nc_indice).strip('.nc') prefix = '%s_ref-%s' % ('_'.join(f.split('_')[0:-1]), period) ref_indices.append(call(resource=nc_indice, variable=variable,prefix=prefix, calc=[{'func':'mean','name': variable}],calc_grouping=['all'],time_range=time_range)) return ref_indices
def write_to_file(nc_indice, data): from netCDF4 import Dataset from shutil import copy from os.path import split, join from flyingpigeon.utils import get_variable from flyingpigeon.metadata import get_frequency #path, nc_indice = split(indice_file) var = get_variable(nc_indice) fq = get_frequency(nc_indice) agg = nc_indice.split('_')[-2] nc = nc_indice.replace(var,'tree').replace(agg,fq) copy(nc_indice, nc) ds = Dataset(nc, mode= 'a') vals = ds.variables[var] ds.renameVariable(var,'tree') vals[:,:,:] = data[:,:,:] vals.long_name = 'Favourabilliy for tree species' vals.standard_name = 'tree' vals.units = '0-1' ds.close() return nc
def get_level(resource, level): from flyingpigeon.ocgis_module import call from netCDF4 import Dataset from flyingpigeon.utils import get_variable from numpy import squeeze try: level_data = call(resource, level_range=[int(level), int(level)]) if type(resource) == list: resource.sort() variable = get_variable(level_data) logger.info('found %s in file' % variable) ds = Dataset(level_data, mode='a') var = ds.variables.pop(variable) dims = var.dimensions new_var = ds.createVariable('z%s' % level, var.dtype, dimensions=(dims[0], dims[2], dims[3])) # i = where(var[:]==level) new_var[:, :, :] = squeeze(var[:, 0, :, :]) ds.close() logger.info('level %s extracted' % level) data = call(level_data, variable='z%s' % level) except Exception as e: logger.error('failed to extract level %s ' % e) return data
def get_pca(resource): """ calculation of principal components :param resource: netCDF file containing pressure values for a defined region and selected timesteps :return pca: sklean objct """ from netCDF4 import Dataset, num2date from flyingpigeon.utils import get_variable var = get_variable(resource) print 'variable name: %s' % var ds = Dataset(resource) vals = ds.variables[var] lat = ds.variables['lat'] lon = ds.variables['lon'] #time = ds.variables['time'] # make array of seasons: # convert netCDF timesteps to datetime #timestamps = num2date(time[:], time.units, time.calendar) #season = [get_season(s) for s in timestamps] from sklearn.decomposition import PCA import numpy as np # reshape data = np.array(vals) adata = data.reshape(vals[:].shape[0], (vals[:].shape[1] * vals[:].shape[2]) ) pca = PCA(n_components=50).fit_transform(adata) return vals, pca #, season
def get_level(resource, level): from flyingpigeon.ocgis_module import call from netCDF4 import Dataset from flyingpigeon.utils import get_variable from numpy import squeeze from os import path try: if type(resource) == list: resource = sorted(resource, key=lambda i: path.splitext(path.basename(i))[0]) # resource.sort() level_data = call(resource, level_range=[int(level), int(level)]) variable = get_variable(level_data) LOGGER.info('found %s in file' % variable) ds = Dataset(level_data, mode='a') var = ds.variables.pop(variable) dims = var.dimensions new_var = ds.createVariable('z%s' % level, var.dtype, dimensions=(dims[0], dims[2], dims[3])) # i = where(var[:]==level) new_var[:, :, :] = squeeze(var[:, 0, :, :]) # TODO: Here may be an error! in case of exception, dataset will not close! # Exception arise for example for 20CRV2 data... try: new_var.setncatts({k: var.getncattr(k) for k in var.ncattrs()}) except: LOGGER.info('Could not set attributes for z%s' % level) ds.close() LOGGER.info('level %s extracted' % level) data = call(level_data, variable='z%s' % level) except: LOGGER.exception('failed to extract level') return data
def get_reference(ncs_indices, period='all'): """ calculates the netCDF files containing the mean climatology for statistical GAM training :param ncs_indices: list of climate indices defining the growing conditions of tree species :param refperiod: time period for statistic training :return present: present conditions """ from datetime import datetime as dt from flyingpigeon.ocgis_module import call from flyingpigeon.utils import get_variable from os.path import basename if not period == 'all': s, e = period.split('-') start = dt.strptime(s + '-01-01', '%Y-%m-%d') end = dt.strptime(e + '-12-31', '%Y-%m-%d') time_range = [start, end] else: time_range = None ref_indices = [] for nc_indice in ncs_indices: variable = get_variable(nc_indice) f = basename(nc_indice).strip('.nc') prefix = '%s_ref-%s' % ('_'.join(f.split('_')[0:-1]), period) ref_indices.append(call(resource=nc_indice, variable=variable, prefix=prefix, calc=[{'func': 'mean', 'name': variable}], calc_grouping=['all'], time_range=time_range)) return ref_indices
def get_level(resource, level): from flyingpigeon.ocgis_module import call from netCDF4 import Dataset from flyingpigeon.utils import get_variable from numpy import squeeze try: level_data = call(resource, level_range=[int(level),int(level)]) if type(resource) == list: resource.sort() variable = get_variable(level_data) logger.info('found %s in file' % variable) ds = Dataset(level_data, mode='a') var = ds.variables.pop(variable) dims = var.dimensions new_var = ds.createVariable('z%s'% level, var.dtype, dimensions=(dims[0],dims[2],dims[3])) # i = where(var[:]==level) new_var[:,:,:] = squeeze(var[:,0,:,:]) ds.close() logger.info('level %s extracted' % level) data = call(level_data , variable = 'z%s'%level) except Exception as e: logger.error('failed to extract level %s ' % e) return data
def get_gam(ncs_reference, PAmask): from netCDF4 import Dataset from os.path import basename from numpy import squeeze, ravel, isnan, nan, array, reshape from flyingpigeon.utils import get_variable from rpy2.robjects.packages import importr import rpy2.robjects as ro import rpy2.robjects.numpy2ri rpy2.robjects.numpy2ri.activate() mgcv = importr("mgcv") base = importr("base") stats = importr("stats") data = {'PA': ro.FloatVector(ravel(PAmask))} domain = PAmask.shape form = 'PA ~ ' ncs_reference.sort() for i , nc in enumerate(ncs_reference): var = get_variable(nc) agg = basename(nc).split('_')[-2] ds = Dataset(nc) vals = squeeze(ds.variables[var]) vals[isnan(PAmask)] = nan indice = '%s_%s' % (var, agg) data[str(indice)] = ro.FloatVector(ravel(vals)) if i == 0: form = form + 's(%s, k=3)' % indice else: form = form + ' + s(%s, k=3)' % indice dataf = ro.DataFrame(data) eq = ro.Formula(str(form)) gam_model = mgcv.gam(base.eval(eq), data=dataf, family=stats.binomial(), scale=-1, na_action=stats.na_exclude) grdevices = importr('grDevices') output_info = "info.pdf" grdevices.pdf(file=output_info) # plotting code here for i in range(1,len(ncs_reference)+1): #ylim = ro.IntVector([-6,6]) mgcv.plot_gam(gam_model, shade='T', col='black',select=i,ylab='Predicted Probability',rug=False , cex_lab = 1.4, cex_axis = 1.4, ) #ylim=ylim, trans=base.eval(trans), grdevices.dev_off() predict_gam = mgcv.predict_gam(gam_model, type="response", progress="text", na_action=stats.na_exclude) #, prediction = array(predict_gam).reshape(domain) return gam_model, prediction, output_info
def execute(self): from flyingpigeon.ocgis_module import call from flyingpigeon.utils import get_time, get_variable, sort_by_filename from datetime import datetime as dt from netCDF4 import Dataset from numpy import savetxt, column_stack, squeeze ncs = self.getInputValues(identifier='netcdf_file') logging.info("ncs: %s " % ncs) coords = self.getInputValues(identifier='coords') logging.info("coords %s", coords) nc_exp = sort_by_filename(ncs) # dictionary {experiment:[files]} filenames = [] (fp_tar, tarout_file) = tempfile.mkstemp(dir=".", suffix='.tar') tar = tarfile.open(tarout_file, "w") for key in nc_exp.keys(): logging.info('start calculation for %s ' % key ) ncs = nc_exp[key] nc = ncs[0] times = get_time(nc) var = get_variable(nc) concat_vals = [dt.strftime(t, format='%Y-%d-%m_%H:%M:%S') for t in times] header = 'date_time' filename = '%s.csv' % key filenames.append(filename) for ugid, p in enumerate(coords, start=1): self.status.set('processing point : {0}'.format(p), 20) p = p.split(',') self.status.set('splited x and y coord : {0}'.format(p), 20) point = Point(float(p[0]), float(p[1])) #get the timeseries at gridpoint timeseries = call(resource=ncs, geom=point, select_nearest=True) ds = Dataset(timeseries) vals = squeeze(ds.variables[var]) header = header + ',%s_%s' % (p[0], p[1]) concat_vals = column_stack([concat_vals, vals]) savetxt(filename, concat_vals, fmt='%s', delimiter=',', header=header) tar.add( filename ) tar.close() self.tarout.setValue( tarout_file )
def spaghetti(resouces, variable=None, title=None, file_extension='png'): """ creates a png file containing the appropriate spaghetti plot as a field mean of the values. :param resouces: list of files containing the same variable :param variable: variable to be visualised. If None (default), variable will be detected :param title: string to be used as title :retruns str: path to png file """ from flyingpigeon.calculation import fieldmean try: fig = plt.figure(figsize=(20, 10), dpi=600, facecolor='w', edgecolor='k') LOGGER.debug('Start visualisation spaghetti plot') # === prepare invironment if type(resouces) != list: resouces = [resouces] if variable is None: variable = utils.get_variable(resouces[0]) if title is None: title = "Field mean of %s " % variable LOGGER.info('plot values preparation done') except: msg = "plot values preparation failed" LOGGER.exception(msg) raise Exception(msg) try: for c, nc in enumerate(resouces): # get timestapms try: dt = utils.get_time(nc) # [datetime.strptime(elem, '%Y-%m-%d') for elem in strDate[0]] ts = fieldmean(nc) plt.plot(dt, ts) # fig.line( dt,ts ) except: msg = "spaghetti plot failed for %s " % nc LOGGER.exception(msg) plt.title(title, fontsize=20) plt.grid() output_png = fig2plot(fig=fig, file_extension=file_extension) plt.close() LOGGER.info('timeseries spaghetti plot done for %s with %s lines.' % (variable, c)) except: msg = 'matplotlib spaghetti plot failed' LOGGER.exception(msg) return output_png
def get_prediction(gam_model, ncs_indices): #mask=None """ predict the probabillity based on the gam_model and the given climate index datasets :param gam_model: fitted gam (output from sdm.get_gam) :pram nsc_indices: list of netCDF files containing climate indices of one dataset :param mask: 2D array of True/False to exclude areas (e.g ocean) for prediction """ from netCDF4 import Dataset from os.path import basename from numpy import squeeze, ravel, array, reshape #, zeros, broadcast_arrays, nan from flyingpigeon.utils import get_variable from rpy2.robjects.packages import importr import rpy2.robjects as ro import rpy2.robjects.numpy2ri rpy2.robjects.numpy2ri.activate() mgcv = importr("mgcv") stats = importr("stats") ncs_indices.sort() data = {} for i, nc in enumerate(ncs_indices): var = get_variable(nc) agg = basename(nc).split('_')[-2] ds = Dataset(nc) vals = squeeze(ds.variables[var]) if i == 0: dims = vals.shape #if mask != None: #mask = broadcast_arrays(vals, mask)[1] #vals[mask==False] = nan indice = '%s_%s' % (var, agg) data[str(indice)] = ro.FloatVector(ravel(vals)) dataf = ro.DataFrame(data) predict_gam = mgcv.predict_gam( gam_model, newdata=dataf, type="response", progress="text", newdata_guaranteed=True) #, na_action=`na.pass` prediction = array(predict_gam).reshape(dims) return prediction
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' ncfiles = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) if 'variable' in request.inputs: var = request.inputs['variable'][0].data else: var = get_variable(ncfiles[0]) # var = ncfiles[0].split("_")[0] response.update_status('plotting variable {}'.format(var), 10) try: plotout_spagetti_file = vs.spaghetti( ncfiles, variable=var, title='Field mean of {}'.format(var), ) LOGGER.info("spagetti plot done") response.update_status( 'Spagetti plot for %s %s files done' % (len(ncfiles), var), 50) response.outputs['plotout_spagetti'].file = plotout_spagetti_file except Exception as e: raise Exception("spagetti plot failed : {}".format(e)) try: plotout_uncertainty_file = vs.uncertainty( ncfiles, variable=var, title='Ensemble uncertainty for {}'.format(var), ) response.update_status( 'Uncertainty plot for {} {} files done'.format( len(ncfiles), var), 90) response.outputs[ 'plotout_uncertainty'].file = plotout_uncertainty_file LOGGER.info("uncertainty plot done") except Exception as err: raise Exception("uncertainty plot failed {}".format(err.message)) response.update_status('visualisation done', 100) return response
def get_prediction(gam_model, ncs_indices ): #mask=None """ predict the probabillity based on the gam_model and the given climate index datasets :param gam_model: fitted gam (output from sdm.get_gam) :pram nsc_indices: list of netCDF files containing climate indices of one dataset :param mask: 2D array of True/False to exclude areas (e.g ocean) for prediction """ from netCDF4 import Dataset from os.path import basename from numpy import squeeze, ravel, array, reshape#, zeros, broadcast_arrays, nan from flyingpigeon.utils import get_variable from rpy2.robjects.packages import importr import rpy2.robjects as ro import rpy2.robjects.numpy2ri rpy2.robjects.numpy2ri.activate() mgcv = importr("mgcv") stats = importr("stats") ncs_indices.sort() data = {} for i , nc in enumerate(ncs_indices): var = get_variable(nc) agg = basename(nc).split('_')[-2] ds = Dataset(nc) vals = squeeze(ds.variables[var]) if i == 0: dims = vals.shape #if mask != None: #mask = broadcast_arrays(vals, mask)[1] #vals[mask==False] = nan indice = '%s_%s' % (var, agg) data[str(indice)] = ro.FloatVector(ravel(vals)) dataf = ro.DataFrame(data) predict_gam = mgcv.predict_gam(gam_model, newdata=dataf, type="response", progress="text", newdata_guaranteed = True) #, na_action=`na.pass` prediction = array(predict_gam).reshape(dims) return prediction
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' ncfiles = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) var = request.inputs['variableIn'] if var is None: from flyingpigeon.utils import get_variable var = get_variable(ncfiles[0]) response.update_status('plotting variable %s' % var, 10) try: plotout_spagetti_file = vs.spaghetti(ncfiles, variable=var, title='Fieldmean of %s ' % (var), dir_out=None) LOGGER.info("spagetti plot done") response.update_status( 'Spagetti plot for %s %s files done' % (len(ncfiles), var), 50) except: LOGGER.exception("spagetti plot failed") try: plotout_uncertainty_file = vs.uncertainty( ncfiles, variable=var, title='Ensemble uncertainty for %s ' % (var), dir_out=None) response.update_status( 'Uncertainty plot for %s %s files done' % (len(ncfiles), var), 90) LOGGER.info("uncertainty plot done") except: LOGGER.exception("uncertainty plot failed") response.outputs['plotout_spagetti'].file = plotout_spagetti_file response.outputs['plotout_uncertainty'].file = plotout_uncertainty_file response.update_status('visualisation done', 100) return response
def execute(self): init_process_logger('log.txt') self.output_log.setValue('log.txt') from flyingpigeon.utils import archiveextract ncfiles = archiveextract(self.getInputValues(identifier='resource')) var = self.variableIn.getValue() if var is None: from flyingpigeon.utils import get_variable var = get_variable(ncfiles[0]) self.status.set('plotting variable %s' % var, 10) try: plotout_spagetti_file = vs.spaghetti( ncfiles, variable=var, title='Fieldmean of %s ' % (var), dir_out=None ) logger.info("spagetti plot done") self.status.set('Spagetti plot for %s %s files done' % (len(ncfiles), var), 50) except: logger.exception("spagetti plot failed") try: plotout_uncertainty_file = vs.uncertainty( ncfiles, variable=var, title='Ensemble uncertainty for %s ' % (var), dir_out=None ) self.status.set('Uncertainty plot for %s %s files done' % (len(ncfiles), var), 90) logger.info("uncertainty plot done") except: logger.exception("uncertainty plot failed") self.plotout_spagetti.setValue(plotout_spagetti_file) self.plotout_uncertainty.setValue(plotout_uncertainty_file) self.status.set('visualisation done', 100)
def write_to_file(nc_indice, data): """ repaces the values in an indice file with given data :param nc_indice: base netCDF file (indice file) :param data: data to be filled into the netCDF file :returns str: path to netCDF file """ try: from netCDF4 import Dataset from shutil import copy from os.path import split, join from flyingpigeon.utils import get_variable from flyingpigeon.metadata import get_frequency from numpy import nan # path, nc_indice = split(indice_file) var = get_variable(nc_indice) fq = get_frequency(nc_indice) agg = nc_indice.split('_')[-2] nc = nc_indice.replace(var, 'tree').replace(agg, fq) copy(nc_indice, nc) ds = Dataset(nc, mode='a') vals = ds.variables[var] ds.renameVariable(var, 'tree') vals[:, :, :] = data[:, :, :] vals.long_name = 'Favourabilliy for tree species' vals.standard_name = 'tree' vals.units = '0-1' vals.missing_value = nan ds.close() except: msg = 'failed to fill data to netCDF file' logger.exception(msg) return nc
def calc_indice_percentile(resources=[], variable=None, prefix=None, indices='TG90p', refperiod=None, groupings='yr', polygons=None, percentile=90, mosaik = False, dir_output=None, dimension_map = None): """ Calculates given indices for suitable files in the appopriate time grouping and polygon. :param resource: list of filenames in drs convention (netcdf) :param variable: variable name to be selected in the in netcdf file (default=None) :param indices: list of indices (default ='TG90p') :param prefix: filename prefix :param refperiod: reference refperiod touple = (start,end) :param grouping: indices time aggregation (default='yr') :param dir_output: output directory for result file (netcdf) :param dimension_map: optional dimension map if different to standard (default=None) :return: list of netcdf files with calculated indices. Files are saved into out_dir """ from os.path import join, dirname, exists from os import remove import uuid from numpy import ma from datetime import datetime as dt from flyingpigeon.ocgis_module import call from flyingpigeon.subset import clipping from flyingpigeon.utils import get_values, get_time if type(resources) != list: resources = list([resources]) if type(indices) != list: indices = list([indices]) if type(groupings) != list: groupings = list([groupings]) if type(refperiod) == list: refperiod = refperiod[0] if refperiod != None: start = dt.strptime(refperiod.split('-')[0] , '%Y%m%d') end = dt.strptime(refperiod.split('-')[1] , '%Y%m%d') time_range = [start, end] else: time_range = None if dir_output != None: if not exists(dir_output): makedirs(dir_output) ######################################################################################################################## # Compute a custom percentile basis using ICCLIM. ###################################################################### ######################################################################################################################## from ocgis.contrib import library_icclim as lic nc_indices = [] nc_dic = sort_by_filename(resources) for grouping in groupings: calc_group = calc_grouping(grouping) for key in nc_dic.keys(): resource = nc_dic[key] if variable == None: variable = get_variable(resource) if polygons == None: nc_reference = call(resource=resource, prefix=str(uuid.uuid4()), time_range=time_range, output_format='nc', dir_output=dir_output) else: nc_reference = clipping(resource=resource, prefix=str(uuid.uuid4()), time_range=time_range, output_format='nc', polygons=polygons, dir_output=dir_output, mosaik = mosaik) arr = get_values(nc_files=nc_reference) dt_arr = get_time(nc_files=nc_reference) arr = ma.masked_array(arr) dt_arr = ma.masked_array(dt_arr) percentile = percentile window_width = 5 for indice in indices: name = indice.replace('_', str(percentile)) var = indice.split('_')[0] operation = None if 'T' in var: if percentile >= 50: operation = 'Icclim%s90p' % var func = 'icclim_%s90p' % var # icclim_TG90p else: operation = 'Icclim%s10p' % var func = 'icclim_%s10p' % var ################################ # load the appropriate operation ################################ ops = [op for op in dir(lic) if operation in op] if len(ops) == 0: raise Exception("operator does not exist %s", operation) exec "percentile_dict = lic.%s.get_percentile_dict(arr, dt_arr, percentile, window_width)" % ops[0] calc = [{'func': func, 'name': name, 'kwds': {'percentile_dict': percentile_dict}}] if polygons == None: nc_indices.append(call(resource=resource, prefix=key.replace(variable,name).replace('_day_', '_%s_' % grouping), calc=calc, calc_grouping=calc_group, output_format='nc', dir_output=dir_output)) else: nc_indices.extend(clipping(resource=resource, prefix=key.replace(variable,name).replace('_day_', '_%s_' % grouping), calc=calc, calc_grouping=calc_group, output_format='nc', dir_output=dir_output, polygons=polygons, mosaik = mosaik, )) return nc_indices
def method_A(resource=[], start=None, end=None, timeslice=20, variable=None, title=None, cmap='seismic'): """returns the result :param resource: list of paths to netCDF files :param start: beginning of reference period (if None (default), the first year of the consistent ensemble will be detected) :param end: end of comparison period (if None (default), the last year of the consistent ensemble will be detected) :param timeslice: period length for mean calculation of reference and comparison period :param variable: OBSOLETE :param title: str to be used as title for the signal mal :param cmap: define the color scheme for signal map plotting :return: signal.nc, low_agreement_mask.nc, high_agreement_mask.nc, text.txt, # graphic.png, """ from os.path import split from tempfile import mkstemp from cdo import Cdo cdo = Cdo() cdo.forceOutput = True # preparing the resource try: file_dic = sort_by_filename(resource, historical_concatination=True) LOGGER.info('file names sorted experimets: %s' % len(file_dic.keys())) except: msg = 'failed to sort the input files' LOGGER.exception(msg) # check that all datasets contains the same variable try: var_name = set() for key in file_dic.keys(): var_name = var_name.union([get_variable(file_dic[key])]) LOGGER.debug(var_name) except: LOGGER.exception('failed to get the variable in common') if len(var_name) == 1: variable = [str(n) for n in var_name][0] LOGGER.info('varible %s detected in all members of the ensemble' % variable) else: raise Exception( 'none or more than one variables are found in the ensemble members' ) # TODO: drop missfitting grids # timemerge for seperate datasets try: mergefiles = [] for key in file_dic.keys(): # if variable is None: # variable = get_variable(file_dic[key]) # LOGGER.info('variable detected %s ' % variable) try: if type(file_dic[key]) == list and len(file_dic[key]) > 1: _, nc_merge = mkstemp(dir='.', suffix='.nc') mergefiles.append( cdo.mergetime(input=file_dic[key], output=nc_merge)) else: mergefiles.extend(file_dic[key]) except: LOGGER.exception('failed to merge files for %s ' % key) LOGGER.info('datasets merged %s ' % mergefiles) except: msg = 'seltime and mergetime failed' LOGGER.exception(msg) # dataset documentation try: text_src = open('infiles.txt', 'a') for key in file_dic.keys(): text_src.write(key + '\n') text_src.close() except: msg = 'failed to write source textfile' LOGGER.exception(msg) _, text_src = mkstemp(dir='.', suffix='.txt') # configure reference and compare period # TODO: filter files by time try: if start is None: st_set = set() en_set = set() for f in mergefiles: times = get_time(f) st_set.update([times[0].year]) if end is None: en_set.update([times[-1].year]) start = max(st_set) if end is None: end = min(en_set) LOGGER.info('Start and End: %s - %s ' % (start, end)) if start >= end: LOGGER.error( 'ensemble is inconsistent!!! start year is later than end year' ) except: msg = 'failed to detect start and end times of the ensemble' LOGGER.exception(msg) # set the periodes: try: LOGGER.debug(type(start)) # start = int(start) # end = int(end) if timeslice is None: timeslice = int((end - start) / 3) if timeslice == 0: timeslice = 1 else: timeslice = int(timeslice) start1 = start start2 = start1 + timeslice - 1 end1 = end - timeslice + 1 end2 = end LOGGER.info('timeslice and periodes set') except: msg = 'failed to set the periodes' LOGGER.exception(msg) try: files = [] for i, mf in enumerate(mergefiles): files.append( cdo.selyear('{0}/{1}'.format(start1, end2), input=[mf.replace(' ', '\ ')], output='file_{0}_.nc'.format(i))) # python version LOGGER.info('timeseries selected from defined start to end year') except: msg = 'seltime and mergetime failed' LOGGER.exception(msg) try: # ensemble mean nc_ensmean = cdo.ensmean(input=files, output='nc_ensmean.nc') LOGGER.info('ensemble mean calculation done') except: msg = 'ensemble mean failed' LOGGER.exception(msg) try: # ensemble std nc_ensstd = cdo.ensstd(input=files, output='nc_ensstd.nc') LOGGER.info('ensemble std and calculation done') except: msg = 'ensemble std or failed' LOGGER.exception(msg) # get the get the signal as difference from the beginning (first years) and end period (last years), : try: selyearstart = cdo.selyear('%s/%s' % (start1, start2), input=nc_ensmean, output='selyearstart.nc') selyearend = cdo.selyear('%s/%s' % (end1, end2), input=nc_ensmean, output='selyearend.nc') meanyearst = cdo.timmean(input=selyearstart, output='meanyearst.nc') meanyearend = cdo.timmean(input=selyearend, output='meanyearend.nc') signal = cdo.sub(input=[meanyearend, meanyearst], output='signal.nc') LOGGER.info('Signal calculation done') except: msg = 'calculation of signal failed' LOGGER.exception(msg) _, signal = mkstemp(dir='.', suffix='.nc') # get the intermodel standard deviation (mean over whole period) try: # std_selyear = cdo.selyear('%s/%s' % (end1,end2), input=nc_ensstd, output='std_selyear.nc') # std = cdo.timmean(input = std_selyear, output = 'std.nc') std = cdo.timmean(input=nc_ensstd, output='std.nc') std2 = cdo.mulc('2', input=std, output='std2.nc') LOGGER.info('calculation of internal model std for time period done') except: msg = 'calculation of internal model std failed' LOGGER.exception(msg) try: absolut = cdo.abs(input=signal, output='absolut_signal.nc') high_agreement_mask = cdo.gt( input=[absolut, std2], output='large_change_with_high_model_agreement.nc') low_agreement_mask = cdo.lt( input=[absolut, std], output='small_signal_or_low_agreement_of_models.nc') LOGGER.info('high and low mask done') except: msg = 'calculation of robustness mask failed' LOGGER.exception(msg) _, high_agreement_mask = mkstemp(dir='.', suffix='.nc') _, low_agreement_mask = mkstemp(dir='.', suffix='.nc') return signal, low_agreement_mask, high_agreement_mask, text_src
def execute(self): import time # performance test process_start_time = time.time() # measure process execution time ... from os import path from tempfile import mkstemp from flyingpigeon import analogs from datetime import datetime as dt from flyingpigeon.ocgis_module import call from flyingpigeon.datafetch import reanalyses from flyingpigeon.utils import get_variable, rename_variable self.status.set('execution started at : %s ' % dt.now(),5) start_time = time.time() # measure init ... resource = self.getInputValues(identifier='resource') bbox_obj = self.BBox.getValue() refSt = self.getInputValues(identifier='refSt') refEn = self.getInputValues(identifier='refEn') dateSt = self.getInputValues(identifier='dateSt') dateEn = self.getInputValues(identifier='dateEn') normalize = self.getInputValues(identifier='normalize')[0] distance = self.getInputValues(identifier='dist')[0] outformat = self.getInputValues(identifier='outformat')[0] timewin = int(self.getInputValues(identifier='timewin')[0]) experiment = self.getInputValues(identifier='experiment')[0] dataset , var = experiment.split('_') refSt = dt.strptime(refSt[0],'%Y-%m-%d') refEn = dt.strptime(refEn[0],'%Y-%m-%d') dateSt = dt.strptime(dateSt[0],'%Y-%m-%d') dateEn = dt.strptime(dateEn[0],'%Y-%m-%d') if normalize == 'None': seacyc = False else: seacyc = True if outformat == 'ascii': outformat = '.txt' elif outformat == 'netCDF': outformat = '.nc' else: logger.error('output format not valid') if bbox_obj is not None: logger.info("bbox_obj={0}".format(bbox_obj.coords)) bbox = [bbox_obj.coords[0][0], bbox_obj.coords[0][1],bbox_obj.coords[1][0],bbox_obj.coords[1][1]] logger.info("bbox={0}".format(bbox)) else: bbox=None #start = min( refSt, dateSt ) #end = max( refEn, dateEn ) # region = self.getInputValues(identifier='region')[0] # bbox = [float(b) for b in region.split(',')] try: if dataset == 'NCEP': if 'z' in var: variable='hgt' level=var.strip('z') #conform_units_to=None else: variable='slp' level=None #conform_units_to='hPa' elif '20CRV2' in var: if 'z' in level: variable='hgt' level=var.strip('z') #conform_units_to=None else: variable='prmsl' level=None #conform_units_to='hPa' else: logger.error('Reanalyses dataset not known') logger.info('environment set') except Exception as e: msg = 'failed to set environment %s ' % e logger.error(msg) raise Exception(msg) logger.debug("init took %s seconds.", time.time() - start_time) self.status.set('Read in the arguments', 5) ################# # get input data ################# start_time = time.time() # measure get_input_data ... self.status.set('fetching input data', 7) try: input = reanalyses(start = dateSt.year, end = dateEn.year, variable=var, dataset=dataset) nc_subset = call(resource=input, variable=var, geom=bbox) except Exception as e : msg = 'failed to fetch or subset input files %s' % e logger.error(msg) raise Exception(msg) logger.debug("get_input_subset_dataset took %s seconds.", time.time() - start_time) self.status.set('**** Input data fetched', 10) ######################## # input data preperation ######################## self.status.set('Start preparing input data', 12) start_time = time.time() # mesure data preperation ... try: self.status.set('Preparing simulation data', 15) simulation = call(resource=nc_subset, time_range=[dateSt , dateEn]) except: msg = 'failed to prepare simulation period' logger.debug(msg) try: self.status.set('Preparing target data', 17) var_target = get_variable(resource) #var_simulation = get_variable(simulation) archive = call(resource=resource, variable=var_target, time_range=[refSt , refEn], geom=bbox, t_calendar='standard',# conform_units_to=conform_units_to, spatial_wrapping='wrap', regrid_destination=simulation, regrid_options='bil') except Exception as e: msg = 'failed subset archive dataset %s ' % e logger.debug(msg) raise Exception(msg) try: if var != var_target: rename_variable(archive, oldname=var_target, newname=var) logger.info('varname %s in netCDF renamed to %s' %(var_target, var)) except Exception as e: msg = 'failed to rename variable in target files %s ' % e logger.debug(msg) raise Exception(msg) try: if seacyc == True: seasoncyc_base , seasoncyc_sim = analogs.seacyc(archive, simulation, method=normalize) else: seasoncyc_base , seasoncyc_sim = None except Exception as e: msg = 'failed to prepare seasonal cycle reference files %s ' % e logger.debug(msg) raise Exception(msg) ip, output = mkstemp(dir='.',suffix='.txt') output_file = path.abspath(output) files=[path.abspath(archive), path.abspath(simulation), output_file] logger.debug("data preperation took %s seconds.", time.time() - start_time) ############################ # generating the config file ############################ self.status.set('writing config file', 15) start_time = time.time() # measure write config ... try: config_file = analogs.get_configfile( files=files, seasoncyc_base = seasoncyc_base, seasoncyc_sim = seasoncyc_sim, timewin=timewin, varname=var, seacyc=seacyc, cycsmooth=91, nanalog=nanalog, seasonwin=seasonwin, distfun=distance, outformat=outformat, calccor=True, silent=False, period=[dt.strftime(refSt,'%Y-%m-%d'),dt.strftime(refEn,'%Y-%m-%d')], bbox="%s,%s,%s,%s" % (bbox[0],bbox[2],bbox[1],bbox[3])) except Exception as e: msg = 'failed to generate config file %s ' % e logger.debug(msg) raise Exception(msg) logger.debug("write_config took %s seconds.", time.time() - start_time) ####################### # CASTf90 call ####################### import subprocess import shlex start_time = time.time() # measure call castf90 self.status.set('Start CASTf90 call', 20) try: #self.status.set('execution of CASTf90', 50) cmd = 'analogue.out %s' % path.relpath(config_file) #system(cmd) args = shlex.split(cmd) output,error = subprocess.Popen(args, stdout = subprocess.PIPE, stderr= subprocess.PIPE).communicate() logger.info('analogue.out info:\n %s ' % output) logger.debug('analogue.out errors:\n %s ' % error) self.status.set('**** CASTf90 suceeded', 90) except Exception as e: msg = 'CASTf90 failed %s ' % e logger.error(msg) raise Exception(msg) logger.debug("castf90 took %s seconds.", time.time() - start_time) self.status.set('preparting output', 99) self.config.setValue( config_file ) self.analogs.setValue( output_file ) self.simulation_netcdf.setValue( simulation ) self.target_netcdf.setValue( archive ) self.status.set('execution ended', 100) logger.debug("total execution took %s seconds.", time.time() - process_start_time)
def set_metadata_segetalflora(resource): """ :param resources: imput files """ # gather the set_metadata dic_segetalflora = { 'keywords' : 'Segetalflora', 'tier': '2', 'in_var' : 'tas', 'description':'Number of European segetalflora species', 'method':'regression equation', 'institution':'Julius Kuehn-Institut (JKI) Federal Research Centre for Cultivated Plants', 'institution_url':'www.jki.bund.de', 'institute_id' : "JKI", 'contact_mail_3':'*****@*****.**', 'version' : '1.0', } dic_climatetype = { '1' : 'cold northern species group', '2' : 'warm northern species group', '3' : 'moderate warm-toned species group', '4' : 'moderate warm-toned to mediterranean species group', '5' : 'mediterranean species group', '6' : 'climate-indifferent species', '7' : 'climate-undefinable species', 'all' : 'species of all climate types' } try: set_basic_md(resource) except Exception as e: logger.error(e) try: set_dynamic_md(resource) except Exception as e: logger.error(e) #set the segetalflora specific metadata try: ds = Dataset(resource, mode='a') ds.setncatts(dic_segetalflora) ds.close() except Exception as e: logger.error(e) # set the variable attributes: from flyingpigeon.utils import get_variable try: ds = Dataset(resource, mode='a') var = get_variable(resource) if 'all' in var: climat_type = 'all' else: climat_type = var[-1] culture_type = var.strip('sf').strip(climat_type) sf = ds.variables[var] sf.setncattr('units',1) sf.setncattr('standard_name', 'sf%s%s' % (culture_type, climat_type)) sf.setncattr('long_name', 'Segetal flora %s land use for %s' % (culture_type, dic_climatetype['%s' % climat_type])) ds.close() except Exception as e: logger.error('failed to set sf attributes %s ' % e) # sort the attributes: try: ds = Dataset(resource, mode='a') att = ds.ncattrs() att.sort() for a in att: entry = ds.getncattr(a) ds.setncattr(a,entry) history = '%s , Segetalflora Impact Model V1.0' % (ds.history) ds.setncattr('history',history) ds.close() except Exception as e: logger.error('failed to sort attributes %s ' % e) return resource
def execute(self): logger.info('Start process') init_process_logger('log.txt') self.output_log.setValue('log.txt') from datetime import datetime as dt from flyingpigeon import weatherregimes as wr from tempfile import mkstemp ################################ # reading in the input arguments ################################ try: resource = self.getInputValues(identifier='resource') url_Rdat = self.getInputValues(identifier='Rdat')[0] url_dat = self.getInputValues(identifier='dat')[0] url_ref_file = self.getInputValues( identifier='netCDF') # can be None season = self.getInputValues(identifier='season')[0] period = self.getInputValues(identifier='period')[0] anualcycle = self.getInputValues(identifier='anualcycle')[0] except Exception as e: logger.debug('failed to read in the arguments %s ' % e) try: start = dt.strptime(period.split('-')[0], '%Y%m%d') end = dt.strptime(period.split('-')[1], '%Y%m%d') # kappa = int(self.getInputValues(identifier='kappa')[0]) logger.info('period %s' % str(period)) logger.info('season %s' % str(season)) logger.info('read in the arguments') logger.info('url_ref_file: %s' % url_ref_file) logger.info('url_Rdat: %s' % url_Rdat) logger.info('url_dat: %s' % url_dat) except Exception as e: logger.debug('failed to convert arguments %s ' % e) ############################ # fetching trainging data ############################ from flyingpigeon.utils import download, get_time from os.path import abspath try: dat = abspath(download(url_dat)) Rdat = abspath(download(url_Rdat)) logger.info('training data fetched') except Exception as e: logger.error('failed to fetch training data %s' % e) ########################################################## # get the required bbox and time region from resource data ########################################################## # from flyingpigeon.weatherregimes import get_level from flyingpigeon.ocgis_module import call from flyingpigeon.utils import get_variable time_range = [start, end] variable = get_variable(resource) if len(url_ref_file) > 0: ref_file = download(url_ref_file[0]) model_subset = call( resource=resource, variable=variable, time_range= time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap', regrid_destination=ref_file, regrid_options='bil') logger.info('Dataset subset with regridding done: %s ' % model_subset) else: model_subset = call( resource=resource, variable=variable, time_range= time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap', ) logger.info('Dataset time period extracted: %s ' % model_subset) ####################### # computing anomalies ####################### cycst = anualcycle.split('-')[0] cycen = anualcycle.split('-')[0] reference = [ dt.strptime(cycst, '%Y%m%d'), dt.strptime(cycen, '%Y%m%d') ] model_anomal = wr.get_anomalies(model_subset, reference=reference) ##################### # extracting season ##################### model_season = wr.get_season(model_anomal, season=season) ####################### # call the R scripts ####################### import shlex import subprocess from flyingpigeon import config from os.path import curdir, exists, join try: rworkspace = curdir Rsrc = config.Rsrc_dir() Rfile = 'weatherregimes_projection.R' yr1 = start.year yr2 = end.year time = get_time(model_season, format='%Y%m%d') # ip, output_graphics = mkstemp(dir=curdir ,suffix='.pdf') ip, file_pca = mkstemp(dir=curdir, suffix='.txt') ip, file_class = mkstemp(dir=curdir, suffix='.Rdat') ip, output_frec = mkstemp(dir=curdir, suffix='.txt') args = [ 'Rscript', join(Rsrc, Rfile), '%s/' % curdir, '%s/' % Rsrc, '%s' % model_season, '%s' % variable, '%s' % str(time).strip("[]").replace("'", "").replace(" ", ""), # '%s' % output_graphics, '%s' % dat, '%s' % Rdat, '%s' % file_pca, '%s' % file_class, '%s' % output_frec, '%s' % season, '%s' % start.year, '%s' % end.year, '%s' % 'MODEL' ] logger.info('Rcall builded') except Exception as e: msg = 'failed to build the R command %s' % e logger.error(msg) raise Exception(msg) try: output, error = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() # , shell=True logger.info('R outlog info:\n %s ' % output) logger.debug('R outlog errors:\n %s ' % error) if len(output) > 0: self.status.set('**** weatherregime in R suceeded', 90) else: logger.error('NO! output returned from R call') except Exception as e: msg = 'weatherregime in R %s ' % e logger.error(msg) raise Exception(msg) ################# # set the outputs ################# # self.Routput_graphic.setValue( output_graphics ) self.output_pca.setValue(file_pca) self.output_classification.setValue(file_class) self.output_netcdf.setValue(model_season) self.output_frequency.setValue(output_frec)
def calc_indice_percentile(resources=[], variable=None, prefix=None, indices='TG90p', refperiod=None, groupings='yr', polygons=None, percentile=90, mosaic = False, dir_output=None, dimension_map = None): """ Calculates given indices for suitable files in the appropriate time grouping and polygon. :param resource: list of filenames in data reference syntax (DRS) convention (netcdf) :param variable: variable name to be selected in the in netcdf file (default=None) :param indices: list of indices (default ='TG90p') :param prefix: filename prefix :param refperiod: reference period tuple = (start,end) :param grouping: indices time aggregation (default='yr') :param dir_output: output directory for result file (netcdf) :param dimension_map: optional dimension map if different to standard (default=None) :return: list of netcdf files with calculated indices. Files are saved into out_dir. """ from os.path import join, dirname, exists from os import remove import uuid from numpy import ma from datetime import datetime as dt from flyingpigeon.ocgis_module import call from flyingpigeon.subset import clipping from flyingpigeon.utils import get_values, get_time if type(resources) != list: resources = list([resources]) if type(indices) != list: indices = list([indices]) if type(groupings) != list: groupings = list([groupings]) if type(refperiod) == list: refperiod = refperiod[0] if refperiod != None: start = dt.strptime(refperiod.split('-')[0] , '%Y%m%d') end = dt.strptime(refperiod.split('-')[1] , '%Y%m%d') time_range = [start, end] else: time_range = None if dir_output != None: if not exists(dir_output): makedirs(dir_output) ######################################################################################################################## # Compute a custom percentile basis using ICCLIM. ###################################################################### ######################################################################################################################## from ocgis.contrib import library_icclim as lic nc_indices = [] nc_dic = sort_by_filename(resources) for grouping in groupings: calc_group = calc_grouping(grouping) for key in nc_dic.keys(): resource = nc_dic[key] if variable == None: variable = get_variable(resource) if polygons == None: nc_reference = call(resource=resource, prefix=str(uuid.uuid4()), time_range=time_range, output_format='nc', dir_output=dir_output) else: nc_reference = clipping(resource=resource, prefix=str(uuid.uuid4()), time_range=time_range, output_format='nc', polygons=polygons, dir_output=dir_output, mosaic = mosaic) arr = get_values(resource=nc_reference) dt_arr = get_time(resource=nc_reference) arr = ma.masked_array(arr) dt_arr = ma.masked_array(dt_arr) percentile = percentile window_width = 5 for indice in indices: name = indice.replace('_', str(percentile)) var = indice.split('_')[0] operation = None if 'T' in var: if percentile >= 50: operation = 'Icclim%s90p' % var func = 'icclim_%s90p' % var # icclim_TG90p else: operation = 'Icclim%s10p' % var func = 'icclim_%s10p' % var ################################ # load the appropriate operation ################################ ops = [op for op in dir(lic) if operation in op] if len(ops) == 0: raise Exception("operator does not exist %s", operation) exec "percentile_dict = lic.%s.get_percentile_dict(arr, dt_arr, percentile, window_width)" % ops[0] calc = [{'func': func, 'name': name, 'kwds': {'percentile_dict': percentile_dict}}] if polygons == None: nc_indices.append(call(resource=resource, prefix=key.replace(variable,name).replace('_day_', '_%s_' % grouping), calc=calc, calc_grouping=calc_group, output_format='nc', dir_output=dir_output)) else: nc_indices.extend(clipping(resource=resource, prefix=key.replace(variable,name).replace('_day_', '_%s_' % grouping), calc=calc, calc_grouping=calc_group, output_format='nc', dir_output=dir_output, polygons=polygons, mosaic = mosaic, )) return nc_indices #def calc_indice_unconventional(resource=[], variable=None, prefix=None, #indices=None, polygons=None, groupings=None, #dir_output=None, dimension_map = None): #""" #Calculates given indices for suitable files in the appropriate time grouping and polygon. #:param resource: list of filenames in data reference syntax (DRS) convention (netcdf) #:param variable: variable name to be selected in the in netcdf file (default=None) #:param indices: list of indices (default ='TGx') #:param polygons: list of polygons (default =None) #:param grouping: indices time aggregation (default='yr') #:param out_dir: output directory for result file (netcdf) #:param dimension_map: optional dimension map if different to standard (default=None) #:return: list of netcdf files with calculated indices. Files are saved into dir_output #""" #from os.path import join, dirname, exists #from os import remove #import uuid #from flyingpigeon import ocgis_module #from flyingpigeon.subset import get_ugid, get_geom #if type(resource) != list: #resource = list([resource]) #if type(indices) != list: #indices = list([indices]) #if type(polygons) != list and polygons != None: #polygons = list([polygons]) #elif polygons == None: #polygons = [None] #else: #logger.error('Polygons not found') #if type(groupings) != list: #groupings = list([groupings]) #if dir_output != None: #if not exists(dir_output): #makedirs(dir_output) #experiments = sort_by_filename(resource) #outputs = [] #print('environment for calc_indice_unconventional set') #logger.info('environment for calc_indice_unconventional set') #for key in experiments: #if variable == None: #variable = get_variable(experiments[key][0]) #try: #ncs = experiments[key] #for indice in indices: #logger.info('indice: %s' % indice) #try: #for grouping in groupings: #logger.info('grouping: %s' % grouping) #try: #calc_group = calc_grouping(grouping) #logger.info('calc_group: %s' % calc_group) #for polygon in polygons: #try: #domain = key.split('_')[1].split('-')[0] #if polygon == None: #if prefix == None: #prefix = key.replace(variable, indice).replace('_day_','_%s_' % grouping ) #geom = None #ugid = None #else: #if prefix == None: #prefix = key.replace(variable, indice).replace('_day_','_%s_' % grouping ).replace(domain,polygon) #geom = get_geom(polygon=polygon) #ugid = get_ugid(polygons=polygon, geom=geom) #if indice == 'TGx': #calc=[{'func': 'max', 'name': 'TGx'}] #tmp = ocgis_module.call(resource=ncs,# conform_units_to='celcius', #variable=variable, dimension_map=dimension_map, #calc=calc, calc_grouping=calc_group, prefix=prefix, #dir_output=dir_output, geom=geom, select_ugid=ugid) #elif indice == 'TGn': #calc=[{'func': 'min', 'name': 'TGn'}] #tmp = ocgis_module.call(resource=ncs, #conform_units_to='celcius', #variable=variable, dimension_map=dimension_map, #calc=calc, calc_grouping= calc_group, prefix=prefix, #dir_output=dir_output, geom=geom, select_ugid = ugid) #elif indice == 'TGx5day': #calc = [{'func': 'moving_window', 'name': 'TGx5day', 'kwds': {'k': 5, 'operation': 'mean', 'mode': 'same' }}] #tmp2 = ocgis_module.call(resource=ncs, #conform_units_to='celcius', #variable=variable, dimension_map=dimension_map, #calc=calc, prefix=str(uuid.uuid4()), #geom=geom, select_ugid = ugid) #calc=[{'func': 'max', 'name': 'TGx5day'}] #logger.info('moving window calculated : %s' % tmp2) #tmp = ocgis_module.call(resource=tmp2, #variable=indice, dimension_map=dimension_map, #calc=calc, calc_grouping=calc_group, prefix=prefix, #dir_output=dir_output) #remove(tmp2) #elif indice == 'TGn5day': #calc = [{'func': 'moving_window', 'name': 'TGn5day', 'kwds': {'k': 5, 'operation': 'mean', 'mode': 'same' }}] #tmp2 = ocgis_module.call(resource=ncs, #conform_units_to='celcius', #variable=variable, dimension_map=dimension_map, #calc=calc, prefix=str(uuid.uuid4()), #geom=geom, select_ugid = ugid) #calc=[{'func': 'min', 'name': 'TGn5day'}] #logger.info('moving window calculated : %s' % tmp2) #tmp = ocgis_module.call(resource=tmp2, #variable=indice, dimension_map=dimension_map, #calc=calc, calc_grouping=calc_group, prefix=prefix, #dir_output=dir_output) #remove(tmp2) #else: #logger.error('Indice %s is not a known inidce' % (indice)) #outputs.append(tmp) #logger.info('indice file calcualted %s ' % (tmp)) #except Exception as e: #logger.debug('could not calc indice %s for key %s, polygon %s and calc_grouping %s : %s' % (indice, key, polygon, grouping, e )) #except Exception as e: #logger.debug('could not calc indice %s for key %s and calc_grouping %s : %s' % ( indice, key, polygon, e )) #except Exception as e: #logger.debug('could not calc indice %s for key %s: %s'% (indice, key, e )) #except Exception as e: #logger.debug('could not calc key %s: %s' % (key, e)) #return outputs
def get_anomalies(nc_file, frac=0.2, reference=None, method='ocgis', sseas='serial', variable=None): """ Anomalisation of data subsets for weather classification by subtracting a smoothed annual cycle :param nc_file: input netCDF file :param frac: Number between 0-1 for strength of smoothing (0 = close to the original data, 1 = flat line) default = 0.2 :param reference: Period to calculate annual cycle :returns str: path to output netCDF file """ from netCDF4 import Dataset if variable is None: variable = utils.get_variable(nc_file) # if more when 2 variables: if (variable.count(variable)==0): _ds=Dataset(nc_file) # Works only if we have one 3D variables for j in variable: if len(_ds.variables[j].dimensions)==3: _var=j variable=_var _ds.close() LOGGER.debug('3D Variable selected: %s'%(variable)) try: if (method == 'cdo'): from cdo import Cdo from os import system ip2, nc_anual_cycle = mkstemp(dir='.', suffix='.nc') cdo = Cdo() #ip, nc_anual_cycle_tmp = mkstemp(dir='.', suffix='.nc') # TODO: if reference is none, use utils.get_time for nc_file to set the ref range # But will need to fix 360_day issue (use get_time_nc from analogs) # com = 'seldate' # comcdo = 'cdo %s,%s-%s-%s,%s-%s-%s %s %s' % (com, reference[0].year, reference[0].month, reference[0].day, # reference[1].year, reference[1].month, reference[1].day, # nc_file, nc_anual_cycle_tmp) # LOGGER.debug('CDO: %s' % (comcdo)) # system(comcdo) # Sub cdo with this trick... Cdo keeps the precision and anomalies are integers... calc = '%s=%s'%(variable, variable) nc_anual_cycle_tmp = call(nc_file, time_range=reference, variable=variable, calc=calc) nc_anual_cycle = cdo.ydaymean(input=nc_anual_cycle_tmp, output=nc_anual_cycle) else: calc = [{'func': 'mean', 'name': variable}] calc_grouping = calc_grouping = ['day', 'month'] nc_anual_cycle = call(nc_file, calc=calc, calc_grouping=calc_grouping, variable=variable, time_range=reference) LOGGER.info('annual cycle calculated: %s' % (nc_anual_cycle)) except Exception as e: msg = 'failed to calcualte annual cycle %s' % e LOGGER.error(msg) raise Exception(msg) try: # spline for smoothing #import statsmodels.api as sm #from numpy import tile, empty, linspace from cdo import Cdo cdo = Cdo() # variable = utils.get_variable(nc_file) ds = Dataset(nc_anual_cycle, mode='a') vals = ds.variables[variable] vals_sm = empty(vals.shape) ts = vals.shape[0] x = linspace(1, ts*3, num=ts*3, endpoint=True) if ('serial' not in sseas): # Multiprocessing ======================= from multiprocessing import Pool pool = Pool() valex = [0.] valex = valex*vals.shape[1]*vals.shape[2] # TODO redo with reshape ind = 0 for lat in range(vals.shape[1]): for lon in range(vals.shape[2]): valex[ind] = vals[:, lat, lon] ind += 1 LOGGER.debug('Start smoothing with multiprocessing') # TODO fraction option frac=... is not used here tmp_sm = pool.map(_smooth, valex) pool.close() pool.join() # TODO redo with reshape ind=0 for lat in range(vals.shape[1]): for lon in range(vals.shape[2]): vals_sm[:, lat, lon] = tmp_sm[ind] ind+=1 else: # Serial ================================== vals_sm = empty(vals.shape) for lat in range(vals.shape[1]): for lon in range(vals.shape[2]): try: y = tile(vals[:, lat, lon], 3) # ys = smooth(y, window_size=91, order=2, deriv=0, rate=1)[ts:ts*2] ys = sm.nonparametric.lowess(y, x, frac=frac)[ts:ts*2, 1] vals_sm[:, lat, lon] = ys except: msg = 'failed for lat %s lon %s' % (lat, lon) LOGGER.exception(msg) raise Exception(msg) LOGGER.debug('done for %s - %s ' % (lat, lon)) vals[:, :, :] = vals_sm[:, :, :] ds.close() LOGGER.info('smothing of annual cycle done') except: msg = 'failed smothing of annual cycle' LOGGER.exception(msg) raise Exception(msg) try: ip, nc_anomal = mkstemp(dir='.', suffix='.nc') try: nc_anomal = cdo.sub(input=[nc_file, nc_anual_cycle], output=nc_anomal) LOGGER.info('cdo.sub; anomalisation done: %s ' % nc_anomal) except: # bug cdo: https://code.mpimet.mpg.de/boards/1/topics/3909 ip3, nc_in1 = mkstemp(dir='.', suffix='.nc') ip4, nc_in2 = mkstemp(dir='.', suffix='.nc') ip5, nc_out = mkstemp(dir='.', suffix='.nc') nc_in1 = cdo.selvar(variable, input=nc_file, output=nc_in1) nc_in2 = cdo.selvar(variable, input=nc_anual_cycle, output=nc_in2) nc_out = cdo.sub(input=[nc_in1, nc_in2], output=nc_out) nc_anomal = nc_out except: msg = 'failed substraction of annual cycle' LOGGER.exception(msg) raise Exception(msg) return nc_anomal
def calc_indice_percentile(resource=[], variable=None, prefix=None, indices='TG90p', refperiod=None, grouping='yr', polygons=None, percentile=90, mosaic=False, dir_output=None, dimension_map=None): """ Calculates given indices for suitable dataset in the appropriate time grouping and polygon. :param resource: list of filenames in data reference syntax (DRS) convention (netcdf) :param variable: variable name to be selected in the in netcdf file (default=None) :param indices: string of indice (default ='TG90p') :param prefix: filename prefix :param refperiod: reference period = [datetime,datetime] :param grouping: indices time aggregation (default='yr') :param dir_output: output directory for result file (netcdf) :param dimension_map: optional dimension map if different to standard (default=None) :return: reference_file, indice_file """ from os.path import join, dirname, exists from os import remove import uuid from numpy import ma from datetime import datetime as dt from flyingpigeon.ocgis_module import call from flyingpigeon.subset import clipping from flyingpigeon.utils import get_values, get_time # TODO: see ticket https://github.com/bird-house/flyingpigeon/issues/200 raise NotImplementedError('Sorry! Function is under construction.') if type(resource) != list: resource = list([resource]) # if type(indices) != list: # indices = list([indices]) # # if type(groupings) != list: # groupings = list([groupings]) # # if type(refperiod) == list: # refperiod = refperiod[0] # # if refperiod is not None: # start = dt.strptime(refperiod.split('-')[0], '%Y%m%d') # end = dt.strptime(refperiod.split('-')[1], '%Y%m%d') # time_range = [start, end] # else: # time_range = None ################################################ # Compute a custom percentile basis using ICCLIM ################################################ from ocgis.contrib import library_icclim as lic calc_group = calc_grouping(grouping) if variable is None: variable = get_variable(resource) if polygons is None: nc_reference = call(resource=resource, prefix=str(uuid.uuid4()), time_range=refperiod, output_format='nc') else: nc_reference = clipping(resource=resource, prefix=str(uuid.uuid4()), time_range=refperiod, output_format='nc', polygons=polygons, mosaic=mosaic) # arr = get_values(resource=nc_reference) # dt_arr = get_time(resource=nc_reference) # arr = ma.masked_array(arr) # dt_arr = ma.masked_array(dt_arr) # percentile = percentile # window_width = 5 # for indice in indices: # name = indice.replace('_', str(percentile)) # var = indice.split('_')[0] # # operation = None # if 'T' in var: # if percentile >= 50: # operation = 'Icclim%s90p' % var # func = 'icclim_%s90p' % var # icclim_TG90p # else: # operation = 'Icclim%s10p' % var # func = 'icclim_%s10p' % var # # ################################ # # load the appropriate operation # ################################ # # ops = [op for op in dir(lic) if operation in op] # if len(ops) == 0: # raise Exception("operator does not exist %s", operation) # # exec "percentile_dict = lic.%s.get_percentile_dict(arr, dt_arr, percentile, window_width)" % ops[0] # calc = [{'func': func, 'name': name, 'kwds': {'percentile_dict': percentile_dict}}] # # if polygons is None: # nc_indices.extend(call(resource=resource, # prefix=key.replace(variable, name).replace('_day_', '_%s_' % grouping), # calc=calc, # calc_grouping=calc_group, # output_format='nc')) # else: # nc_indices.extend(clipping(resource=resource, # prefix=key.replace(variable, name).replace('_day_', '_%s_' % grouping), # calc=calc, # calc_grouping=calc_group, # output_format='nc', # polygons=polygons, # mosaic=mosaic, # )) # if len(nc_indices) is 0: # LOGGER.debug('No indices are calculated') # return None return nc_indices
def uncertainty(resouces , variable=None, title=None, dir_out=None): """ retunes an html file containing the appropriate uncertainty plot. :param resouces: list of files containing the same variable :param variable: variable to be visualised, if None (default) variable will be detected :param title: sting to be used as title """ logger.debug('Start visualisation uncertainty plot') from bokeh.plotting import figure, output_file, save import cdo cdo = cdo.Cdo() # === prepare invironment if type(resouces) == str: resouces = list([resouces]) if variable == None: variable = utils.get_variable(resouces[0]) if title == None: title = "Field mean of %s " % variable if dir_out == None: dir_out = '.' # === prepare bokeh try: o1 , output_html = mkstemp(dir=dir_out, suffix='.html') fig = figure(x_axis_type = "datetime", tools="pan,wheel_zoom,box_zoom,reset,previewsave") output_file(output_html, title=variable, autosave=True,) # === get the datetime dates = set() for nc in resouces: logger.debug('looping files : %s ' % (nc)) # get timestapms rawDate = cdo.showdate(input=[nc]) # ds.variables['time'][:] strDate = rawDate[0].split(' ') logger.debug('len strDate : %s ' % (len(strDate))) dates = dates.union(strDate) #dates.union( utils.get_time(nc)) #self.show_status('dates : %s ' % len(dates), 62) ldates = list(dates) ldates.sort() ddates = dict( (ldates[i], i) for i in range(0,len(ldates))) # initialise matirx ma = np.empty([len(ddates), len(resouces)])*np.nan #self.show_status('ddates : %s ' % ddates, 62) # fill matrix for y in range(0,len(resouces)) : rawDate = cdo.showdate(input=[resouces[y]]) # ds.variables['time'][:] strDate = rawDate[0].split(' ') ds=Dataset(resouces[y]) data = np.squeeze(ds.variables[variable][:]) if len(data.shape) == 3: meanData = np.mean(data,axis=1) ts = np.mean(meanData,axis=1) else: ts = data logger.debug('ts array : %s ' % (len(ts)) ) for t in range(0, len(strDate)) : x = ddates.get(strDate[t],0) ma[x,y] = ts[t] # get datetimes dt = [datetime.strptime(elem, '%Y-%m-%d') for elem in ldates] mdat = np.ma.masked_array(ma ,np.isnan(ma)) #self.show_status('matrix masked %s ' % mdat , 80) #logger.debug('matrix %s ', mdat.shape) ma_mean = np.mean(mdat,axis=1) logger.debug('mean %s '% len(ma_mean)) ma_min = np.min(mdat,axis=1) ma_max = np.max(mdat,axis=1) #ma_sub = np.subtract(ma_max, ma_min) #ma_per75 = np.percentile(mdat,75, axis=0) #ma_per25 = np.percentile(mdat,25, axis=0) logger.debug('ma Vaules %s' % len(mdat.data)) #line(dt, ma_min , color='grey' ,line_width=1) #line(dt, ma_max , color='grey' , line_width=2 ) fig.line(dt, ma_mean , color='red', line_width=1) x = [] y = [] x = np.append(dt,dt[::-1]) y = np.append(ma_min, ma_max[::-1]) fig.patch(x,y, color='grey', alpha=0.8, line_color=None) fig.title = "Mean and Uncertainty of %s " % variable fig.grid save(fig) logger.debug('timesseries uncertainty plot done for %s'% variable) except Exception as e: logger.exception('bokeh uncertainty plot failed for %s' % variable) raise return output_html
def spaghetti(resouces, variable=None, title=None, dir_out=None): """ creates a png file containing the appropriate spaghetti plot as a field mean of the values. :param resouces: list of files containing the same variable :param variable: variable to be visualised. If None (default), variable will be detected :param title: string to be used as title :param dir_out: directory for output files :retruns str: path to png file """ try: fig = plt.figure(figsize=(20,10), dpi=600, facecolor='w', edgecolor='k') logger.debug('Start visualisation spaghetti plot') # === prepare invironment if type(resouces) != list: resouces = [resouces] if variable == None: variable = utils.get_variable(resouces[0]) if title == None: title = "Field mean of %s " % variable if dir_out == None: dir_out = os.curdir logger.info('plot values preparation done') except Exception as e: msg = "plot values preparation failed: %s" % (e) logger.exception(msg) raise Exception(msg) try: o1 , output_png = mkstemp(dir=dir_out, suffix='.png') for c , nc in enumerate(resouces): # get timestapms try: d = utils.get_time(nc) # [datetime.strptime(elem, '%Y-%m-%d') for elem in strDate[0]] dt = [datetime.strptime(str(i), '%Y-%m-%d %H:%M:%S') for i in d ] ds=Dataset(nc) data = np.squeeze(ds.variables[variable][:]) if len(data.shape) == 3: meanData = np.mean(data,axis=1) ts = np.mean(meanData,axis=1) else: ts = data[:] plt.plot( dt,ts ) #fig.line( dt,ts ) except Exception as e: msg = "lineplot failed for %s" % (nc) logger.exception(msg) raise Exception(msg) plt.title(title, fontsize=20) plt.grid() fig.savefig(output_png) plt.close() logger.info('timeseries spaghetti plot done for %s with %s lines.'% (variable, c)) except Exception as e: msg = 'matplotlib spaghetti plot failed: %s' % e logger.exception(msg) raise Exception(msg) return output_png
def spaghetti(resouces, variable=None, title=None, dir_out=None): """ retunes a png file containing the appropriate spaghetti plot. :param resouces: list of files containing the same variable :param variable: variable to be visualised, if None (default) variable will be detected :param title: sting to be used as title :param dir_out: directory for output files """ fig = plt.figure(figsize=(20,10), dpi=600, facecolor='w', edgecolor='k') logger.debug('Start visualisation spagetti plot') # === prepare invironment if type(resouces) == str: resouces = list([resouces]) if variable == None: variable = utils.get_variable(resouces[0]) if title == None: title = "Field mean of %s " % variable if dir_out == None: dir_out = os.curdir try: o1 , output_png = mkstemp(dir=dir_out, suffix='.png') for c , nc in enumerate(resouces): # get timestapms try: d = utils.get_time(nc) # [datetime.strptime(elem, '%Y-%m-%d') for elem in strDate[0]] dt = [datetime.strptime(str(i), '%Y-%m-%d %H:%M:%S') for i in d ] ds=Dataset(nc) data = np.squeeze(ds.variables[variable][:]) if len(data.shape) == 3: meanData = np.mean(data,axis=1) ts = np.mean(meanData,axis=1) else: ts = data plt.plot( dt,ts ) #fig.line( dt,ts ) except Exception as e: logger.debug('lineplot failed for %s: %s\n' % (nc, e)) # plot into current figure # , legend= nc #fig.legend()[0].orientation = "bottom_left" # fig.legend().orientation = "bottom_left" plt.title(title, fontsize=20) plt.grid()# .grid_line_alpha=0.3 #lt.rcParams.update({'font.size': 22}) #window_size = 30 #window = np.ones(window_size)/float(window_size) fig.savefig(output_png) #bplt.hold('off') plt.close() logger.debug('timesseries spagetti plot done for %s with %s lines.'% (variable, c)) except Exception as e: msg = 'matplotlib spagetti plot failed for %s' % variable logger.debug(msg) #raise Exception(msg) return output_png
def get_gam(ncs_indices, coordinate): from netCDF4 import Dataset from os.path import basename from shapely.geometry import Point from numpy import squeeze, ravel, isnan, nan, array, reshape from flyingpigeon.utils import get_variable, get_values, unrotate_pole from flyingpigeon.ocgis_module import call try: from rpy2.robjects.packages import importr import rpy2.robjects as ro import rpy2.robjects.numpy2ri rpy2.robjects.numpy2ri.activate() base = importr("base") stats = importr("stats") mgcv = importr("mgcv") logger.info('rpy2 modules imported') except Exception as e: msg = 'failed to import rpy2 modules %s' % e logger.debug(msg) raise Exception(msg) for i, ncs in enumerate(ncs_indices): # ocgis need unrotated coordinates to extract points # unrotate_pole writes lats lons into the file. # ACHTUNG: will fail if the data is stored on a file system with no write permissions try: lats, lons = unrotate_pole(ncs, write_to_file=True) point = Point(float(coordinate[0]), float(coordinate[1])) # get the values variable = get_variable(ncs) agg = basename(ncs).split('_')[-2] indice = '%s_%s' % (variable, agg) timeseries = call(resource=ncs, geom=point, select_nearest=True) ts = Dataset(timeseries) vals = squeeze(ts.variables[variable][:]) from numpy import min, max, mean, append, zeros, ones dif = max(vals) - min(vals) a = append(vals - dif ,vals) vals = append(a, vals+dif) if i == 0 : from numpy import zeros, ones a = append (zeros(len(vals)) , ones(len(vals)) ) PA = append(a , zeros(len(vals))) data = {'PA': ro.FloatVector(PA)} data[str(indice)] = ro.FloatVector(vals) form = 'PA ~ ' form = form + 's(%s, k=3)' % indice else: form = form + ' + s(%s, k=3)' % indice data[str(indice)] = ro.FloatVector(vals) except Exception as e: msg = 'Failed to prepare data %s' % e logger.debug(msg) try: logger.info(data) dataf = ro.DataFrame(data) eq = ro.Formula(str(form)) gam_model = mgcv.gam(base.eval(eq), data=dataf, family=stats.binomial(), scale=-1, na_action=stats.na_exclude) # logger.info('GAM model trained') except Exception as e: msg = 'Failed to generate GAM model %s' % e logger.debug(msg) # ### ########################### # # plot response curves # ### ########################### try: from flyingpigeon.visualisation import concat_images from tempfile import mkstemp grdevices = importr('grDevices') graphicDev = importr('Cairo') infos = [] for i in range(1,len(ncs_indices)+1): ip, info = mkstemp(dir='.',suffix='.png') #grdevices.png(filename=info) #graphicDev.CairoPDF(info, width = 7, height = 7, pointsize = 12) graphicDev.CairoPNG(info, width = 640 , height = 480, pointsize = 12) # 640, 480) #, pointsize = 12 width = 30, height = 30, print 'file opened!' infos.append(info) #grdevices.png(filename=info) ylim = ro.IntVector([-6,6]) trans = ro.r('function(x){exp(x)/(1+exp(x))}') mgcv.plot_gam(gam_model, trans=trans, shade='T', col='black',select=i,ylab='Predicted Probability',rug=False , cex_lab = 1.4, cex_axis = 1.4, ) # print 'gam plotted ;-)' #ylim=ylim, , grdevices.dev_off() #graphicDev.dev_off() #graphicDev.Cairo_onSave( dev_cur(), onSave=True ) print(' %s plots generated ' % len(infos)) infos_concat = concat_images(infos, orientation='h') except Exception as e: msg = 'Failed to plot statistical graphic %s' % e logger.debug(msg) raise Exception(msg) return gam_model, infos_concat
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' ncs = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) var = get_variable(ncs[0]) LOGGER.info('variable to be plotted: {}'.format(var)) # mosaic = self.mosaic.getValue() if 'region' in request.inputs: regions = [inp.data for inp in request.inputs['region']] try: png_region = vs.plot_polygons(regions) except Exception as ex: msg = 'failed to plot the polygon to world map: {}'.format( str(ex)) LOGGER.exception(msg) raise Exception(msg) o1, png_region = mkstemp(dir='.', suffix='.png') # clip the demanded polygons subsets = clipping( resource=ncs, variable=var, polygons=regions, mosaic=True, spatial_wrapping='wrap', ) else: subsets = ncs png_region = vs.plot_extend(ncs[0]) response.update_status('Arguments set for subset process', 0) try: tar_subsets = archive(subsets) except Exception as ex: msg = 'failed to archive subsets: {}'.format(ex) LOGGER.exception(msg) raise Exception(msg) _, tar_subsets = mkstemp(dir='.', suffix='.tar') try: png_uncertainty = vs.uncertainty(subsets, variable=var) except Exception as ex: msg = 'failed to generate the uncertainty plot: {}'.format(ex) LOGGER.exception(msg) raise Exception(msg) _, png_uncertainty = mkstemp(dir='.', suffix='.png') try: png_spaghetti = vs.spaghetti( subsets, variable=var, ) except Exception as ex: msg = 'failed to generate the spaghetti plot: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) _, png_spaghetti = mkstemp(dir='.', suffix='.png') try: from flyingpigeon import robustness as ro signal, low_agreement_mask, high_agreement_mask, text_src = ro.signal_noise_ratio( resource=subsets, # start=None, end=None, # timeslice=None, # variable=var ) # if title is None: title = 'signal robustness of %s ' % ( var) # , end1, end2, start1, start2 png_robustness = vs.map_robustness( signal, high_agreement_mask, low_agreement_mask, # cmap=cmap, # title=title ) LOGGER.info('robustness graphic generated') except Exception as ex: msg = 'failed to generate the robustness plot: {}'.format(ex) LOGGER.exception(msg) raise Exception(msg) _, png_robustness = mkstemp(dir='.', suffix='.png') factsheet = vs.factsheetbrewer(png_region=png_region, png_uncertainty=png_uncertainty, png_spaghetti=png_spaghetti, png_robustness=png_robustness) response.outputs['output_nc'].file = tar_subsets response.outputs['output_factsheet'].file = factsheet response.update_status("done", 100) return response
def calc_indice_simple(resource=[], variable=None, prefix=None,indices=None, polygons=None, mosaik = False, groupings='yr', dir_output=None, dimension_map = None, memory_limit=None): """ Calculates given simple indices for suitable files in the appopriate time grouping and polygon. :param resource: list of filenames in drs convention (netcdf) :param variable: variable name to be selected in the in netcdf file (default=None) :param indices: list of indices (default ='SU') :param polygons: list of polgons (default ='FRA') :param grouping: indices time aggregation (default='yr') :param out_dir: output directory for result file (netcdf) :param dimension_map: optional dimension map if different to standard (default=None) :return: list of netcdf files with calculated indices. Files are saved into out_dir """ from os.path import join, dirname, exists from flyingpigeon import ocgis_module from flyingpigeon.subset import clipping import uuid #DIR_SHP = config.shapefiles_dir() #env.DIR_SHPCABINET = DIR_SHP #env.OVERWRITE = True if type(resource) != list: resource = list([resource]) if type(indices) != list: indices = list([indices]) if type(polygons) != list and polygons != None: polygons = list([polygons]) if type(groupings) != list: groupings = list([groupings]) if dir_output != None: if not exists(dir_output): makedirs(dir_output) #from flyingpigeon.subset import select_ugid # tile_dim = 25 output = None experiments = sort_by_filename(resource) outputs = [] for key in experiments: if variable == None: variable = get_variable(experiments[key][0]) #variable = key.split('_')[0] try: if variable == 'pr': calc = 'pr=pr*86400' ncs = ocgis_module.call(resource=experiments[key], variable=variable, dimension_map=dimension_map, calc=calc, memory_limit=memory_limit, #alc_grouping= calc_group, prefix=str(uuid.uuid4()), dir_output=dir_output, output_format='nc') else: ncs = experiments[key] for indice in indices: logger.info('indice: %s' % indice) try: calc = [{'func' : 'icclim_' + indice, 'name' : indice}] logger.info('calc: %s' % calc) for grouping in groupings: logger.info('grouping: %s' % grouping) try: calc_group = calc_grouping(grouping) logger.info('calc_group: %s' % calc_group) if polygons == None: try: if prefix == None: prefix = key.replace(variable, indice).replace('_day_','_%s_' % grouping ) tmp = ocgis_module.call(resource=ncs, variable=variable, dimension_map=dimension_map, calc=calc, calc_grouping= calc_group, prefix=prefix, dir_output=dir_output, output_format='nc') outputs.extend( [tmp] ) except Exception as e: msg = 'could not calc indice %s for domain in %s' %( indice, key) logger.exception( msg ) raise Exception(msg) else: try: if prefix == None: prefix = key.replace(variable, indice).replace('_day_','_%s_' % grouping ) tmp = clipping(resource=ncs, variable=variable, dimension_map=dimension_map, calc=calc, calc_grouping= calc_group, prefix=prefix, polygons=polygons, mosaik=mosaik, dir_output=dir_output, output_format='nc') outputs.extend( [tmp] ) except Exception as e: msg = 'could not calc indice %s for domain in %s' %( indice, key) logger.exception( msg ) raise Exception(msg) logger.info('indice file calculated') except Exception as e: msg = 'could not calc indice %s for key %s and grouping %s' % (indice, key, grouping) logger.exception(msg) raise Exception(msg) except Exception as e: msg = 'could not calc indice %s for key %s' % ( indice, key) logger.exception(msg) raise Exception(msg) except Exception as e: msg = 'could not calc key %s' % key logger.exception(msg) raise Exception(msg) return outputs
def seacyc(archive, simulation, method='base'): """ Subtracts the seasonal cycle. :param archive: netCDF file containing the reference period :param simulation: netCDF file containing the period to be analysed :param method: method to generate the seasonal cycle files base = seasonal cycle generated from reference period sim = seasonal cycle generated from period to be analysed own = seasonal cycle generated for both time windows :return [str,str]: two netCDF filenames for analysis and reference period (located in working directory) """ try: logger.debug('seacyc started with method: %s' % method) from shutil import copy from flyingpigeon.ocgis_module import call from flyingpigeon.utils import get_variable from cdo import Cdo cdo = Cdo() if method == 'base': seasoncyc_base = cdo.ydaymean( input=archive, output='seasoncyc_base.nc') variable = get_variable(archive) # seasoncyc_base = call(resource=archive, # variable=variable, # prefix='seasoncyc_base', #calc=[{'func': 'mean', 'name': variable}], # calc_grouping=['day','month'] ) logger.debug('seasoncyc_base calculated : %s' % seasoncyc_base) cdo.ydaymean(input=archive, output='seasoncyc_base.nc') seasoncyc_sim = 'seasoncyc_sim.nc' copy(seasoncyc_base, seasoncyc_sim) elif method == 'sim': # seasoncyc_sim = call(resource=archive, # variable=variable, # prefix='seasoncyc_sim', #calc=[{'func': 'mean', 'name': variable}], # calc_grouping=['day','month'] ) cdo.ydaymean(input=simulation, output='seasoncyc_sim.nc') seasoncyc_base = 'seasoncyc_base.nc' copy(seasoncyc_sim, seasoncyc_base) elif method == 'own': # seasoncyc_base = call(resource=archive, # variable=variable, # prefix='seasoncyc_base', #calc=[{'func': 'mean', 'name': variable}], # calc_grouping=['day','month'] ) seasoncyc_base = cdo.ydaymean( input=archive, output='seasoncyc_base.nc') # seasoncyc_sim = call(resource=archive, # variable=variable, # prefix='seasoncyc_sim', #calc=[{'func': 'mean', 'name': variable}], # calc_grouping=['day','month'] ) seasoncyc_sim = cdo.ydaymean( input=simulation, output='seasoncyc_sim.nc') else: raise Exception('normalisation method not found') except Exception as e: msg = 'seacyc function failed : %s ' % e logger.debug(msg) raise Exception(msg) return seasoncyc_base, seasoncyc_sim
def execute(self): logger.info('Start process') from datetime import datetime as dt from flyingpigeon import weatherregimes as wr from tempfile import mkstemp ################################ # reading in the input arguments ################################ try: logger.info('read in the arguments') resource = self.getInputValues(identifier='resource') season = self.getInputValues(identifier='season')[0] bbox = self.getInputValues(identifier='BBox')[0] #model_var = self.getInputValues(identifier='reanalyses')[0] period = self.getInputValues(identifier='period')[0] anualcycle = self.getInputValues(identifier='anualcycle')[0] # model, var = model_var.split('_') bbox = [float(b) for b in bbox.split(',')] start = dt.strptime(period.split('-')[0] , '%Y%m%d') end = dt.strptime(period.split('-')[1] , '%Y%m%d') kappa = int(self.getInputValues(identifier='kappa')[0]) logger.info('bbox %s' % bbox) logger.info('period %s' % str(period)) logger.info('season %s' % str(season)) except Exception as e: logger.debug('failed to read in the arguments %s ' % e) ############################################################ ### get the required bbox and time region from resource data ############################################################ # from flyingpigeon.weatherregimes import get_level from flyingpigeon.ocgis_module import call from flyingpigeon.utils import get_variable time_range = [start, end] variable = get_variable(resource) model_subset = call(resource=resource, variable=variable, geom=bbox, spatial_wrapping='wrap', time_range=time_range, #conform_units_to=conform_units_to ) logger.info('Dataset subset done: %s ' % model_subset) ############################################## ### computing anomalies ############################################## cycst = anualcycle.split('-')[0] cycen = anualcycle.split('-')[0] reference = [dt.strptime(cycst,'%Y%m%d'), dt.strptime(cycen,'%Y%m%d')] model_anomal = wr.get_anomalies(model_subset, reference=reference) ##################### ### extracting season ##################### model_season = wr.get_season(model_anomal, season=season) ####################### ### call the R scripts ####################### import shlex import subprocess from flyingpigeon import config from os.path import curdir, exists, join try: rworkspace = curdir Rsrc = config.Rsrc_dir() Rfile = 'weatherregimes_model.R' infile = model_season #model_subset #model_ponderate modelname = 'MODEL' yr1 = start.year yr2 = end.year ip, output_graphics = mkstemp(dir=curdir ,suffix='.pdf') ip, file_pca = mkstemp(dir=curdir ,suffix='.dat') ip, file_class = mkstemp(dir=curdir ,suffix='.Rdat') args = ['Rscript', join(Rsrc,Rfile), '%s/' % curdir, '%s/' % Rsrc, '%s'% infile, '%s' % variable, '%s' % output_graphics, '%s' % file_pca, '%s' % file_class, '%s' % season, '%s' % start.year, '%s' % end.year, '%s' % 'MODEL', '%s' % kappa] logger.info('Rcall builded') except Exception as e: msg = 'failed to build the R command %s' % e logger.error(msg) raise Exception(msg) try: output,error = subprocess.Popen(args, stdout = subprocess.PIPE, stderr= subprocess.PIPE).communicate() #, shell=True logger.info('R outlog info:\n %s ' % output) logger.debug('R outlog errors:\n %s ' % error) if len(output) > 0: self.status.set('**** weatherregime in R suceeded', 90) else: logger.error('NO! output returned from R call') except Exception as e: msg = 'weatherregime in R %s ' % e logger.error(msg) raise Exception(msg) ############################################ ### set the outputs ############################################ self.Routput_graphic.setValue( output_graphics ) self.output_pca.setValue( file_pca ) self.output_classification.setValue( file_class ) self.output_netcdf.setValue( model_season )
def method_A(resource=[], start=None, end=None, timeslice=20, variable=None, title=None, cmap='seismic' ): """returns the result :param resource: list of paths to netCDF files :param start: beginning of reference period (if None (default), the first year of the consistent ensemble will be detected) :param end: end of comparison period (if None (default), the last year of the consistent ensemble will be detected) :param timeslice: period length for mean calculation of reference and comparison period :param variable: variable name to be detected in the netCDF file. If not set (not recommended), the variable name will be detected :param title: str to be used as title for the signal mal :param cmap: define the color scheme for signal map plotting :return: signal.nc, low_agreement_mask.nc, high_agreement_mask.nc, graphic.png, text.txt """ from os.path import split from cdo import Cdo cdo = Cdo() cdo.forceOutput = True try: # preparing the resource # from flyingpigeon.ocgis_module import call file_dic = sort_by_filename(resource, historical_concatination = True) #print file_dic logger.info('file names sorted experimets: %s' % len(file_dic.keys())) except Exception as e: msg = 'failed to sort the input files' logger.exception(msg) raise Exception(msg) try: mergefiles = [] for key in file_dic.keys(): if type(file_dic[key]) == list and len(file_dic[key]) > 1: input = [] for i in file_dic[key]: print i input.extend([i.replace(' ','\\\ ')]) mergefiles.append(cdo.mergetime(input=input, output=key+'_mergetime.nc')) else: mergefiles.extend(file_dic[key]) # files.append(cdo.selyear('%s/%s' % (start1,end2), input = tmpfile , output = key+'.nc' )) #python version logger.info('datasets merged %s ' % mergefiles) except Exception as e: msg = 'seltime and mergetime failed %s' % e logger.exception(msg) raise Exception(e) try: text_src = open('infiles.txt', 'a') for key in file_dic.keys(): text_src.write(key + '\n') text_src.close() except Exception as e: msg = 'failed to write source textfile' logger.exception(msg) raise Exception(msg) # configure reference and compare period try: if start == None: st_set = set() en_set = set() for f in mergefiles: print f times = get_time(f) st_set.update([times[0].year]) if end == None: en_set.update([times[-1].year]) start = max(st_set) if end == None: end = min(en_set) logger.info('Start and End: %s - %s ' % (start, end)) if start >= end: logger.error('ensemble is inconsistent!!! start year is later than end year') except Exception as e: msg = 'failed to detect start and end times of the ensemble' logger.exception(msg) raise Exception(msg) # set the periodes: try: start = int(start) end = int(end) if timeslice == None: timeslice = int((end - start) / 3) if timeslice == 0: timeslice = 1 else: timeslice = int(timeslice) start1 = start start2 = start1 + timeslice - 1 end1 = end - timeslice + 1 end2 = end logger.info('timeslice and periodes set') except Exception as e: msg = 'failed to set the periodes' logger.exception(msg) raise Exception(msg) try: files = [] for i, mf in enumerate(mergefiles): files.append(cdo.selyear('{0}/{1}'.format(start1,end2), input=[mf.replace(' ','\ ')] , output='file_{0}_.nc'.format(i) )) #python version logger.info('timeseries selected from defined start to end year') except Exception as e: msg = 'seltime and mergetime failed' logger.exception(msg) raise Exception(msg) try: # ensemble mean nc_ensmean = cdo.ensmean(input=files , output='nc_ensmean.nc') logger.info('ensemble mean calculation done') except Exception as e: msg = 'ensemble mean failed' logger.exception(msg) raise Exception(msg) try: # ensemble std nc_ensstd = cdo.ensstd(input=files , output='nc_ensstd.nc') logger.info('ensemble std and calculation done') except Exception as e: msg = 'ensemble std or failed' logger.exception(msg) raise Exception(msg) # get the get the signal as difference from the beginning (first years) and end period (last years), : try: selyearstart = cdo.selyear('%s/%s' % (start1,start2), input = nc_ensmean, output = 'selyearstart.nc' ) selyearend = cdo.selyear('%s/%s' % (end1,end2), input = nc_ensmean, output = 'selyearend.nc' ) meanyearst = cdo.timmean(input = selyearstart, output= 'meanyearst.nc') meanyearend = cdo.timmean(input = selyearend, output= 'meanyearend.nc') signal = cdo.sub(input=[meanyearend, meanyearst], output = 'signal.nc') logger.info('Signal calculation done') except Exception as e: msg = 'calculation of signal failed' logger.exception(msg) raise Exception(msg) # get the intermodel standard deviation (mean over whole period) try: #std_selyear = cdo.selyear('%s/%s' % (end1,end2), input=nc_ensstd, output='std_selyear.nc') #std = cdo.timmean(input = std_selyear, output = 'std.nc') std = cdo.timmean(input = nc_ensstd, output = 'std.nc') std2 = cdo.mulc('2', input = std, output = 'std2.nc') logger.info('calculation of internal model std for time period done') except Exception as e: msg = 'calculation of internal model std failed' logger.exception(msg) raise Exception(msg) try: absolut = cdo.abs(input=signal, output='absolut_signal.nc') high_agreement_mask = cdo.gt(input=[absolut,std2], output= 'large_change_with_high_model_agreement.nc') low_agreement_mask = cdo.lt(input=[absolut,std], output= 'small_signal_or_low_agreement_of_models.nc') logger.info('high and low mask done') except Exception as e: msg = 'calculation of robustness mask failed' logger.exception(msg) raise Exception(msg) try: if variable == None: variable = get_variable(signal) logger.info('variable to be plotted: %s' % variable) if title == None: title='Change of %s (difference of mean %s-%s to %s-%s)' % (variable, end1, end2, start1, start2) graphic = None graphic = map_ensembleRobustness(signal, high_agreement_mask, low_agreement_mask, variable=variable, cmap=cmap, title = title) logger.info('graphic generated') except Exception as e: msg('graphic generation failed: %s' % e) logger.debug(msg) raise Exception(msg) return signal, low_agreement_mask, high_agreement_mask, graphic, text_src #
def test_get_variable(): variable = utils.get_variable(local_path(TESTDATA["cmip5_tasmax_2007_nc"])) assert "tasmax" == variable variable = utils.get_variable(local_path(TESTDATA["cordex_tasmax_2007_nc"])) assert "tasmax" == variable
def get_gam(ncs_reference, PAmask, modelname=None): """ GAM statistical training based on presence/absence mask and indices :param ncs_reference: list of netCDF files containing the indices :param PAmask: presence/absence mask as output from get_PAmask :param modelname: modelname to be used for potting :return gam_model, prediction, infos_concat: Rstatisics, occurence predicion based on ncs_reference files, graphical visualisation of regression curves """ try: from netCDF4 import Dataset from os.path import basename from numpy import squeeze, ravel, isnan, nan, array, reshape from flyingpigeon.utils import get_variable from rpy2.robjects.packages import importr import rpy2.robjects as ro import rpy2.robjects.numpy2ri rpy2.robjects.numpy2ri.activate() base = importr("base") stats = importr("stats") mgcv = importr("mgcv") logger.info('rpy2 modules imported') except: msg = 'failed to import rpy2 modules' logger.exception(msg) raise try: data = {'PA': ro.FloatVector(ravel(PAmask))} domain = PAmask.shape logger.info('mask data converted to R float vector') except: msg = 'failed to convert mask to R vector' logger.exception(msg) raise Exception try: form = 'PA ~ ' ncs_reference.sort() for i, nc in enumerate(ncs_reference): var = get_variable(nc) agg = basename(nc).split('_')[-2] ds = Dataset(nc) vals = squeeze(ds.variables[var]) # vals[vals > 1000] = 0 vals[isnan(PAmask)] = nan indice = '%s_%s' % (var, agg) data[str(indice)] = ro.FloatVector(ravel(vals)) if i == 0: form = form + 's(%s, k=3)' % indice else: form = form + ' + s(%s, k=3)' % indice logger.info('form string generated for gam model') except: msg = 'form string generation for gam failed' logger.exception(msg) # raise Exception try: dataf = ro.DataFrame(data) eq = ro.Formula(str(form)) gam_model = mgcv.gam(base.eval(eq), data=dataf, family=stats.binomial(), scale=-1, na_action=stats.na_exclude) logger.info('GAM model trained') except: msg = 'failed to train the GAM model' logger.exception(msg) # #################### # plot response curves # #################### try: try: from tempfile import mkstemp grdevices = importr('grDevices') ip, statinfos = mkstemp(dir='.', suffix='.pdf') grdevices.pdf(file=statinfos) for i in range(1, len(ncs_reference) + 1): try: trans = ro.r('function(x){exp(x)/(1+exp(x))}') _ = mgcv.plot_gam(gam_model, trans=trans, shade='T', col='black', select=i, ylab='Predicted Probability', main=modelname, rug=False, cex_lab=1.4, cex_axis=4.2) logger.info('plot GAM curves for %s.', i) except: logger.exception('failed to plot GAM curves for %s.', i) _ = grdevices.dev_off() except: logger.exception('GAM plot failedin SDM process') try: predict_gam = mgcv.predict_gam(gam_model, type="response", progress="text", na_action=stats.na_pass) prediction = array(predict_gam).reshape(domain) logger.info('SDM prediction for reference period processed') except: logger.exception('failed to process SDM prediction') prediction = None except: logger.exception('failed to plot GAM curves') _, infos_concat = mkstemp(dir='.', suffix='.pdf') return gam_model, prediction, statinfos
def get_gam(ncs_reference, PAmask): from netCDF4 import Dataset from os.path import basename from numpy import squeeze, ravel, isnan, nan, array, reshape from flyingpigeon.utils import get_variable try: from rpy2.robjects.packages import importr import rpy2.robjects as ro import rpy2.robjects.numpy2ri rpy2.robjects.numpy2ri.activate() base = importr("base") stats = importr("stats") mgcv = importr("mgcv") logger.info('rpy2 modules imported') except Exception as e: msg = 'failed to import rpy2 modules %s' % e logger.debug(msg) raise Exception(msg) try: data = {'PA': ro.FloatVector(ravel(PAmask))} domain = PAmask.shape logger.info('mask data converted to R float vector') except Exception as e: msg = 'failed to convert mask to R vector' form = 'PA ~ ' ncs_reference.sort() try: for i, nc in enumerate(ncs_reference): var = get_variable(nc) agg = basename(nc).split('_')[-2] ds = Dataset(nc) vals = squeeze(ds.variables[var]) vals[vals > 1000] = 0 vals[isnan(PAmask)] = nan indice = '%s_%s' % (var, agg) data[str(indice)] = ro.FloatVector(ravel(vals)) if i == 0: form = form + 's(%s, k=3)' % indice else: form = form + ' + s(%s, k=3)' % indice except Exception as e: logger.debug('form string generation for gam failed') dataf = ro.DataFrame(data) eq = ro.Formula(str(form)) gam_model = mgcv.gam(base.eval(eq), data=dataf, family=stats.binomial(), scale=-1, na_action=stats.na_exclude) # grdevices = importr('grDevices') ### ########################### # plot response curves ### ########################### from flyingpigeon.visualisation import concat_images from tempfile import mkstemp infos = [] for i in range(1, len(ncs_reference) + 1): #ip, info = mkstemp(dir='.',suffix='.pdf') ip, info = mkstemp(dir='.', suffix='.png') infos.append(info) grdevices.png(filename=info) #grdevices.pdf(filename=info) #ylim = ro.IntVector([-6,6]) trans = ro.r('function(x){exp(x)/(1+exp(x))}') mgcv.plot_gam( gam_model, trans=trans, shade='T', col='black', select=i, ylab='Predicted Probability', rug=False, cex_lab=1.4, cex_axis=1.4, ) # #ylim=ylim, , grdevices.dev_off() infos_concat = concat_images(infos, orientation='h') predict_gam = mgcv.predict_gam(gam_model, type="response", progress="text", na_action=stats.na_exclude) #, prediction = array(predict_gam).reshape(domain) return gam_model, prediction, infos_concat
def set_metadata_segetalflora(resource): """ :param resources: imput files """ # gather the set_metadata dic_segetalflora = { 'keywords' : 'Segetalflora', 'tier': '2', 'in_var' : 'tas', 'description':'Number of European segetalflora species', 'method':'regression equation', 'institution':'Julius Kuehn-Institut (JKI) Federal Research Centre for Cultivated Plants', 'institution_url':'www.jki.bund.de', 'institute_id' : "JKI", 'contact_mail_3':'*****@*****.**', 'version' : '1.0', } dic_climatetype = { '1' : 'cold northern species group', '2' : 'warm northern species group', '3' : 'moderate warm-toned species group', '4' : 'moderate warm-toned to mediterranean species group', '5' : 'mediterranean species group', '6' : 'climate-indifferent species', '7' : 'climate-undefinable species', 'all' : 'species of all climate types' } try: set_basic_md(resource) except Exception as e: LOGGER.error(e) try: set_dynamic_md(resource) except Exception as e: LOGGER.error(e) #set the segetalflora specific metadata try: ds = Dataset(resource, mode='a') ds.setncatts(dic_segetalflora) ds.close() except Exception as e: LOGGER.error(e) # set the variable attributes: from flyingpigeon.utils import get_variable try: ds = Dataset(resource, mode='a') var = get_variable(resource) if 'all' in var: climat_type = 'all' else: climat_type = var[-1] culture_type = var.strip('sf').strip(climat_type) sf = ds.variables[var] sf.setncattr('units',1) sf.setncattr('standard_name', 'sf%s%s' % (culture_type, climat_type)) sf.setncattr('long_name', 'Segetal flora %s land use for %s' % (culture_type, dic_climatetype['%s' % climat_type])) ds.close() except Exception as e: LOGGER.error('failed to set sf attributes %s ' % e) # sort the attributes: try: ds = Dataset(resource, mode='a') att = ds.ncattrs() att.sort() for a in att: entry = ds.getncattr(a) ds.setncattr(a,entry) history = '%s , Segetalflora Impact Model V1.0' % (ds.history) ds.setncattr('history',history) ds.close() except Exception as e: LOGGER.error('failed to sort attributes %s ' % e) return resource
def execute(self): logger.info('Start process') from datetime import datetime as dt from flyingpigeon import weatherregimes as wr from tempfile import mkstemp ################################ # reading in the input arguments ################################ try: resource = self.getInputValues(identifier='resource') url_Rdat = self.getInputValues(identifier='Rdat')[0] url_dat = self.getInputValues(identifier='dat')[0] url_ref_file = self.getInputValues(identifier='netCDF') # can be None season = self.getInputValues(identifier='season')[0] period = self.getInputValues(identifier='period')[0] anualcycle = self.getInputValues(identifier='anualcycle')[0] except Exception as e: logger.debug('failed to read in the arguments %s ' % e) try: start = dt.strptime(period.split('-')[0] , '%Y%m%d') end = dt.strptime(period.split('-')[1] , '%Y%m%d') # kappa = int(self.getInputValues(identifier='kappa')[0]) logger.info('period %s' % str(period)) logger.info('season %s' % str(season)) logger.info('read in the arguments') logger.info('url_ref_file: %s' % url_ref_file) logger.info('url_Rdat: %s' % url_Rdat) logger.info('url_dat: %s' % url_dat) except Exception as e: logger.debug('failed to convert arguments %s ' % e) ############################ # fetching trainging data ############################ from flyingpigeon.utils import download, get_time from os.path import abspath try: dat = abspath(download(url_dat)) Rdat = abspath(download(url_Rdat)) logger.info('training data fetched') except Exception as e: logger.error('failed to fetch training data %s' % e) ############################################################ ### get the required bbox and time region from resource data ############################################################ # from flyingpigeon.weatherregimes import get_level from flyingpigeon.ocgis_module import call from flyingpigeon.utils import get_variable time_range = [start, end] variable = get_variable(resource) if len(url_ref_file) > 0: ref_file = download(url_ref_file[0]) model_subset = call(resource=resource, variable=variable, time_range=time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap', regrid_destination=ref_file, regrid_options='bil') logger.info('Dataset subset with regridding done: %s ' % model_subset) else: model_subset = call(resource=resource, variable=variable, time_range=time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap', ) logger.info('Dataset time period extracted: %s ' % model_subset) ############################################## ### computing anomalies ############################################## cycst = anualcycle.split('-')[0] cycen = anualcycle.split('-')[0] reference = [dt.strptime(cycst,'%Y%m%d'), dt.strptime(cycen,'%Y%m%d')] model_anomal = wr.get_anomalies(model_subset, reference=reference) ##################### ### extracting season ##################### model_season = wr.get_season(model_anomal, season=season) ####################### ### call the R scripts ####################### import shlex import subprocess from flyingpigeon import config from os.path import curdir, exists, join try: rworkspace = curdir Rsrc = config.Rsrc_dir() Rfile = 'weatherregimes_projection.R' yr1 = start.year yr2 = end.year time = get_time(model_season, format='%Y%m%d') #ip, output_graphics = mkstemp(dir=curdir ,suffix='.pdf') ip, file_pca = mkstemp(dir=curdir ,suffix='.txt') ip, file_class = mkstemp(dir=curdir ,suffix='.Rdat') ip, output_frec = mkstemp(dir=curdir ,suffix='.txt') args = ['Rscript', join(Rsrc,Rfile), '%s/' % curdir, '%s/' % Rsrc, '%s' % model_season, '%s' % variable, '%s' % str(time).strip("[]").replace("'","").replace(" ",""), # '%s' % output_graphics, '%s' % dat, '%s' % Rdat, '%s' % file_pca, '%s' % file_class, '%s' % output_frec, '%s' % season, '%s' % start.year, '%s' % end.year, '%s' % 'MODEL'] logger.info('Rcall builded') except Exception as e: msg = 'failed to build the R command %s' % e logger.error(msg) raise Exception(msg) try: output,error = subprocess.Popen(args, stdout = subprocess.PIPE, stderr= subprocess.PIPE).communicate() #, shell=True logger.info('R outlog info:\n %s ' % output) logger.debug('R outlog errors:\n %s ' % error) if len(output) > 0: self.status.set('**** weatherregime in R suceeded', 90) else: logger.error('NO! output returned from R call') except Exception as e: msg = 'weatherregime in R %s ' % e logger.error(msg) raise Exception(msg) ############################################ ### set the outputs ############################################ #self.Routput_graphic.setValue( output_graphics ) self.output_pca.setValue( file_pca ) self.output_classification.setValue( file_class ) self.output_netcdf.setValue( model_season ) self.output_frequency.setValue( output_frec )
def calc_indice_unconventional(resource=[], variable=None, prefix=None, indices=None, polygons=None, groupings=None, dir_output=None, dimension_map = None): """ Calculates given indices for suitable files in the appopriate time grouping and polygon. :param resource: list of filenames in drs convention (netcdf) :param variable: variable name to be selected in the in netcdf file (default=None) :param indices: list of indices (default ='TGx') :param polygons: list of polgons (default =None) :param grouping: indices time aggregation (default='yr') :param out_dir: output directory for result file (netcdf) :param dimension_map: optional dimension map if different to standard (default=None) :return: list of netcdf files with calculated indices. Files are saved into dir_output """ from os.path import join, dirname, exists from os import remove import uuid from flyingpigeon import ocgis_module from flyingpigeon.subset import get_ugid, get_geom if type(resource) != list: resource = list([resource]) if type(indices) != list: indices = list([indices]) if type(polygons) != list and polygons != None: polygons = list([polygons]) elif polygons == None: polygons = [None] else: logger.error('Polygons not found') if type(groupings) != list: groupings = list([groupings]) if dir_output != None: if not exists(dir_output): makedirs(dir_output) experiments = sort_by_filename(resource) outputs = [] # print('environment for calc_indice_unconventional set') logger.info('environment for calc_indice_unconventional set') for key in experiments: if variable == None: variable = get_variable(experiments[key][0]) try: ncs = experiments[key] for indice in indices: logger.info('indice: %s' % indice) try: for grouping in groupings: logger.info('grouping: %s' % grouping) try: calc_group = calc_grouping(grouping) logger.info('calc_group: %s' % calc_group) for polygon in polygons: try: domain = key.split('_')[1].split('-')[0] if polygon == None: if prefix == None: prefix = key.replace(variable, indice).replace('_day_','_%s_' % grouping ) geom = None ugid = None else: if prefix == None: prefix = key.replace(variable, indice).replace('_day_','_%s_' % grouping ).replace(domain,polygon) geom = get_geom(polygon=polygon) ugid = get_ugid(polygons=polygon, geom=geom) if indice == 'TGx': calc=[{'func': 'max', 'name': 'TGx'}] tmp = ocgis_module.call(resource=ncs,# conform_units_to='celcius', variable=variable, dimension_map=dimension_map, calc=calc, calc_grouping=calc_group, prefix=prefix, dir_output=dir_output, geom=geom, select_ugid=ugid) elif indice == 'TGn': calc=[{'func': 'min', 'name': 'TGn'}] tmp = ocgis_module.call(resource=ncs, #conform_units_to='celcius', variable=variable, dimension_map=dimension_map, calc=calc, calc_grouping= calc_group, prefix=prefix, dir_output=dir_output, geom=geom, select_ugid = ugid) elif indice == 'TGx5day': calc = [{'func': 'moving_window', 'name': 'TGx5day', 'kwds': {'k': 5, 'operation': 'mean', 'mode': 'same' }}] tmp2 = ocgis_module.call(resource=ncs, #conform_units_to='celcius', variable=variable, dimension_map=dimension_map, calc=calc, prefix=str(uuid.uuid4()), geom=geom, select_ugid = ugid) calc=[{'func': 'max', 'name': 'TGx5day'}] logger.info('moving window calculated : %s' % tmp2) tmp = ocgis_module.call(resource=tmp2, variable=indice, dimension_map=dimension_map, calc=calc, calc_grouping=calc_group, prefix=prefix, dir_output=dir_output) remove(tmp2) elif indice == 'TGn5day': calc = [{'func': 'moving_window', 'name': 'TGn5day', 'kwds': {'k': 5, 'operation': 'mean', 'mode': 'same' }}] tmp2 = ocgis_module.call(resource=ncs, #conform_units_to='celcius', variable=variable, dimension_map=dimension_map, calc=calc, prefix=str(uuid.uuid4()), geom=geom, select_ugid = ugid) calc=[{'func': 'min', 'name': 'TGn5day'}] logger.info('moving window calculated : %s' % tmp2) tmp = ocgis_module.call(resource=tmp2, variable=indice, dimension_map=dimension_map, calc=calc, calc_grouping=calc_group, prefix=prefix, dir_output=dir_output) remove(tmp2) else: logger.error('Indice %s is not a known inidce' % (indice)) outputs.append(tmp) logger.info('indice file calcualted %s ' % (tmp)) except Exception as e: logger.exception('could not calc indice %s for key %s, polygon %s and calc_grouping %s : %s' % (indice, key, polygon, grouping, e )) except Exception as e: logger.exception('could not calc indice %s for key %s and calc_grouping %s : %s' % ( indice, key, polygon, e )) except Exception as e: logger.exception('could not calc indice %s for key %s: %s'% (indice, key, e )) except Exception as e: logger.exception('could not calc key %s: %s' % (key, e)) return outputs
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' LOGGER.info('Start process') response.update_status('execution started at : {}'.format(dt.now()), 5) process_start_time = time.time() # measure process execution time ... start_time = time.time() # measure init ... ################################ # reading in the input arguments ################################ response.update_status('execution started at : %s ' % dt.now(), 5) start_time = time.time() # measure init ... ################################ # reading in the input arguments ################################ try: response.update_status('read input parameter : %s ' % dt.now(), 5) resource = archiveextract(resource=rename_complexinputs(request.inputs['resource'])) refSt = request.inputs['refSt'][0].data refEn = request.inputs['refEn'][0].data dateSt = request.inputs['dateSt'][0].data dateEn = request.inputs['dateEn'][0].data seasonwin = request.inputs['seasonwin'][0].data nanalog = request.inputs['nanalog'][0].data # bbox = [-80, 20, 50, 70] # TODO: Add checking for wrong cordinates and apply default if nesessary #level = 500 level = request.inputs['level'][0].data if (level == 500): dummylevel = 1000 # dummy workaround for cdo sellevel else: dummylevel = 500 LOGGER.debug('LEVEL selected: %s hPa' % (level)) bbox=[] bboxStr = request.inputs['BBox'][0].data bboxStr = bboxStr.split(',') #for i in bboxStr: bbox.append(int(i)) bbox.append(float(bboxStr[0])) bbox.append(float(bboxStr[2])) bbox.append(float(bboxStr[1])) bbox.append(float(bboxStr[3])) LOGGER.debug('BBOX for ocgis: %s ' % (bbox)) LOGGER.debug('BBOX original: %s ' % (bboxStr)) # if bbox_obj is not None: # LOGGER.info("bbox_obj={0}".format(bbox_obj.coords)) # bbox = [bbox_obj.coords[0][0], # bbox_obj.coords[0][1], # bbox_obj.coords[1][0], # bbox_obj.coords[1][1]] # LOGGER.info("bbox={0}".format(bbox)) # else: # bbox = None # region = self.getInputValues(identifier='region')[0] # bbox = [float(b) for b in region.split(',')] # bbox_obj = self.BBox.getValue() normalize = request.inputs['normalize'][0].data distance = request.inputs['dist'][0].data outformat = request.inputs['outformat'][0].data timewin = request.inputs['timewin'][0].data # model_var = request.inputs['reanalyses'][0].data # model, var = model_var.split('_') # experiment = self.getInputValues(identifier='experiment')[0] # dataset, var = experiment.split('_') # LOGGER.info('environment set') LOGGER.info('input parameters set') response.update_status('Read in and convert the arguments', 5) except Exception as e: msg = 'failed to read input prameter %s ' % e LOGGER.error(msg) raise Exception(msg) ###################################### # convert types and set environment ###################################### try: # refSt = dt.strptime(refSt[0], '%Y-%m-%d') # refEn = dt.strptime(refEn[0], '%Y-%m-%d') # dateSt = dt.strptime(dateSt[0], '%Y-%m-%d') # dateEn = dt.strptime(dateEn[0], '%Y-%m-%d') #not nesessary if fix ocgis_module.py refSt = dt.combine(refSt,dt_time(12,0)) refEn = dt.combine(refEn,dt_time(12,0)) dateSt = dt.combine(dateSt,dt_time(12,0)) dateEn = dt.combine(dateEn,dt_time(12,0)) # refSt = refSt.replace(hour=12) # refEn = refEn.replace(hour=12) # dateSt = dateSt.replace(hour=12) # dateEn = dateEn.replace(hour=12) if normalize == 'None': seacyc = False else: seacyc = True if outformat == 'ascii': outformat = '.txt' elif outformat == 'netCDF': outformat = '.nc' else: LOGGER.error('output format not valid') start = min(refSt, dateSt) end = max(refEn, dateEn) # if bbox_obj is not None: # LOGGER.info("bbox_obj={0}".format(bbox_obj.coords)) # bbox = [bbox_obj.coords[0][0], # bbox_obj.coords[0][1], # bbox_obj.coords[1][0], # bbox_obj.coords[1][1]] # LOGGER.info("bbox={0}".format(bbox)) # else: # bbox = None LOGGER.info('environment set') except Exception as e: msg = 'failed to set environment %s ' % e LOGGER.error(msg) raise Exception(msg) LOGGER.debug("init took %s seconds.", time.time() - start_time) response.update_status('Read in and convert the arguments', 5) ######################## # input data preperation ######################## # TODO: Check if files containing more than one dataset response.update_status('Start preparing input data', 12) start_time = time.time() # mesure data preperation ... try: # TODO: Add selection of the level. maybe bellow in call(..., level_range=[...,...]) if type(resource) == list: #resource.sort() resource = sorted(resource, key=lambda i: path.splitext(path.basename(i))[0]) else: resource=[resource] #=============================================================== # TODO: REMOVE resources which are out of interest from the list # (years > and < than requested for calculation) tmp_resource = [] for re in resource: s,e = get_timerange(re) tmpSt = dt.strptime(s,'%Y%m%d') tmpEn = dt.strptime(e,'%Y%m%d') if ((tmpSt <= end ) and (tmpEn >= start)): tmp_resource.append(re) LOGGER.debug('Selected file: %s ' % (re)) resource = tmp_resource # =============================================================== #================================================================ # Try to fix memory issue... (ocgis call for files like 20-30 gb... ) # IF 4D - select pressure level before domain cut # # resource properties ds = Dataset(resource[0]) variable = get_variable(resource[0]) var = ds.variables[variable] dims = list(var.dimensions) dimlen = len(dims) try: model_id = ds.getncattr('model_id') except AttributeError: model_id = 'Unknown model' LOGGER.debug('MODEL: %s ' % (model_id)) lev_units = 'hPa' if (dimlen>3) : lev = ds.variables[dims[1]] # actually index [1] need to be detected... assuming zg(time, plev, lat, lon) lev_units = lev.units if (lev_units=='Pa'): level = level*100 dummylevel=dummylevel*100 # TODO: OR check the NAME and units of vertical level and find 200 , 300, or 500 mbar in it # Not just level = level * 100. # Get Levels from cdo import Cdo cdo = Cdo() lev_res=[] if(dimlen>3): for res_fn in resource: tmp_f = 'lev_' + path.basename(res_fn) comcdo = '%s,%s' % (level,dummylevel) cdo.sellevel(comcdo, input=res_fn, output=tmp_f) lev_res.append(tmp_f) else: lev_res = resource # Get domain regr_res=[] for res_fn in lev_res: tmp_f = 'dom_' + path.basename(res_fn) comcdo = '%s,%s,%s,%s' % (bbox[0],bbox[2],bbox[1],bbox[3]) cdo.sellonlatbox(comcdo, input=res_fn, output=tmp_f) regr_res.append(tmp_f) #archive_tmp = call(resource=resource, time_range=[refSt, refEn], geom=bbox, spatial_wrapping='wrap') #simulation_tmp = call(resource=resource, time_range=[dateSt, dateEn], geom=bbox, spatial_wrapping='wrap') #============================ archive_tmp = call(resource=regr_res, time_range=[refSt, refEn], spatial_wrapping='wrap') simulation_tmp = call(resource=regr_res, time_range=[dateSt, dateEn], spatial_wrapping='wrap') ####################################################################################### # TEMORAL dirty workaround to get the level and it's units - will be func in utils.py if (dimlen>3) : archive = get_level(archive_tmp, level = level) simulation = get_level(simulation_tmp,level = level) variable = 'z%s' % level # TODO: here should be modulated else: archive = archive_tmp simulation = simulation_tmp # 3D, move forward ####################################################################################### if seacyc is True: seasoncyc_base, seasoncyc_sim = analogs.seacyc(archive, simulation, method=normalize) else: seasoncyc_base = None seasoncyc_sim = None except Exception as e: msg = 'failed to prepare archive and simulation files %s ' % e LOGGER.debug(msg) raise Exception(msg) ip, output = mkstemp(dir='.', suffix='.txt') output_file = path.abspath(output) files = [path.abspath(archive), path.abspath(simulation), output_file] LOGGER.debug("data preperation took %s seconds.", time.time() - start_time) ############################ # generating the config file ############################ # TODO: add MODEL name as argument response.update_status('writing config file', 15) start_time = time.time() # measure write config ... try: config_file = analogs.get_configfile( files=files, seasoncyc_base=seasoncyc_base, seasoncyc_sim=seasoncyc_sim, base_id=model_id, sim_id=model_id, timewin=timewin, varname=variable, seacyc=seacyc, cycsmooth=91, nanalog=nanalog, seasonwin=seasonwin, distfun=distance, outformat=outformat, calccor=True, silent=False, period=[dt.strftime(refSt, '%Y-%m-%d'), dt.strftime(refEn, '%Y-%m-%d')], bbox="%s,%s,%s,%s" % (bbox[0], bbox[2], bbox[1], bbox[3])) except Exception as e: msg = 'failed to generate config file %s ' % e LOGGER.debug(msg) raise Exception(msg) LOGGER.debug("write_config took %s seconds.", time.time() - start_time) ############## # CASTf90 call ############## import subprocess import shlex start_time = time.time() # measure call castf90 response.update_status('Start CASTf90 call', 20) try: # response.update_status('execution of CASTf90', 50) cmd = 'analogue.out %s' % path.relpath(config_file) # system(cmd) args = shlex.split(cmd) output, error = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() LOGGER.info('analogue.out info:\n %s ' % output) LOGGER.debug('analogue.out errors:\n %s ' % error) response.update_status('**** CASTf90 suceeded', 70) except Exception as e: msg = 'CASTf90 failed %s ' % e LOGGER.error(msg) raise Exception(msg) LOGGER.debug("castf90 took %s seconds.", time.time() - start_time) response.update_status('preparing output', 70) response.outputs['config'].file = config_file #config_output_url # config_file ) response.outputs['analogs'].file = output_file response.outputs['output_netcdf'].file = simulation ######################## # generate analog viewer ######################## formated_analogs_file = analogs.reformat_analogs(output_file) # response.outputs['formated_analogs'].storage = FileStorage() response.outputs['formated_analogs'].file = formated_analogs_file LOGGER.info('analogs reformated') response.update_status('reformatted analog file', 80) viewer_html = analogs.render_viewer( # configfile=response.outputs['config'].get_url(), configfile=config_file, # datafile=response.outputs['formated_analogs'].get_url()) datafile=formated_analogs_file) response.outputs['output'].file = viewer_html response.update_status('Successfully generated analogs viewer', 90) LOGGER.info('rendered pages: %s ', viewer_html) response.update_status('execution ended', 100) LOGGER.debug("total execution took %s seconds.", time.time() - process_start_time) return response
def get_gam(ncs_reference, PAmask): from netCDF4 import Dataset from os.path import basename from numpy import squeeze, ravel, isnan, nan, array, reshape from flyingpigeon.utils import get_variable try: from rpy2.robjects.packages import importr import rpy2.robjects as ro import rpy2.robjects.numpy2ri rpy2.robjects.numpy2ri.activate() base = importr("base") stats = importr("stats") mgcv = importr("mgcv") logger.info('rpy2 modules imported') except Exception as e: msg = 'failed to import rpy2 modules %s' % e logger.debug(msg) raise Exception(msg) try: data = {'PA': ro.FloatVector(ravel(PAmask))} domain = PAmask.shape logger.info('mask data converted to R float vector') except Exception as e: msg = 'failed to convert mask to R vector' form = 'PA ~ ' ncs_reference.sort() try: for i , nc in enumerate(ncs_reference): var = get_variable(nc) agg = basename(nc).split('_')[-2] ds = Dataset(nc) vals = squeeze(ds.variables[var]) vals[vals > 1000 ] = 0 vals[isnan(PAmask)] = nan indice = '%s_%s' % (var, agg) data[str(indice)] = ro.FloatVector(ravel(vals)) if i == 0: form = form + 's(%s, k=3)' % indice else: form = form + ' + s(%s, k=3)' % indice except Exception as e: logger.debug('form string generation for gam failed') dataf = ro.DataFrame(data) eq = ro.Formula(str(form)) gam_model = mgcv.gam(base.eval(eq), data=dataf, family=stats.binomial(), scale=-1, na_action=stats.na_exclude) # grdevices = importr('grDevices') ### ########################### # plot response curves ### ########################### from flyingpigeon.visualisation import concat_images from tempfile import mkstemp infos = [] for i in range(1,len(ncs_reference)+1): #ip, info = mkstemp(dir='.',suffix='.pdf') ip, info = mkstemp(dir='.',suffix='.png') infos.append(info) grdevices.png(filename=info) #grdevices.pdf(filename=info) #ylim = ro.IntVector([-6,6]) trans = ro.r('function(x){exp(x)/(1+exp(x))}') mgcv.plot_gam(gam_model, trans=trans, shade='T', col='black',select=i,ylab='Predicted Probability',rug=False , cex_lab = 1.4, cex_axis = 1.4, ) # #ylim=ylim, , grdevices.dev_off() infos_concat = concat_images(infos, orientation='h') predict_gam = mgcv.predict_gam(gam_model, type="response", progress="text", na_action=stats.na_exclude) #, prediction = array(predict_gam).reshape(domain) return gam_model, prediction, infos_concat
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' process_start_time = time.time() # measure process execution time ... response.update_status('execution started at : %s ' % dt.now(), 5) start_time = time.time() # measure init ... resource = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) refSt = request.inputs['refSt'][0].data refEn = request.inputs['refEn'][0].data dateSt = request.inputs['dateSt'][0].data dateEn = request.inputs['dateEn'][0].data regrset = request.inputs['regrset'][0].data # fix 31 December issue # refSt = dt.combine(refSt,dt_time(12,0)) # refEn = dt.combine(refEn,dt_time(12,0)) # dateSt = dt.combine(dateSt,dt_time(12,0)) # dateEn = dt.combine(dateEn,dt_time(12,0)) seasonwin = request.inputs['seasonwin'][0].data nanalog = request.inputs['nanalog'][0].data # bbox = [-80, 20, 50, 70] # TODO: Add checking for wrong cordinates and apply default if nesessary bbox = [] bboxStr = request.inputs['BBox'][0].data bboxStr = bboxStr.split(',') bbox.append(float(bboxStr[0])) bbox.append(float(bboxStr[2])) bbox.append(float(bboxStr[1])) bbox.append(float(bboxStr[3])) direction = request.inputs['direction'][0].data normalize = request.inputs['normalize'][0].data distance = request.inputs['dist'][0].data outformat = request.inputs['outformat'][0].data timewin = request.inputs['timewin'][0].data model_var = request.inputs['reanalyses'][0].data model, var = model_var.split('_') try: if direction == 're2mo': anaSt = dt.combine(dateSt, dt_time( 0, 0)) #dt.strptime(dateSt[0], '%Y-%m-%d') anaEn = dt.combine(dateEn, dt_time( 0, 0)) #dt.strptime(dateEn[0], '%Y-%m-%d') refSt = dt.combine(refSt, dt_time( 12, 0)) #dt.strptime(refSt[0], '%Y-%m-%d') refEn = dt.combine(refEn, dt_time( 12, 0)) #dt.strptime(refEn[0], '%Y-%m-%d') r_time_range = [anaSt, anaEn] m_time_range = [refSt, refEn] elif direction == 'mo2re': anaSt = dt.combine(dateSt, dt_time( 12, 0)) #dt.strptime(refSt[0], '%Y-%m-%d') anaEn = dt.combine(dateEn, dt_time( 12, 0)) #dt.strptime(refEn[0], '%Y-%m-%d') refSt = dt.combine(refSt, dt_time( 0, 0)) #dt.strptime(dateSt[0], '%Y-%m-%d') refEn = dt.combine(refEn, dt_time( 0, 0)) #dt.strptime(dateEn[0], '%Y-%m-%d') r_time_range = [refSt, refEn] m_time_range = [anaSt, anaEn] else: LOGGER.exception( 'failed to find time periods for comparison direction') except: msg = 'failed to put simulation and reference time in order' LOGGER.exception(msg) raise Exception(msg) if normalize == 'None': seacyc = False else: seacyc = True if outformat == 'ascii': outformat = '.txt' elif outformat == 'netCDF': outformat = '.nc' else: LOGGER.exception('output format not valid') try: if model == 'NCEP': getlevel = True if 'z' in var: level = var.strip('z') variable = 'hgt' # conform_units_to='hPa' else: variable = 'slp' level = None # conform_units_to='hPa' elif '20CRV2' in model: getlevel = False if 'z' in var: variable = 'hgt' level = var.strip('z') # conform_units_to=None else: variable = 'prmsl' level = None # conform_units_to='hPa' else: LOGGER.exception('Reanalyses model not known') LOGGER.info('environment set') except: msg = 'failed to set environment' LOGGER.exception(msg) raise Exception(msg) # LOGGER.exception("init took %s seconds.", time.time() - start_time) response.update_status('Read in the arguments', 6) ################# # get input data ################# # TODO: do not forget to select years start_time = time.time() # measure get_input_data ... response.update_status('fetching input data', 7) try: if direction == 're2mo': nc_reanalyses = reanalyses(start=anaSt.year, end=anaEn.year, variable=var, dataset=model, getlevel=getlevel) else: nc_reanalyses = reanalyses(start=refSt.year, end=refEn.year, variable=var, dataset=model, getlevel=getlevel) if type(nc_reanalyses) == list: nc_reanalyses = sorted( nc_reanalyses, key=lambda i: path.splitext(path.basename(i))[0]) else: nc_reanalyses = [nc_reanalyses] # For 20CRV2 geopotential height, daily dataset for 100 years is about 50 Gb # So it makes sense, to operate it step-by-step # TODO: need to create dictionary for such datasets (for models as well) # TODO: benchmark the method bellow for NCEP z500 for 60 years, may be use the same (!) # TODO Now everything regrid to the reanalysis if ('20CRV2' in model) and ('z' in var): tmp_total = [] origvar = get_variable(nc_reanalyses) for z in nc_reanalyses: tmp_n = 'tmp_%s' % (uuid.uuid1()) b0 = call(resource=z, variable=origvar, level_range=[int(level), int(level)], geom=bbox, spatial_wrapping='wrap', prefix='levdom_' + path.basename(z)[0:-3]) tmp_total.append(b0) tmp_total = sorted( tmp_total, key=lambda i: path.splitext(path.basename(i))[0]) inter_subset_tmp = call(resource=tmp_total, variable=origvar, time_range=r_time_range) # Clean for i in tmp_total: tbr = 'rm -f %s' % (i) #system(tbr) # Create new variable ds = Dataset(inter_subset_tmp, mode='a') z_var = ds.variables.pop(origvar) dims = z_var.dimensions new_var = ds.createVariable('z%s' % level, z_var.dtype, dimensions=(dims[0], dims[2], dims[3])) new_var[:, :, :] = squeeze(z_var[:, 0, :, :]) # new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()}) ds.close() nc_subset = call(inter_subset_tmp, variable='z%s' % level) else: nc_subset = call( resource=nc_reanalyses, variable=var, geom=bbox, spatial_wrapping='wrap', time_range=r_time_range, # conform_units_to=conform_units_to ) # nc_subset = call(resource=nc_reanalyses, variable=var, geom=bbox, spatial_wrapping='wrap') # XXXXXX wrap # LOGGER.exception("get_input_subset_model took %s seconds.", time.time() - start_time) response.update_status('**** Input reanalyses data fetched', 10) except: msg = 'failed to fetch or subset input files' LOGGER.exception(msg) raise Exception(msg) ######################## # input data preperation ######################## response.update_status('Start preparing input data', 12) # Filter resource: if type(resource) == list: resource = sorted(resource, key=lambda i: path.splitext(path.basename(i))[0]) else: resource = [resource] tmp_resource = [] m_start = m_time_range[0] m_end = m_time_range[1] for re in resource: s, e = get_timerange(re) tmpSt = dt.strptime(s, '%Y%m%d') tmpEn = dt.strptime(e, '%Y%m%d') if ((tmpSt <= m_end) and (tmpEn >= m_start)): tmp_resource.append(re) LOGGER.debug('Selected file: %s ' % (re)) resource = tmp_resource start_time = time.time() # mesure data preperation ... # TODO: Check the callendars ! for model vs reanalyses. # TODO: Check the units! model vs reanalyses. try: m_total = [] modvar = get_variable(resource) # resource properties ds = Dataset(resource[0]) m_var = ds.variables[modvar] dims = list(m_var.dimensions) dimlen = len(dims) try: model_id = ds.getncattr('model_id') except AttributeError: model_id = 'Unknown model' LOGGER.debug('MODEL: %s ' % (model_id)) lev_units = 'hPa' if (dimlen > 3): lev = ds.variables[dims[1]] # actually index [1] need to be detected... assuming zg(time, plev, lat, lon) lev_units = lev.units if (lev_units == 'Pa'): m_level = str(int(level) * 100) else: m_level = level else: m_level = None if level == None: level_range = None else: level_range = [int(m_level), int(m_level)] for z in resource: tmp_n = 'tmp_%s' % (uuid.uuid1()) # select level and regrid b0 = call( resource=z, variable=modvar, level_range=level_range, spatial_wrapping='wrap', #cdover='system', regrid_destination=nc_reanalyses[0], regrid_options='bil', prefix=tmp_n) # select domain b01 = call(resource=b0, geom=bbox, spatial_wrapping='wrap', prefix='levregr_' + path.basename(z)[0:-3]) tbr = 'rm -f %s' % (b0) #system(tbr) tbr = 'rm -f %s' % (tmp_n) #system(tbr) # get full resource m_total.append(b01) ds.close() model_subset = call(m_total, time_range=m_time_range) for i in m_total: tbr = 'rm -f %s' % (i) #system(tbr) if m_level is not None: # Create new variable in model set ds = Dataset(model_subset, mode='a') mod_var = ds.variables.pop(modvar) dims = mod_var.dimensions new_modvar = ds.createVariable('z%s' % level, mod_var.dtype, dimensions=(dims[0], dims[2], dims[3])) new_modvar[:, :, :] = squeeze(mod_var[:, 0, :, :]) # new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()}) ds.close() mod_subset = call(model_subset, variable='z%s' % level) else: mod_subset = model_subset # if direction == 're2mo': # try: # response.update_status('Preparing simulation data', 15) # reanalyses_subset = call(resource=nc_subset, time_range=[anaSt, anaEn]) # except: # msg = 'failed to prepare simulation period' # LOGGER.exception(msg) # try: # response.update_status('Preparing target data', 17) # var_target = get_variable(resource) # # var_simulation = get_variable(simulation) # model_subset_tmp = call(resource=resource, variable=var_target, # time_range=[refSt, refEn], # t_calendar='standard', # spatial_wrapping='wrap', # regrid_destination=nc_reanalyses[0], # regrid_options='bil') # # model_subset = call(resource=resource, variable=var_target, # # time_range=[refSt, refEn], # # geom=bbox, # # t_calendar='standard', # # # conform_units_to=conform_units_to, # # spatial_wrapping='wrap', # # regrid_destination=reanalyses_subset, # # regrid_options='bil') # XXXXXXXXXXXX ADD WRAP rem calendar # model_subset = call(resource=model_subset_tmp,variable=var_target, geom=bbox, spatial_wrapping='wrap', t_calendar='standard') # # ISSUE: the regrided model has white border with null! Check it. # # check t_calendar! # except: # msg = 'failed subset archive model' # LOGGER.exception(msg) # raise Exception(msg) # else: # try: # response.update_status('Preparing target data', 15) # var_target = get_variable(resource) # # var_simulation = get_variable(simulation) # model_subset = call(resource=resource, variable=var_target, # time_range=[refSt, refEn], # geom=bbox, # t_calendar='standard', # # conform_units_to=conform_units_to, # # spatial_wrapping='wrap', # ) # except: # msg = 'failed subset archive model' # LOGGER.exception(msg) # raise Exception(msg) # try: # response.update_status('Preparing simulation data', 17) # reanalyses_subset = call(resource=nc_subset, # time_range=[anaSt, anaEn], # regrid_destination=model_subset, # regrid_options='bil') # except: # msg = 'failed to prepare simulation period' # LOGGER.exception(msg) except: msg = 'failed to subset simulation or reference data' LOGGER.exception(msg) raise Exception(msg) # -------------------------------------------- try: if direction == 'mo2re': simulation = mod_subset archive = nc_subset base_id = model sim_id = model_id elif direction == 're2mo': simulation = nc_subset archive = mod_subset base_id = model_id sim_id = model else: LOGGER.exception('direction not valid: %s ' % direction) except: msg = 'failed to find comparison direction' LOGGER.exception(msg) raise Exception(msg) try: if level is not None: out_var = 'z%s' % level else: var_archive = get_variable(archive) var_simulation = get_variable(simulation) if var_archive != var_simulation: rename_variable(archive, oldname=var_archive, newname=var_simulation) out_var = var_simulation LOGGER.info('varname %s in netCDF renamed to %s' % (var_archive, var_simulation)) except: msg = 'failed to rename variable in target files' LOGGER.exception(msg) raise Exception(msg) try: if seacyc is True: seasoncyc_base, seasoncyc_sim = analogs.seacyc( archive, simulation, method=normalize) else: seasoncyc_base = None seasoncyc_sim = None except: msg = 'failed to prepare seasonal cycle reference files' LOGGER.exception(msg) raise Exception(msg) ip, output = mkstemp(dir='.', suffix='.txt') output_file = path.abspath(output) files = [path.abspath(archive), path.abspath(simulation), output_file] # LOGGER.exception("data preperation took %s seconds.", time.time() - start_time) ############################ # generating the config file ############################ response.update_status('writing config file', 18) start_time = time.time() # measure write config ... try: config_file = analogs.get_configfile( files=files, seasoncyc_base=seasoncyc_base, seasoncyc_sim=seasoncyc_sim, base_id=base_id, sim_id=sim_id, timewin=timewin, varname=var, seacyc=seacyc, cycsmooth=91, nanalog=nanalog, seasonwin=seasonwin, distfun=distance, outformat=outformat, calccor=True, silent=False, period=[ dt.strftime(refSt, '%Y-%m-%d'), dt.strftime(refEn, '%Y-%m-%d') ], bbox="%s,%s,%s,%s" % (bbox[0], bbox[2], bbox[1], bbox[3])) except: msg = 'failed to generate config file' LOGGER.exception(msg) raise Exception(msg) # LOGGER.exception("write_config took %s seconds.", time.time() - start_time) ####################### # CASTf90 call ####################### import subprocess import shlex start_time = time.time() # measure call castf90 response.update_status('Start CASTf90 call', 20) try: # response.update_status('execution of CASTf90', 50) cmd = 'analogue.out %s' % path.relpath(config_file) # system(cmd) args = shlex.split(cmd) output, error = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() LOGGER.info('analogue.out info:\n %s ' % output) LOGGER.exception('analogue.out errors:\n %s ' % error) response.update_status('**** CASTf90 suceeded', 90) except: msg = 'CASTf90 failed' LOGGER.exception(msg) raise Exception(msg) LOGGER.debug("castf90 took %s seconds.", time.time() - start_time) response.update_status('preparting output', 91) # Stopper to keep twitcher results, for debug # dummy=dummy response.outputs[ 'config'].file = config_file #config_output_url # config_file ) response.outputs['analogs'].file = output_file response.outputs['output_netcdf'].file = simulation response.outputs['target_netcdf'].file = archive ######################## # generate analog viewer ######################## formated_analogs_file = analogs.reformat_analogs(output_file) # response.outputs['formated_analogs'].storage = FileStorage() response.outputs['formated_analogs'].file = formated_analogs_file LOGGER.info('analogs reformated') response.update_status('reformatted analog file', 95) viewer_html = analogs.render_viewer( # configfile=response.outputs['config'].get_url(), configfile=config_file, # datafile=response.outputs['formated_analogs'].get_url()) datafile=formated_analogs_file) response.outputs['output'].file = viewer_html response.update_status('Successfully generated analogs viewer', 99) LOGGER.info('rendered pages: %s ', viewer_html) response.update_status('execution ended', 100) LOGGER.debug("total execution took %s seconds.", time.time() - process_start_time) return response
def calc_indice_percentile(resources=[], variable=None, prefix=None, indices='TG90p', refperiod=None, groupings='yr', polygons=None, percentile=90, mosaic=False, dir_output=None, dimension_map=None): """ Calculates given indices for suitable files in the appropriate time grouping and polygon. :param resource: list of filenames in data reference syntax (DRS) convention (netcdf) :param variable: variable name to be selected in the in netcdf file (default=None) :param indices: list of indices (default ='TG90p') :param prefix: filename prefix :param refperiod: reference period tuple = (start,end) :param grouping: indices time aggregation (default='yr') :param dir_output: output directory for result file (netcdf) :param dimension_map: optional dimension map if different to standard (default=None) :return: list of netcdf files with calculated indices. Files are saved into out_dir. """ from os.path import join, dirname, exists from os import remove import uuid from numpy import ma from datetime import datetime as dt from flyingpigeon.ocgis_module import call from flyingpigeon.subset import clipping from flyingpigeon.utils import get_values, get_time if type(resources) != list: resources = list([resources]) if type(indices) != list: indices = list([indices]) if type(groupings) != list: groupings = list([groupings]) if type(refperiod) == list: refperiod = refperiod[0] if refperiod is None: start = dt.strptime(refperiod.split('-')[0], '%Y%m%d') end = dt.strptime(refperiod.split('-')[1], '%Y%m%d') time_range = [start, end] else: time_range = None if dir_output is None: if not exists(dir_output): makedirs(dir_output) ################################################ # Compute a custom percentile basis using ICCLIM ################################################ from ocgis.contrib import library_icclim as lic nc_indices = [] nc_dic = sort_by_filename(resources) for grouping in groupings: calc_group = calc_grouping(grouping) for key in nc_dic.keys(): resource = nc_dic[key] if variable is None: variable = get_variable(resource) if polygons is None: nc_reference = call(resource=resource, prefix=str(uuid.uuid4()), time_range=time_range, output_format='nc', dir_output=dir_output) else: nc_reference = clipping(resource=resource, prefix=str(uuid.uuid4()), time_range=time_range, output_format='nc', polygons=polygons, dir_output=dir_output, mosaic=mosaic) arr = get_values(resource=nc_reference) dt_arr = get_time(resource=nc_reference) arr = ma.masked_array(arr) dt_arr = ma.masked_array(dt_arr) percentile = percentile window_width = 5 for indice in indices: name = indice.replace('_', str(percentile)) var = indice.split('_')[0] operation = None if 'T' in var: if percentile >= 50: operation = 'Icclim%s90p' % var func = 'icclim_%s90p' % var # icclim_TG90p else: operation = 'Icclim%s10p' % var func = 'icclim_%s10p' % var ################################ # load the appropriate operation ################################ ops = [op for op in dir(lic) if operation in op] if len(ops) == 0: raise Exception("operator does not exist %s", operation) exec "percentile_dict = lic.%s.get_percentile_dict(arr, dt_arr, percentile, window_width)" % ops[ 0] calc = [{ 'func': func, 'name': name, 'kwds': { 'percentile_dict': percentile_dict } }] if polygons is None: nc_indices.extend( call(resource=resource, prefix=key.replace(variable, name).replace( '_day_', '_%s_' % grouping), calc=calc, calc_grouping=calc_group, output_format='nc', dir_output=dir_output)) else: nc_indices.extend( clipping( resource=resource, prefix=key.replace(variable, name).replace( '_day_', '_%s_' % grouping), calc=calc, calc_grouping=calc_group, output_format='nc', dir_output=dir_output, polygons=polygons, mosaic=mosaic, )) if len(nc_indices) is 0: logger.debug('No indices are calculated') return None return nc_indices
def uncertainty(resouces , variable=None, ylim=None, title=None, dir_out=None): """ creates a png file containing the appropriate uncertainty plot. :param resouces: list of files containing the same variable :param variable: variable to be visualised. If None (default), variable will be detected :param title: string to be used as title :returns str: path/to/file.png """ logger.debug('Start visualisation uncertainty plot') import pandas as pd import numpy as np import netCDF4 from os.path import basename # === prepare invironment if type(resouces) == str: resouces = list([resouces]) if variable == None: variable = utils.get_variable(resouces[0]) if title == None: title = "Field mean of %s " % variable if dir_out == None: dir_out = '.' try: fig = plt.figure(figsize=(20,10), dpi=600, facecolor='w', edgecolor='k') o1 , output_png = mkstemp(dir=dir_out, suffix='.png') variable = utils.get_variable(resouces[0]) df = pd.DataFrame() logger.info('variable %s found in resources.' % variable) for f in resouces: try: ds = Dataset(f) data = np.squeeze(ds.variables[variable][:]) if len(data.shape) == 3: meanData = np.mean(data,axis=1) ts = np.mean(meanData,axis=1) else: ts = data[:] times = ds.variables['time'] jd = netCDF4.num2date(times[:],times.units) hs = pd.Series(ts, index=jd, name=basename(f)) hd = hs.to_frame() df[basename(f)] = hs# except Exception as e: logger.debug('failed to calculate timeseries for%s : %s ' %(f, e)) try: rollmean = df.rolling(window=30,center=True).mean() logger.info('rolling mean calculated for all input data') rmean = rollmean.median(axis=1, skipna=False)# quantile([0.5], axis=1, numeric_only=False ) q05 = rollmean.quantile([0.05], axis=1,)# numeric_only=False) q33 = rollmean.quantile([0.33], axis=1,)# numeric_only=False) q66 = rollmean.quantile([0.66], axis=1, )#numeric_only=False) q95 = rollmean.quantile([0.95], axis=1, )#numeric_only=False) logger.info('quantile calculated for all input data') except Exception as e: logger.debug('failed to calculate quantiles %s ' % e) try: plt.fill_between(rollmean.index.values, np.squeeze(q05.values), np.squeeze( q95.values), alpha=0.5, color='grey') plt.fill_between(rollmean.index.values, np.squeeze( q33.values),np.squeeze( q66.values), alpha=0.5, color='grey') plt.plot(rollmean.index.values, np.squeeze(rmean.values), c='r', lw=3) plt.xlim(min(df.index.values), max(df.index.values)) plt.ylim(ylim) plt.title(title, fontsize=20) plt.grid()# .grid_line_alpha=0.3 fig.savefig(output_png) plt.close() logger.debug('timeseries uncertainty plot done for %s'% variable) except Exception as e: logger.debug('failed to calculate quantiles %s ' % e) except Exception as e: logger.exception('uncertainty plot failed for %s' % variable) raise return output_png
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' LOGGER.info('Start process') from datetime import datetime as dt from flyingpigeon import weatherregimes as wr from tempfile import mkstemp response.update_status('execution started at : {}'.format(dt.now()), 5) ################################ # reading in the input arguments ################################ LOGGER.info('read in the arguments') # resources = self.getInputValues(identifier='resources') season = request.inputs['season'][0].data LOGGER.info('season %s', season) # bbox = [-80, 20, 50, 70] # TODO: Add checking for wrong cordinates and apply default if nesessary bbox = [] bboxStr = request.inputs['BBox'][0].data bboxStr = bboxStr.split(',') bbox.append(float(bboxStr[0])) bbox.append(float(bboxStr[2])) bbox.append(float(bboxStr[1])) bbox.append(float(bboxStr[3])) LOGGER.debug('BBOX for ocgis: {}'.format(bbox)) LOGGER.debug('BBOX original: {}'.format(bboxStr)) model_var = request.inputs['reanalyses'][0].data model, variable = model_var.split('_') period = request.inputs['period'][0].data LOGGER.info('period: {}'.format(period)) anualcycle = request.inputs['anualcycle'][0].data kappa = request.inputs['kappa'][0].data LOGGER.info('kappa: {}', kappa) method = request.inputs['method'][0].data LOGGER.info('Calc annual cycle with {}'.format(method)) sseas = request.inputs['sseas'][0].data LOGGER.info('Annual cycle calc with {}'.format(sseas)) start = dt.strptime(period.split('-')[0], '%Y%m%d') end = dt.strptime(period.split('-')[1], '%Y%m%d') LOGGER.debug('start: {0}, end: {1}'.format(start, end)) ########################### # set the environment ########################### response.update_status('fetching data from archive', 10) try: if model == 'NCEP': getlevel = False if 'z' in variable: level = variable.strip('z') conform_units_to = None else: level = None conform_units_to = 'hPa' elif '20CRV2' in model: getlevel = False if 'z' in variable: level = variable.strip('z') conform_units_to = None else: level = None conform_units_to = 'hPa' else: LOGGER.exception('Reanalyses dataset not known') LOGGER.info('environment set for model: {}'.format(model)) except Exception as ex: msg = 'failed to set environment: {}'.format(ex) LOGGER.exception(msg) raise Exception(msg) ########################################## # fetch Data from original data archive ########################################## from flyingpigeon.datafetch import reanalyses as rl from flyingpigeon.utils import get_variable # from os.path import basename, splitext from os import system from netCDF4 import Dataset from numpy import squeeze try: model_nc = rl(start=start.year, end=end.year, dataset=model, variable=variable, getlevel=getlevel) LOGGER.info('reanalyses data fetched') except Exception as ex: msg = 'failed to get reanalyses data: {}'.format(ex) LOGGER.exception(msg) raise Exception(msg) response.update_status('fetching data done', 15) ############################################################ # get the required bbox and time region from resource data ############################################################ response.update_status('subsetting region of interest', 17) # from flyingpigeon.weatherregimes import get_level # from flyingpigeon.ocgis_module import call time_range = [start, end] ############################################################ # Block of level and domain selection for geop huge dataset ############################################################ LevMulti = False # =========================================================================================== if 'z' in variable: tmp_total = [] origvar = get_variable(model_nc) if LevMulti == False: for z in model_nc: b0 = call(resource=z, variable=origvar, level_range=[int(level), int(level)], geom=bbox, spatial_wrapping='wrap', prefix='levdom_' + basename(z)[0:-3]) tmp_total.append(b0) else: # multiproc - no inprovements yet, need to check in hi perf machine... # ----------------------- try: import ctypes import os # TODO: This lib is for linux mkl_rt = ctypes.CDLL('libmkl_rt.so') nth = mkl_rt.mkl_get_max_threads() LOGGER.debug('Current number of threads: {}'.format(nth)) mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64))) nth = mkl_rt.mkl_get_max_threads() LOGGER.debug('NEW number of threads: {}'.format(nth)) # TODO: Does it \/\/\/ work with default shell=False in subprocess... (?) os.environ['MKL_NUM_THREADS'] = str(nth) os.environ['OMP_NUM_THREADS'] = str(nth) except Exception as ex: msg = 'Failed to set THREADS: {}'.format(ex) LOGGER.debug(msg) # ----------------------- from multiprocessing import Pool pool = Pool() # from multiprocessing.dummy import Pool as ThreadPool # pool = ThreadPool() tup_var = [origvar] * len(model_nc) tup_lev = [level] * len(model_nc) tup_bbox = [bbox] * len(model_nc) tup_args = zip(model_nc, tup_var, tup_lev, tup_bbox) tmp_total = pool.map(ocgis_call_wrap, tup_args) pool.close() pool.join() LOGGER.debug('Temporal subset files: {}'.format(tmp_total)) tmp_total = sorted(tmp_total, key=lambda i: splitext(basename(i))[0]) inter_subset_tmp = call(resource=tmp_total, variable=origvar, time_range=time_range) # FIXME: System calls to rm are dangerous! Use os.rmdir instead! # Clean for i in tmp_total: tbr = 'rm -f {}'.format(i) system(tbr) # Create new variable ds = Dataset(inter_subset_tmp, mode='a') z_var = ds.variables.pop(origvar) dims = z_var.dimensions new_var = ds.createVariable('z{}'.format(level), z_var.dtype, dimensions=(dims[0], dims[2], dims[3])) new_var[:, :, :] = squeeze(z_var[:, 0, :, :]) # new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()}) ds.close() model_subset = call(inter_subset_tmp, variable='z{}'.format(level)) else: model_subset = call( resource=model_nc, variable=variable, geom=bbox, spatial_wrapping='wrap', time_range=time_range, # conform_units_to=conform_units_to ) # ============================================================================================= LOGGER.info('Dataset subset done: {}'.format(model_subset)) response.update_status('dataset subsetted', 18) ############################################## # computing anomalies ############################################## response.update_status('computing anomalies ', 19) cycst = anualcycle.split('-')[0] cycen = anualcycle.split('-')[1] reference = [ dt.strptime(cycst, '%Y%m%d'), dt.strptime(cycen, '%Y%m%d') ] LOGGER.info('reference time: {}'.format(reference)) model_anomal = wr.get_anomalies(model_subset, reference=reference, method=method, sseas=sseas) # , variable=variable) ##################### # extracting season ##################### response.update_status('normalizing data', 21) model_season = wr.get_season(model_anomal, season=season) response.update_status('anomalies computed and normalized', 24) ####################### # call the R scripts ####################### response.update_status('Start weather regime clustering ', 25) import subprocess from flyingpigeon import config from os.path import curdir, join try: rworkspace = curdir Rsrc = config.Rsrc_dir() Rfile = 'weatherregimes_model.R' infile = model_season # model_subset #model_ponderate modelname = model yr1 = start.year yr2 = end.year ip, output_graphics = mkstemp(dir=curdir, suffix='.pdf') ip, file_pca = mkstemp(dir=curdir, suffix='.txt') ip, file_class = mkstemp(dir=curdir, suffix='.Rdat') # TODO: Rewrite this using os.path.join or pathlib libraries args = [ 'Rscript', join(Rsrc, Rfile), '%s/' % curdir, '%s/' % Rsrc, '%s' % infile, '%s' % variable, '%s' % output_graphics, '%s' % file_pca, '%s' % file_class, '%s' % season, '%s' % start.year, '%s' % end.year, '%s' % model_var, '%s' % kappa ] LOGGER.info('Rcall builded') LOGGER.debug('ARGS: %s' % (args)) except Exception as ex: msg = 'failed to build the R command: {}'.format(ex) LOGGER.exception(msg) raise Exception(msg) try: output, error = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() LOGGER.info('R outlog info:\n {}'.format(output)) LOGGER.exception('R outlog errors:\n {}'.format(error)) if len(output) > 0: response.update_status('**** weatherregime in R suceeded', 90) else: LOGGER.exception('No output returned from R call') except Exception as ex: msg = 'failed to run the R weatherregime: {}'.format(ex) LOGGER.exception(msg) raise Exception(msg) response.update_status('Weather regime clustering done ', 93) ############################################ # set the outputs ############################################ response.update_status('Set the process outputs ', 95) response.outputs['Routput_graphic'].file = output_graphics response.outputs['output_pca'].file = file_pca response.outputs['output_classification'].file = file_class response.outputs['output_netcdf'].file = model_subset response.update_status('done', 100) return response
def execute(self): import time # performance test process_start_time = time.time() # measure process execution time ... from os import path from tempfile import mkstemp from flyingpigeon import analogs from datetime import datetime as dt from flyingpigeon.ocgis_module import call from flyingpigeon.datafetch import reanalyses from flyingpigeon.utils import get_variable, rename_variable self.status.set('execution started at : %s ' % dt.now(), 5) start_time = time.time() # measure init ... resource = self.getInputValues(identifier='resource') bbox_obj = self.BBox.getValue() refSt = self.getInputValues(identifier='refSt') refEn = self.getInputValues(identifier='refEn') dateSt = self.getInputValues(identifier='dateSt') dateEn = self.getInputValues(identifier='dateEn') normalize = self.getInputValues(identifier='normalize')[0] distance = self.getInputValues(identifier='dist')[0] outformat = self.getInputValues(identifier='outformat')[0] timewin = int(self.getInputValues(identifier='timewin')[0]) experiment = self.getInputValues(identifier='experiment')[0] dataset, var = experiment.split('_') refSt = dt.strptime(refSt[0], '%Y-%m-%d') refEn = dt.strptime(refEn[0], '%Y-%m-%d') dateSt = dt.strptime(dateSt[0], '%Y-%m-%d') dateEn = dt.strptime(dateEn[0], '%Y-%m-%d') if normalize == 'None': seacyc = False else: seacyc = True if outformat == 'ascii': outformat = '.txt' elif outformat == 'netCDF': outformat = '.nc' else: logger.error('output format not valid') if bbox_obj is not None: logger.info("bbox_obj={0}".format(bbox_obj.coords)) bbox = [ bbox_obj.coords[0][0], bbox_obj.coords[0][1], bbox_obj.coords[1][0], bbox_obj.coords[1][1] ] logger.info("bbox={0}".format(bbox)) else: bbox = None #start = min( refSt, dateSt ) #end = max( refEn, dateEn ) # region = self.getInputValues(identifier='region')[0] # bbox = [float(b) for b in region.split(',')] try: if dataset == 'NCEP': if 'z' in var: variable = 'hgt' level = var.strip('z') #conform_units_to=None else: variable = 'slp' level = None #conform_units_to='hPa' elif '20CRV2' in var: if 'z' in level: variable = 'hgt' level = var.strip('z') #conform_units_to=None else: variable = 'prmsl' level = None #conform_units_to='hPa' else: logger.error('Reanalyses dataset not known') logger.info('environment set') except Exception as e: msg = 'failed to set environment %s ' % e logger.error(msg) raise Exception(msg) logger.debug("init took %s seconds.", time.time() - start_time) self.status.set('Read in the arguments', 5) ################# # get input data ################# start_time = time.time() # measure get_input_data ... self.status.set('fetching input data', 7) try: input = reanalyses(start=dateSt.year, end=dateEn.year, variable=var, dataset=dataset) nc_subset = call(resource=input, variable=var, geom=bbox) except Exception as e: msg = 'failed to fetch or subset input files %s' % e logger.error(msg) raise Exception(msg) logger.debug("get_input_subset_dataset took %s seconds.", time.time() - start_time) self.status.set('**** Input data fetched', 10) ######################## # input data preperation ######################## self.status.set('Start preparing input data', 12) start_time = time.time() # mesure data preperation ... try: self.status.set('Preparing simulation data', 15) simulation = call(resource=nc_subset, time_range=[dateSt, dateEn]) except: msg = 'failed to prepare simulation period' logger.debug(msg) try: self.status.set('Preparing target data', 17) var_target = get_variable(resource) #var_simulation = get_variable(simulation) archive = call( resource=resource, variable=var_target, time_range=[refSt, refEn], geom=bbox, t_calendar= 'standard', # conform_units_to=conform_units_to, spatial_wrapping='wrap', regrid_destination=simulation, regrid_options='bil') except Exception as e: msg = 'failed subset archive dataset %s ' % e logger.debug(msg) raise Exception(msg) try: if var != var_target: rename_variable(archive, oldname=var_target, newname=var) logger.info('varname %s in netCDF renamed to %s' % (var_target, var)) except Exception as e: msg = 'failed to rename variable in target files %s ' % e logger.debug(msg) raise Exception(msg) try: if seacyc == True: seasoncyc_base, seasoncyc_sim = analogs.seacyc( archive, simulation, method=normalize) else: seasoncyc_base, seasoncyc_sim = None except Exception as e: msg = 'failed to prepare seasonal cycle reference files %s ' % e logger.debug(msg) raise Exception(msg) ip, output = mkstemp(dir='.', suffix='.txt') output_file = path.abspath(output) files = [path.abspath(archive), path.abspath(simulation), output_file] logger.debug("data preperation took %s seconds.", time.time() - start_time) ############################ # generating the config file ############################ self.status.set('writing config file', 15) start_time = time.time() # measure write config ... try: config_file = analogs.get_configfile( files=files, seasoncyc_base=seasoncyc_base, seasoncyc_sim=seasoncyc_sim, timewin=timewin, varname=var, seacyc=seacyc, cycsmooth=91, nanalog=nanalog, seasonwin=seasonwin, distfun=distance, outformat=outformat, calccor=True, silent=False, period=[ dt.strftime(refSt, '%Y-%m-%d'), dt.strftime(refEn, '%Y-%m-%d') ], bbox="%s,%s,%s,%s" % (bbox[0], bbox[2], bbox[1], bbox[3])) except Exception as e: msg = 'failed to generate config file %s ' % e logger.debug(msg) raise Exception(msg) logger.debug("write_config took %s seconds.", time.time() - start_time) ####################### # CASTf90 call ####################### import subprocess import shlex start_time = time.time() # measure call castf90 self.status.set('Start CASTf90 call', 20) try: #self.status.set('execution of CASTf90', 50) cmd = 'analogue.out %s' % path.relpath(config_file) #system(cmd) args = shlex.split(cmd) output, error = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() logger.info('analogue.out info:\n %s ' % output) logger.debug('analogue.out errors:\n %s ' % error) self.status.set('**** CASTf90 suceeded', 90) except Exception as e: msg = 'CASTf90 failed %s ' % e logger.error(msg) raise Exception(msg) logger.debug("castf90 took %s seconds.", time.time() - start_time) self.status.set('preparting output', 99) self.config.setValue(config_file) self.analogs.setValue(output_file) self.simulation_netcdf.setValue(simulation) self.target_netcdf.setValue(archive) self.status.set('execution ended', 100) logger.debug("total execution took %s seconds.", time.time() - process_start_time)
def calc_indice_simple(resource=[], variable=None, prefix=None, indice='SU', polygons=None, mosaic=False, grouping='yr', dir_output=None, dimension_map=None, memory_limit=None): """ Calculates given simple indices for suitable files in the appropriate time grouping and polygon. :param resource: list of filenames in data reference syntax (DRS) convention (netcdf) :param variable: variable name to be selected in the in netcdf file (default=None) :param indices: Indice (default ='SU') :param polygons: list of polgons (default ='FRA') :param grouping: indices time aggregation (default='yr') :param out_dir: output directory for result file (netcdf) :param dimension_map: optional dimension map if different to standard (default=None) :return: list of netcdf files with calculated indices. Files are saved into out_dir. """ from os.path import join, dirname, exists from flyingpigeon import ocgis_module from flyingpigeon.subset import clipping import uuid if type(resource) != list: resource = list([resource]) # if type(indices) != list: # indices = list([indices]) if type(polygons) != list and polygons is None: polygons = list([polygons]) # if type(groupings) != list: # groupings = list([groupings]) if dir_output is not None: if not exists(dir_output): makedirs(dir_output) datasets = sort_by_filename(resource).keys() if len(datasets) is 1: key = datasets[0] else: LOGGER.warning('more than one dataset in resource') # from flyingpigeon.subset import select_ugid # tile_dim = 25 output = None # experiments = sort_by_filename(resource) outputs = [] # for key in experiments: if variable is None: variable = get_variable(resource) LOGGER.debug('Variable detected % s ' % variable) # variable = key.split('_')[0] try: # icclim can't handling 'kg m2 sec' needs to be 'mm/day' if variable == 'pr': calc = 'pr=pr*86400' ncs = ocgis_module.call(resource=resource, variable=variable, dimension_map=dimension_map, calc=calc, memory_limit=memory_limit, # calc_grouping= calc_group, prefix=str(uuid.uuid4()), dir_output=dir_output, output_format='nc') else: ncs = resource try: calc = [{'func': 'icclim_' + indice, 'name': indice}] LOGGER.info('calc: %s' % calc) try: calc_group = calc_grouping(grouping) LOGGER.info('calc_group: %s' % calc_group) if polygons is None: try: prefix = key.replace(variable, indice).replace('_day_', '_%s_' % grouping) LOGGER.debug(' **** dir_output = %s ' % dir_output) tmp = ocgis_module.call(resource=ncs, variable=variable, dimension_map=dimension_map, calc=calc, calc_grouping=calc_group, prefix=prefix, dir_output=dir_output, output_format='nc') if len(tmp) is not 0: outputs.extend(tmp) else: msg = 'could not calc indice %s for domain ' % (indice) LOGGER.exception(msg) except: msg = 'could not calc indice %s for domain in %s' % (indice) LOGGER.exception(msg) else: try: prefix = key.replace(variable, indice).replace('_day_', '_%s_' % grouping) tmp = clipping(resource=ncs, variable=variable, dimension_map=dimension_map, calc=calc, calc_grouping=calc_group, prefix=prefix, polygons=polygons, mosaic=mosaic, dir_output=dir_output, output_format='nc') if len(tmp) is not 0: outputs.extend(tmp) else: msg = 'could not calc clipped indice %s ' % (indice) LOGGER.exception(msg) except: msg = 'could not calc indice %s for domai' % (indice) LOGGER.debug(msg) # raise Exception(msg) LOGGER.info('indice file calculated: %s' % tmp) except: msg = 'could not calc indice %s for key %s and grouping %s' % (indice, grouping) LOGGER.exception(msg) # raise Exception(msg) except: msg = 'could not calc indice %s ' % (indice) LOGGER.exception(msg) # raise Exception(msg) except: msg = 'could not calculate indices' LOGGER.exception(msg) # raise Exception(msg) LOGGER.info('indice outputs %s ' % outputs) if len(outputs) is 0: LOGGER.debug('No indices are calculated') return None return outputs
def get_anomalies(nc_file, frac=0.2, reference=None): """ Anomalisation of data subsets for weather classification by subtracting a smoothed annual cycle :param nc_file: input netCDF file :param frac: Number between 0-1 for strength of smoothing (0 = close to the original data, 1 = flat line) default = 0.2 :param reference: Period to calculate annual cycle :returns str: path to output netCDF file """ try: variable = utils.get_variable(nc_file) calc = [{'func': 'mean', 'name': variable}] calc_grouping = calc_grouping = ['day', 'month'] nc_anual_cycle = call(nc_file, calc=calc, calc_grouping=calc_grouping, time_range=reference) logger.info('annual cycle calculated') except Exception as e: msg = 'failed to calcualte annual cycle %s' % e logger.error(msg) raise Exception(msg) try: # spline for smoothing import statsmodels.api as sm from numpy import tile, empty, linspace from netCDF4 import Dataset from cdo import Cdo cdo = Cdo() # variable = utils.get_variable(nc_file) ds = Dataset(nc_anual_cycle, mode='a') vals = ds.variables[variable] vals_sm = empty(vals.shape) ts = vals.shape[0] x = linspace(1, ts * 3, num=ts * 3, endpoint=True) for lat in range(vals.shape[1]): for lon in range(vals.shape[2]): try: y = tile(vals[:, lat, lon], 3) # ys = smooth(y, window_size=91, order=2, deriv=0, rate=1)[ts:ts*2] ys = sm.nonparametric.lowess(y, x, frac=frac)[ts:ts * 2, 1] vals_sm[:, lat, lon] = ys except: msg = 'failed for lat %s lon %s' % (lat, lon) logger.exception(msg) raise Exception(msg) logger.debug('done for %s - %s ' % (lat, lon)) vals[:, :, :] = vals_sm[:, :, :] ds.close() logger.info('smothing of annual cycle done') except: msg = 'failed smothing of annual cycle' logger.exception(msg) raise Exception(msg) try: ip, nc_anomal = mkstemp(dir='.', suffix='.nc') nc_anomal = cdo.sub(input=[nc_file, nc_anual_cycle], output=nc_anomal) logger.info('cdo.sub; anomalisation done: %s ' % nc_anomal) except: msg = 'failed substraction of annual cycle' logger.exception(msg) raise Exception(msg) return nc_anomal
def method_A(resource=[], start=None, end=None, timeslice=20, variable=None, title=None, cmap='seismic'): """returns the result :param resource: list of paths to netCDF files :param start: beginning of reference period (if None (default), the first year of the consistent ensemble will be detected) :param end: end of comparison period (if None (default), the last year of the consistent ensemble will be detected) :param timeslice: period length for mean calculation of reference and comparison period :param variable: variable name to be detected in the netCDF file. If not set (not recommended), the variable name will be detected :param title: str to be used as title for the signal mal :param cmap: define the color scheme for signal map plotting :return: signal.nc, low_agreement_mask.nc, high_agreement_mask.nc, graphic.png, text.txt """ from os.path import split from cdo import Cdo cdo = Cdo() cdo.forceOutput = True try: # preparing the resource file_dic = sort_by_filename(resource, historical_concatination=True) logger.info('file names sorted experimets: %s' % len(file_dic.keys())) except Exception as e: msg = 'failed to sort the input files' logger.exception(msg) raise Exception(msg) try: mergefiles = [] for key in file_dic.keys(): if type(file_dic[key]) == list and len(file_dic[key]) > 1: input = [] for i in file_dic[key]: input.extend([i.replace(' ', '\\\ ')]) mergefiles.append( cdo.mergetime(input=input, output=key + '_mergetime.nc')) else: mergefiles.extend(file_dic[key]) logger.info('datasets merged %s ' % mergefiles) except Exception as e: msg = 'seltime and mergetime failed %s' % e logger.exception(msg) raise Exception(e) try: text_src = open('infiles.txt', 'a') for key in file_dic.keys(): text_src.write(key + '\n') text_src.close() except Exception as e: msg = 'failed to write source textfile' logger.exception(msg) raise Exception(msg) # configure reference and compare period try: if start is None: st_set = set() en_set = set() for f in mergefiles: times = get_time(f) st_set.update([times[0].year]) if end is None: en_set.update([times[-1].year]) start = max(st_set) if end is None: end = min(en_set) logger.info('Start and End: %s - %s ' % (start, end)) if start >= end: logger.error( 'ensemble is inconsistent!!! start year is later than end year' ) except Exception as e: msg = 'failed to detect start and end times of the ensemble' logger.exception(msg) raise Exception(msg) # set the periodes: try: start = int(start) end = int(end) if timeslice is None: timeslice = int((end - start) / 3) if timeslice == 0: timeslice = 1 else: timeslice = int(timeslice) start1 = start start2 = start1 + timeslice - 1 end1 = end - timeslice + 1 end2 = end logger.info('timeslice and periodes set') except Exception as e: msg = 'failed to set the periodes' logger.exception(msg) raise Exception(msg) try: files = [] for i, mf in enumerate(mergefiles): files.append( cdo.selyear('{0}/{1}'.format(start1, end2), input=[mf.replace(' ', '\ ')], output='file_{0}_.nc'.format(i))) # python version logger.info('timeseries selected from defined start to end year') except Exception as e: msg = 'seltime and mergetime failed' logger.exception(msg) raise Exception(msg) try: # ensemble mean nc_ensmean = cdo.ensmean(input=files, output='nc_ensmean.nc') logger.info('ensemble mean calculation done') except Exception as e: msg = 'ensemble mean failed' logger.exception(msg) raise Exception(msg) try: # ensemble std nc_ensstd = cdo.ensstd(input=files, output='nc_ensstd.nc') logger.info('ensemble std and calculation done') except Exception as e: msg = 'ensemble std or failed' logger.exception(msg) raise Exception(msg) # get the get the signal as difference from the beginning (first years) and end period (last years), : try: selyearstart = cdo.selyear('%s/%s' % (start1, start2), input=nc_ensmean, output='selyearstart.nc') selyearend = cdo.selyear('%s/%s' % (end1, end2), input=nc_ensmean, output='selyearend.nc') meanyearst = cdo.timmean(input=selyearstart, output='meanyearst.nc') meanyearend = cdo.timmean(input=selyearend, output='meanyearend.nc') signal = cdo.sub(input=[meanyearend, meanyearst], output='signal.nc') logger.info('Signal calculation done') except Exception as e: msg = 'calculation of signal failed' logger.exception(msg) raise Exception(msg) # get the intermodel standard deviation (mean over whole period) try: # std_selyear = cdo.selyear('%s/%s' % (end1,end2), input=nc_ensstd, output='std_selyear.nc') # std = cdo.timmean(input = std_selyear, output = 'std.nc') std = cdo.timmean(input=nc_ensstd, output='std.nc') std2 = cdo.mulc('2', input=std, output='std2.nc') logger.info('calculation of internal model std for time period done') except Exception as e: msg = 'calculation of internal model std failed' logger.exception(msg) raise Exception(msg) try: absolut = cdo.abs(input=signal, output='absolut_signal.nc') high_agreement_mask = cdo.gt( input=[absolut, std2], output='large_change_with_high_model_agreement.nc') low_agreement_mask = cdo.lt( input=[absolut, std], output='small_signal_or_low_agreement_of_models.nc') logger.info('high and low mask done') except Exception as e: msg = 'calculation of robustness mask failed' logger.exception(msg) raise Exception(msg) try: if variable is None: variable = get_variable(signal) logger.info('variable to be plotted: %s' % variable) if title is None: title = 'Change of %s (difference of mean %s-%s to %s-%s)' % ( variable, end1, end2, start1, start2) graphic = None graphic = map_ensembleRobustness(signal, high_agreement_mask, low_agreement_mask, variable=variable, cmap=cmap, title=title) logger.info('graphic generated') except Exception as e: msg('graphic generation failed: %s' % e) logger.debug(msg) raise Exception(msg) return signal, low_agreement_mask, high_agreement_mask, graphic, text_src #
def clipping(resource=[], variable=None, dimension_map=None, calc=None, output_format='nc', calc_grouping=None, time_range=None, time_region=None, historical_concatination=True, prefix=None, spatial_wrapping='wrap', polygons=None, mosaic=False, dir_output=None, memory_limit=None): """ returns list of clipped netCDF files :param resource: list of input netCDF files :param variable: variable (string) to be used in netCDF :param dimesion_map: specify a dimension map if input netCDF has unconventional dimension :param calc: ocgis calculation argument :param calc_grouping: ocgis calculation grouping :param historical_concatination: concat files of RCPs with appropriate historical runs into one timeseries :param prefix: prefix for output file name :param polygons: list of polygons to be used. If more than 1 in the list, an appropriate mosaic will be clipped :param mosaic: Whether the polygons are aggregated into a single geometry (True) or individual files are created for each geometry (False). :param output_format: output_format (default='nc') :param dir_output: specify an output location :param time_range: [start, end] of time subset :param time_region: year, months or days to be extracted in the timeseries :returns list: path to clipped files """ if type(resource) != list: resource = list([resource]) if type(polygons) != list: polygons = list([polygons]) if prefix is not None: if type(prefix) != list: prefix = list([prefix]) geoms = set() ncs = sort_by_filename(resource, historical_concatination=historical_concatination ) # historical_concatenation=True geom_files = [] if mosaic is True: try: nameadd = '_' for polygon in polygons: geoms.add(get_geom(polygon)) nameadd = nameadd + polygon.replace(' ', '') if len(geoms) > 1: LOGGER.error( 'polygons belong to different shapefiles! mosaic option is not possible %s', geoms) else: geom = geoms.pop() ugids = get_ugid(polygons=polygons, geom=geom) except: LOGGER.exception('geom identification failed') for i, key in enumerate(ncs.keys()): try: # if variable is None: variable = get_variable(ncs[key]) LOGGER.info('variable %s detected in resource' % (variable)) if prefix is None: name = key + nameadd else: name = prefix[i] geom_file = call(resource=ncs[key], variable=variable, calc=calc, calc_grouping=calc_grouping, output_format=output_format, prefix=name, geom=geom, select_ugid=ugids, time_range=time_range, time_region=time_region, spatial_wrapping=spatial_wrapping, memory_limit=memory_limit, dir_output=dir_output, dimension_map=dimension_map) geom_files.append(geom_file) LOGGER.info('ocgis mosaik clipping done for %s ' % (key)) except: msg = 'ocgis mosaik clipping failed for %s ' % (key) LOGGER.exception(msg) else: for i, polygon in enumerate(polygons): try: geom = get_geom(polygon) ugid = get_ugid(polygons=polygon, geom=geom) for key in ncs.keys(): try: # if variable is None: variable = get_variable(ncs[key]) LOGGER.info('variable %s detected in resource' % (variable)) if prefix is None: name = key + '_' + polygon.replace(' ', '') else: name = prefix[i] geom_file = call( resource=ncs[key], variable=variable, calc=calc, calc_grouping=calc_grouping, output_format=output_format, prefix=name, geom=geom, select_ugid=ugid, dir_output=dir_output, dimension_map=dimension_map, spatial_wrapping=spatial_wrapping, memory_limit=memory_limit, time_range=time_range, time_region=time_region, ) geom_files.append(geom_file) LOGGER.info('ocgis clipping done for %s ' % (key)) except: msg = 'ocgis clipping failed for %s ' % (key) LOGGER.exception(msg) except: LOGGER.exception('geom identification failed') return geom_files
def execute(self): init_process_logger('log.txt') self.output_log.setValue('log.txt') import time # performance test process_start_time = time.time() # measure process execution time ... from os import path from tempfile import mkstemp from datetime import datetime as dt from flyingpigeon import analogs from flyingpigeon.ocgis_module import call from flyingpigeon.datafetch import reanalyses from flyingpigeon.utils import get_variable self.status.set('execution started at : %s ' % dt.now(), 5) start_time = time.time() # measure init ... ####################### # read input parameters ####################### try: self.status.set('read input parameter : %s ' % dt.now(), 5) resource = self.getInputValues(identifier='resource') refSt = self.getInputValues(identifier='refSt') refEn = self.getInputValues(identifier='refEn') dateSt = self.getInputValues(identifier='dateSt') dateEn = self.getInputValues(identifier='dateEn') normalize = self.getInputValues(identifier='normalize')[0] distance = self.getInputValues(identifier='dist')[0] outformat = self.getInputValues(identifier='outformat')[0] timewin = int(self.getInputValues(identifier='timewin')[0]) bbox_obj = self.BBox.getValue() seasonwin = int(self.getInputValues(identifier='seasonwin')[0]) nanalog = int(self.getInputValues(identifier='nanalog')[0]) # region = self.getInputValues(identifier='region')[0] # bbox = [float(b) for b in region.split(',')] # experiment = self.getInputValues(identifier='experiment')[0] # dataset , var = experiment.split('_') logger.info('input parameters set') except Exception as e: msg = 'failed to read input prameter %s ' % e logger.error(msg) raise Exception(msg) ###################################### # convert types and set environment ###################################### try: refSt = dt.strptime(refSt[0], '%Y-%m-%d') refEn = dt.strptime(refEn[0], '%Y-%m-%d') dateSt = dt.strptime(dateSt[0], '%Y-%m-%d') dateEn = dt.strptime(dateEn[0], '%Y-%m-%d') if normalize == 'None': seacyc = False else: seacyc = True if outformat == 'ascii': outformat = '.txt' elif outformat == 'netCDF': outformat = '.nc' else: logger.error('output format not valid') start = min(refSt, dateSt) end = max(refEn, dateEn) if bbox_obj is not None: logger.info("bbox_obj={0}".format(bbox_obj.coords)) bbox = [bbox_obj.coords[0][0], bbox_obj.coords[0][1], bbox_obj.coords[1][0], bbox_obj.coords[1][1]] logger.info("bbox={0}".format(bbox)) else: bbox = None logger.info('environment set') except Exception as e: msg = 'failed to set environment %s ' % e logger.error(msg) raise Exception(msg) logger.debug("init took %s seconds.", time.time() - start_time) self.status.set('Read in and convert the arguments', 5) ######################## # input data preperation ######################## # TODO: Check if files containing more than one dataset self.status.set('Start preparing input data', 12) start_time = time.time() # mesure data preperation ... try: variable = get_variable(resource) archive = call(resource=resource, time_range=[refSt, refEn], geom=bbox, spatial_wrapping='wrap') simulation = call(resource=resource, time_range=[dateSt, dateEn], geom=bbox, spatial_wrapping='wrap') if seacyc is True: seasoncyc_base, seasoncyc_sim = analogs.seacyc(archive, simulation, method=normalize) else: seasoncyc_base = None seasoncyc_sim = None except Exception as e: msg = 'failed to prepare archive and simulation files %s ' % e logger.debug(msg) raise Exception(msg) ip, output = mkstemp(dir='.', suffix='.txt') output_file = path.abspath(output) files = [path.abspath(archive), path.abspath(simulation), output_file] logger.debug("data preperation took %s seconds.", time.time() - start_time) ############################ # generating the config file ############################ self.status.set('writing config file', 15) start_time = time.time() # measure write config ... try: config_file = analogs.get_configfile( files=files, seasoncyc_base=seasoncyc_base, seasoncyc_sim=seasoncyc_sim, timewin=timewin, varname=variable, seacyc=seacyc, cycsmooth=91, nanalog=nanalog, seasonwin=seasonwin, distfun=distance, outformat=outformat, calccor=True, silent=False, period=[dt.strftime(refSt, '%Y-%m-%d'), dt.strftime(refEn, '%Y-%m-%d')], bbox="%s,%s,%s,%s" % (bbox[0], bbox[2], bbox[1], bbox[3])) except Exception as e: msg = 'failed to generate config file %s ' % e logger.debug(msg) raise Exception(msg) logger.debug("write_config took %s seconds.", time.time() - start_time) ############## # CASTf90 call ############## import subprocess import shlex start_time = time.time() # measure call castf90 self.status.set('Start CASTf90 call', 20) try: # self.status.set('execution of CASTf90', 50) cmd = 'analogue.out %s' % path.relpath(config_file) # system(cmd) args = shlex.split(cmd) output, error = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() logger.info('analogue.out info:\n %s ' % output) logger.debug('analogue.out errors:\n %s ' % error) self.status.set('**** CASTf90 suceeded', 90) except Exception as e: msg = 'CASTf90 failed %s ' % e logger.error(msg) raise Exception(msg) logger.debug("castf90 took %s seconds.", time.time() - start_time) self.status.set('preparting output', 99) self.config.setValue(config_file) self.analogs.setValue(output_file) self.output_netcdf.setValue(simulation) self.status.set('execution ended', 100) logger.debug("total execution took %s seconds.", time.time() - process_start_time)
def get_anomalies(nc_file, frac=0.2, reference=None): ''' anomalisation of data subsets for weather classification. Anomalisation is done by substrcting a smoothed anual cycle :parm nc_file: input netCDF file :param frac: Number between 0-1 for stregth of smoothing (0 = close to the original data, 1=flat line) default=0.2 :param reference: Period to calulate anual cycle :return string: path to output netCDF file ''' try: variable = utils.get_variable(nc_file) calc = [{'func': 'mean', 'name': variable}] calc_grouping = calc_grouping = ['day','year'] nc_anual_cycle = call(nc_file, calc=calc, calc_grouping=calc_grouping, time_range=reference) logger.info('anual cycle calculated') except Exception as e: msg = 'failed to calcualte anual cycle %s' % e logger.error(msg) raise Exception(msg) ### spline for smoothing import statsmodels.api as sm from numpy import tile, empty, linspace from netCDF4 import Dataset from cdo import Cdo cdo = Cdo() try: # variable = utils.get_variable(nc_file) ds = Dataset(nc_anual_cycle, mode='a') vals = ds.variables[variable] vals_sm = empty(vals.shape) ts = vals.shape[0] x = linspace(1, ts*3 , num=ts*3 , endpoint=True) for lat in range(vals.shape[1]): for lon in range(vals.shape[2]): try: y = tile(vals[:,lat,lon], 3) # ys = smooth(y, window_size=91, order=2, deriv=0, rate=1)[ts:ts*2] ys = sm.nonparametric.lowess(y, x, frac=frac )[ts:ts*2,1] vals_sm[:,lat,lon] = ys except Exception as e: msg = 'failed for lat %s lon %s %s ' % (lat,lon,e) logger.debug('failed for lat %s lon %s %s ' % (lat,lon,e)) raise Exception(msg) print 'done for %s - %s ' % (lat, lon) vals[:,:,:] = vals_sm[:,:,:] ds.close() logger.info('smothing of anual cycle done') except Exception as e: msg = 'failed smothing of anual cycle %s ' % e logger.error(msg) raise Exception(msg) try: ip , nc_anomal = mkstemp(dir='.',suffix='.nc') nc_anomal = cdo.sub(input=[nc_file, nc_anual_cycle], output= nc_anomal ) logger.info('anomalisation done: %s ' % nc_anomal) except Exception as e: msg = 'failed substraction of anual cycle %s ' % e logger.error(msg) raise Exception(msg) return nc_anomal
def clipping(resource=[], variable=None, dimension_map=None, calc=None, output_format='nc', calc_grouping= None, time_range=None, time_region=None, historical_concatination=True, prefix=None, spatial_wrapping='wrap', polygons=None, mosaik=False, dir_output=None, memory_limit=None): """ returns list of clipped netCDF files possible entries: :param resource: list of input netCDF files :param variable: variable (string) to be used in netCDF :param dimesion_map: specify a dimension map input netCDF has unconventional dimension :param calc: ocgis calculation argument :param calc_grouping: ocgis calculation grouping :param historical_concatination: concat files of RCPs with appropriate historical runs to one timeseries :param prefix: perfix for output file name :param polygons: list of polygons to be used. if more than 1 in the list, a appropriate mosaik will be clipped :param output_format: output_format (default='nc') :param dir_output: specify a output location """ from flyingpigeon.utils import get_variable, drs_filename from flyingpigeon.ocgis_module import call if type(resource) != list: resource = list([resource]) if type(polygons) != list: polygons = list([polygons]) if prefix != None: if type(prefix) != list: prefix = list([prefix]) geoms = set() ncs = sort_by_filename(resource, historical_concatination=historical_concatination) # historical_concatination=True geom_files = [] if mosaik == True : try: nameadd = '_' for polygon in polygons: geoms.add(get_geom(polygon)) nameadd = nameadd + '-' + polygon if len(geoms) > 1: logger.error('polygons belong to differnt shapefiles! mosaik option is not possible %s', geoms) else: geom = geoms.pop() ugids = get_ugid(polygons=polygons, geom=geom) except Exception as e: logger.debug('geom identification failed %s ' % e) for i, key in enumerate (ncs.keys()): try: if variable == None: variable = get_variable(ncs[key]) logger.info('variable %s detected in resource' % (variable)) if prefix == None: name = key + nameadd else: name = prefix[i] geom_file = call(resource=ncs[key], variable=variable, calc=calc, calc_grouping=calc_grouping, output_format=output_format, prefix=name, geom=geom, select_ugid=ugids, time_range=time_range, time_region=time_region, spatial_wrapping=spatial_wrapping, memory_limit=memory_limit, dir_output=dir_output, dimension_map=dimension_map) geom_files.append( geom_file ) except Exception as e: msg = 'ocgis calculations failed for %s ' % (key) logger.debug(msg) else: for i, polygon in enumerate(polygons): try: geom = get_geom(polygon) ugid = get_ugid(polygons=polygon, geom=geom) for key in ncs.keys(): try: if variable == None: variable = get_variable(ncs[key]) logger.info('variable %s detected in resource' % (variable)) if prefix == None: name = key + '_' + polygon else: name = prefix[i] geom_file = call(resource=ncs[key], variable=variable, calc=calc, calc_grouping=calc_grouping,output_format=output_format, prefix=name, geom=geom, select_ugid=ugid, dir_output=dir_output, dimension_map=dimension_map, spatial_wrapping=spatial_wrapping, memory_limit=memory_limit,time_range=time_range, time_region=time_region, ) geom_files.append( geom_file ) except Exception as e: msg = 'ocgis calculations failed for %s ' % (key) logger.debug(msg) raise except Exception as e: logger.debug('geom identification failed') raise return geom_files
def test_get_variable(self): variable = utils.get_variable(utils.local_path(TESTDATA['cmip5_tasmax_nc'])) nose.tools.ok_("tasmax" == variable, variable)