def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
LOGGER.info('Start process')
from datetime import datetime as dt
from flyingpigeon import weatherregimes as wr
from tempfile import mkstemp
response.update_status('execution started at : {}'.format(dt.now()), 5)
################################
# reading in the input arguments
################################
LOGGER.info('read in the arguments')
# resources = self.getInputValues(identifier='resources')
season = request.inputs['season'][0].data
LOGGER.info('season %s', season)
# bbox = [-80, 20, 50, 70]
# TODO: Add checking for wrong cordinates and apply default if nesessary
bbox = []
bboxStr = request.inputs['BBox'][0].data
bboxStr = bboxStr.split(',')
bbox.append(float(bboxStr[0]))
bbox.append(float(bboxStr[2]))
bbox.append(float(bboxStr[1]))
bbox.append(float(bboxStr[3]))
LOGGER.debug('BBOX for ocgis: {}'.format(bbox))
LOGGER.debug('BBOX original: {}'.format(bboxStr))
model_var = request.inputs['reanalyses'][0].data
model, variable = model_var.split('_')
period = request.inputs['period'][0].data
LOGGER.info('period: {}'.format(period))
anualcycle = request.inputs['anualcycle'][0].data
kappa = request.inputs['kappa'][0].data
LOGGER.info('kappa: {}', kappa)
method = request.inputs['method'][0].data
LOGGER.info('Calc annual cycle with {}'.format(method))
sseas = request.inputs['sseas'][0].data
LOGGER.info('Annual cycle calc with {}'.format(sseas))
start = dt.strptime(period.split('-')[0], '%Y%m%d')
end = dt.strptime(period.split('-')[1], '%Y%m%d')
LOGGER.debug('start: {0}, end: {1}'.format(start, end))
###########################
# set the environment
###########################
response.update_status('fetching data from archive', 10)
try:
if model == 'NCEP':
getlevel = False
if 'z' in variable:
level = variable.strip('z')
conform_units_to = None
else:
level = None
conform_units_to = 'hPa'
elif '20CRV2' in model:
getlevel = False
if 'z' in variable:
level = variable.strip('z')
conform_units_to = None
else:
level = None
conform_units_to = 'hPa'
else:
LOGGER.exception('Reanalyses dataset not known')
LOGGER.info('environment set for model: {}'.format(model))
except Exception as ex:
msg = 'failed to set environment: {}'.format(ex)
LOGGER.exception(msg)
raise Exception(msg)
##########################################
# fetch Data from original data archive
##########################################
from flyingpigeon.datafetch import reanalyses as rl
from flyingpigeon.utils import get_variable
# from os.path import basename, splitext
from os import system
from netCDF4 import Dataset
from numpy import squeeze
try:
model_nc = rl(start=start.year,
end=end.year,
dataset=model,
variable=variable,
getlevel=getlevel)
LOGGER.info('reanalyses data fetched')
except Exception as ex:
msg = 'failed to get reanalyses data: {}'.format(ex)
LOGGER.exception(msg)
raise Exception(msg)
response.update_status('fetching data done', 15)
############################################################
# get the required bbox and time region from resource data
############################################################
response.update_status('subsetting region of interest', 17)
# from flyingpigeon.weatherregimes import get_level
# from flyingpigeon.ocgis_module import call
time_range = [start, end]
############################################################
# Block of level and domain selection for geop huge dataset
############################################################
LevMulti = False
# ===========================================================================================
if 'z' in variable:
tmp_total = []
origvar = get_variable(model_nc)
if LevMulti == False:
for z in model_nc:
b0 = call(resource=z,
variable=origvar,
level_range=[int(level), int(level)],
geom=bbox,
spatial_wrapping='wrap',
prefix='levdom_' + basename(z)[0:-3])
tmp_total.append(b0)
else:
# multiproc - no inprovements yet, need to check in hi perf machine...
# -----------------------
try:
import ctypes
import os
# TODO: This lib is for linux
mkl_rt = ctypes.CDLL('libmkl_rt.so')
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('Current number of threads: {}'.format(nth))
mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64)))
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('NEW number of threads: {}'.format(nth))
# TODO: Does it \/\/\/ work with default shell=False in subprocess... (?)
os.environ['MKL_NUM_THREADS'] = str(nth)
os.environ['OMP_NUM_THREADS'] = str(nth)
except Exception as ex:
msg = 'Failed to set THREADS: {}'.format(ex)
LOGGER.debug(msg)
# -----------------------
from multiprocessing import Pool
pool = Pool()
# from multiprocessing.dummy import Pool as ThreadPool
# pool = ThreadPool()
tup_var = [origvar] * len(model_nc)
tup_lev = [level] * len(model_nc)
tup_bbox = [bbox] * len(model_nc)
tup_args = zip(model_nc, tup_var, tup_lev, tup_bbox)
tmp_total = pool.map(ocgis_call_wrap, tup_args)
pool.close()
pool.join()
LOGGER.debug('Temporal subset files: {}'.format(tmp_total))
tmp_total = sorted(tmp_total,
key=lambda i: splitext(basename(i))[0])
inter_subset_tmp = call(resource=tmp_total,
variable=origvar,
time_range=time_range)
# FIXME: System calls to rm are dangerous! Use os.rmdir instead!
# Clean
for i in tmp_total:
tbr = 'rm -f {}'.format(i)
system(tbr)
# Create new variable
ds = Dataset(inter_subset_tmp, mode='a')
z_var = ds.variables.pop(origvar)
dims = z_var.dimensions
new_var = ds.createVariable('z{}'.format(level),
z_var.dtype,
dimensions=(dims[0], dims[2], dims[3]))
new_var[:, :, :] = squeeze(z_var[:, 0, :, :])
# new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()})
ds.close()
model_subset = call(inter_subset_tmp, variable='z{}'.format(level))
else:
model_subset = call(
resource=model_nc,
variable=variable,
geom=bbox,
spatial_wrapping='wrap',
time_range=time_range,
# conform_units_to=conform_units_to
)
# =============================================================================================
LOGGER.info('Dataset subset done: {}'.format(model_subset))
response.update_status('dataset subsetted', 18)
##############################################
# computing anomalies
##############################################
response.update_status('computing anomalies ', 19)
cycst = anualcycle.split('-')[0]
cycen = anualcycle.split('-')[1]
reference = [
dt.strptime(cycst, '%Y%m%d'),
dt.strptime(cycen, '%Y%m%d')
]
LOGGER.info('reference time: {}'.format(reference))
model_anomal = wr.get_anomalies(model_subset,
reference=reference,
method=method,
sseas=sseas) # , variable=variable)
#####################
# extracting season
#####################
response.update_status('normalizing data', 21)
model_season = wr.get_season(model_anomal, season=season)
response.update_status('anomalies computed and normalized', 24)
#######################
# call the R scripts
#######################
response.update_status('Start weather regime clustering ', 25)
import subprocess
from flyingpigeon import config
from os.path import curdir, join
try:
rworkspace = curdir
Rsrc = config.Rsrc_dir()
Rfile = 'weatherregimes_model.R'
infile = model_season # model_subset #model_ponderate
modelname = model
yr1 = start.year
yr2 = end.year
ip, output_graphics = mkstemp(dir=curdir, suffix='.pdf')
ip, file_pca = mkstemp(dir=curdir, suffix='.txt')
ip, file_class = mkstemp(dir=curdir, suffix='.Rdat')
# TODO: Rewrite this using os.path.join or pathlib libraries
args = [
'Rscript',
join(Rsrc, Rfile),
'%s/' % curdir,
'%s/' % Rsrc,
'%s' % infile,
'%s' % variable,
'%s' % output_graphics,
'%s' % file_pca,
'%s' % file_class,
'%s' % season,
'%s' % start.year,
'%s' % end.year,
'%s' % model_var,
'%s' % kappa
]
LOGGER.info('Rcall builded')
LOGGER.debug('ARGS: %s' % (args))
except Exception as ex:
msg = 'failed to build the R command: {}'.format(ex)
LOGGER.exception(msg)
raise Exception(msg)
try:
output, error = subprocess.Popen(
args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
LOGGER.info('R outlog info:\n {}'.format(output))
LOGGER.exception('R outlog errors:\n {}'.format(error))
if len(output) > 0:
response.update_status('**** weatherregime in R suceeded', 90)
else:
LOGGER.exception('No output returned from R call')
except Exception as ex:
msg = 'failed to run the R weatherregime: {}'.format(ex)
LOGGER.exception(msg)
raise Exception(msg)
response.update_status('Weather regime clustering done ', 93)
############################################
# set the outputs
############################################
response.update_status('Set the process outputs ', 95)
response.outputs['Routput_graphic'].file = output_graphics
response.outputs['output_pca'].file = file_pca
response.outputs['output_classification'].file = file_class
response.outputs['output_netcdf'].file = model_subset
response.update_status('done', 100)
return response
def execute(self):
logger.info('Start process')
init_process_logger('log.txt')
self.output_log.setValue('log.txt')
from datetime import datetime as dt
from flyingpigeon import weatherregimes as wr
from tempfile import mkstemp
################################
# reading in the input arguments
################################
try:
resource = self.getInputValues(identifier='resource')
url_Rdat = self.getInputValues(identifier='Rdat')[0]
url_dat = self.getInputValues(identifier='dat')[0]
url_ref_file = self.getInputValues(
identifier='netCDF') # can be None
season = self.getInputValues(identifier='season')[0]
period = self.getInputValues(identifier='period')[0]
anualcycle = self.getInputValues(identifier='anualcycle')[0]
except Exception as e:
logger.debug('failed to read in the arguments %s ' % e)
try:
start = dt.strptime(period.split('-')[0], '%Y%m%d')
end = dt.strptime(period.split('-')[1], '%Y%m%d')
# kappa = int(self.getInputValues(identifier='kappa')[0])
logger.info('period %s' % str(period))
logger.info('season %s' % str(season))
logger.info('read in the arguments')
logger.info('url_ref_file: %s' % url_ref_file)
logger.info('url_Rdat: %s' % url_Rdat)
logger.info('url_dat: %s' % url_dat)
except Exception as e:
logger.debug('failed to convert arguments %s ' % e)
############################
# fetching trainging data
############################
from flyingpigeon.utils import download, get_time
from os.path import abspath
try:
dat = abspath(download(url_dat))
Rdat = abspath(download(url_Rdat))
logger.info('training data fetched')
except Exception as e:
logger.error('failed to fetch training data %s' % e)
##########################################################
# get the required bbox and time region from resource data
##########################################################
# from flyingpigeon.weatherregimes import get_level
from flyingpigeon.ocgis_module import call
from flyingpigeon.utils import get_variable
time_range = [start, end]
variable = get_variable(resource)
if len(url_ref_file) > 0:
ref_file = download(url_ref_file[0])
model_subset = call(
resource=resource,
variable=variable,
time_range=
time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap',
regrid_destination=ref_file,
regrid_options='bil')
logger.info('Dataset subset with regridding done: %s ' %
model_subset)
else:
model_subset = call(
resource=resource,
variable=variable,
time_range=
time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap',
)
logger.info('Dataset time period extracted: %s ' % model_subset)
#######################
# computing anomalies
#######################
cycst = anualcycle.split('-')[0]
cycen = anualcycle.split('-')[0]
reference = [
dt.strptime(cycst, '%Y%m%d'),
dt.strptime(cycen, '%Y%m%d')
]
model_anomal = wr.get_anomalies(model_subset, reference=reference)
#####################
# extracting season
#####################
model_season = wr.get_season(model_anomal, season=season)
#######################
# call the R scripts
#######################
import shlex
import subprocess
from flyingpigeon import config
from os.path import curdir, exists, join
try:
rworkspace = curdir
Rsrc = config.Rsrc_dir()
Rfile = 'weatherregimes_projection.R'
yr1 = start.year
yr2 = end.year
time = get_time(model_season, format='%Y%m%d')
# ip, output_graphics = mkstemp(dir=curdir ,suffix='.pdf')
ip, file_pca = mkstemp(dir=curdir, suffix='.txt')
ip, file_class = mkstemp(dir=curdir, suffix='.Rdat')
ip, output_frec = mkstemp(dir=curdir, suffix='.txt')
args = [
'Rscript',
join(Rsrc, Rfile),
'%s/' % curdir,
'%s/' % Rsrc,
'%s' % model_season,
'%s' % variable,
'%s' % str(time).strip("[]").replace("'", "").replace(" ", ""),
# '%s' % output_graphics,
'%s' % dat,
'%s' % Rdat,
'%s' % file_pca,
'%s' % file_class,
'%s' % output_frec,
'%s' % season,
'%s' % start.year,
'%s' % end.year,
'%s' % 'MODEL'
]
logger.info('Rcall builded')
except Exception as e:
msg = 'failed to build the R command %s' % e
logger.error(msg)
raise Exception(msg)
try:
output, error = subprocess.Popen(
args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
# , shell=True
logger.info('R outlog info:\n %s ' % output)
logger.debug('R outlog errors:\n %s ' % error)
if len(output) > 0:
self.status.set('**** weatherregime in R suceeded', 90)
else:
logger.error('NO! output returned from R call')
except Exception as e:
msg = 'weatherregime in R %s ' % e
logger.error(msg)
raise Exception(msg)
#################
# set the outputs
#################
# self.Routput_graphic.setValue( output_graphics )
self.output_pca.setValue(file_pca)
self.output_classification.setValue(file_class)
self.output_netcdf.setValue(model_season)
self.output_frequency.setValue(output_frec)
def execute(self):
init_process_logger('log.txt')
self.output_log.setValue('log.txt')
logger.info('Start process')
from datetime import datetime as dt
from flyingpigeon import weatherregimes as wr
from tempfile import mkstemp
self.status.set('execution started at : %s ' % dt.now(), 5)
################################
# reading in the input arguments
################################
try:
logger.info('read in the arguments')
# resources = self.getInputValues(identifier='resources')
season = self.getInputValues(identifier='season')[0]
bbox_obj = self.BBox.getValue()
model_var = self.getInputValues(identifier='reanalyses')[0]
period = self.getInputValues(identifier='period')[0]
anualcycle = self.getInputValues(identifier='anualcycle')[0]
model, variable = model_var.split('_')
kappa = int(self.getInputValues(identifier='kappa')[0])
logger.info('period %s' % str(period))
logger.info('season %s' % str(season))
except Exception as e:
logger.debug('failed to read in the arguments %s ' % e)
try:
start = dt.strptime(period.split('-')[0], '%Y%m%d')
end = dt.strptime(period.split('-')[1], '%Y%m%d')
if bbox_obj is not None:
logger.info("bbox_obj={0}".format(bbox_obj.coords))
bbox = [
bbox_obj.coords[0][0], bbox_obj.coords[0][1],
bbox_obj.coords[1][0], bbox_obj.coords[1][1]
]
logger.info("bbox={0}".format(bbox))
else:
bbox = None
except Exception as e:
logger.debug('failed to transform BBOXObject %s ' % e)
###########################
# set the environment
###########################
self.status.set('fetching data from archive', 10)
try:
if model == 'NCEP':
if 'z' in variable:
level = variable.strip('z')
conform_units_to = None
else:
level = None
conform_units_to = 'hPa'
elif '20CRV2' in model:
if 'z' in variable:
level = variable.strip('z')
conform_units_to = None
else:
level = None
conform_units_to = 'hPa'
else:
logger.error('Reanalyses dataset not known')
logger.info('environment set')
except Exception as e:
msg = 'failed to set environment %s ' % e
logger.error(msg)
raise Exception(msg)
##########################################
# fetch Data from original data archive
##########################################
from flyingpigeon.datafetch import reanalyses as rl
try:
model_nc = rl(start=start.year,
end=end.year,
dataset=model,
variable=variable)
logger.info('reanalyses data fetched')
except Exception as e:
msg = 'failed to get reanalyses data %s' % e
logger.debug(msg)
raise Exception(msg)
self.status.set('fetching data done', 15)
############################################################
# get the required bbox and time region from resource data
############################################################
self.status.set('subsetting region of interest', 17)
# from flyingpigeon.weatherregimes import get_level
from flyingpigeon.ocgis_module import call
time_range = [start, end]
model_subset = call(
resource=model_nc,
variable=variable,
geom=bbox,
spatial_wrapping='wrap',
time_range=time_range,
# conform_units_to=conform_units_to
)
logger.info('Dataset subset done: %s ' % model_subset)
self.status.set('dataset subsetted', 19)
##############################################
# computing anomalies
##############################################
self.status.set('computing anomalies ', 19)
cycst = anualcycle.split('-')[0]
cycen = anualcycle.split('-')[0]
reference = [
dt.strptime(cycst, '%Y%m%d'),
dt.strptime(cycen, '%Y%m%d')
]
logger.debug('reference time: %s' % reference)
model_anomal = wr.get_anomalies(model_subset, reference=reference)
#####################
# extracting season
#####################
self.status.set('normalizing data', 21)
model_season = wr.get_season(model_anomal, season=season)
self.status.set('anomalies computed and normalized', 24)
#######################
# call the R scripts
#######################
self.status.set('Start weather regime clustering ', 25)
import shlex
import subprocess
from flyingpigeon import config
from os.path import curdir, exists, join
try:
rworkspace = curdir
Rsrc = config.Rsrc_dir()
Rfile = 'weatherregimes_model.R'
infile = model_season # model_subset #model_ponderate
modelname = model
yr1 = start.year
yr2 = end.year
ip, output_graphics = mkstemp(dir=curdir, suffix='.pdf')
ip, file_pca = mkstemp(dir=curdir, suffix='.txt')
ip, file_class = mkstemp(dir=curdir, suffix='.Rdat')
args = [
'Rscript',
join(Rsrc, Rfile),
'%s/' % curdir,
'%s/' % Rsrc,
'%s' % infile,
'%s' % variable,
'%s' % output_graphics,
'%s' % file_pca,
'%s' % file_class,
'%s' % season,
'%s' % start.year,
'%s' % end.year,
'%s' % model_var,
'%s' % kappa
]
logger.info('Rcall builded')
except Exception as e:
msg = 'failed to build the R command %s' % e
logger.debug(msg)
raise Exception(msg)
try:
output, error = subprocess.Popen(
args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
logger.info('R outlog info:\n %s ' % output)
logger.debug('R outlog errors:\n %s ' % error)
if len(output) > 0:
self.status.set('**** weatherregime in R suceeded', 90)
else:
logger.error('NO! output returned from R call')
except Exception as e:
msg = 'weatherregime in R %s ' % e
logger.error(msg)
raise Exception(msg)
self.status.set('Weather regime clustering done ', 80)
############################################
# set the outputs
############################################
self.status.set('Set the process outputs ', 95)
self.Routput_graphic.setValue(output_graphics)
self.output_pca.setValue(file_pca)
self.output_classification.setValue(file_class)
self.output_netcdf.setValue(model_season)
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
LOGGER.info('Start process')
response.update_status('execution started at : %s ' % dt.now(), 5)
################################
# reading in the input arguments
################################
try:
response.update_status('execution started at : {}'.format(dt.now()), 5)
################################
# reading in the input arguments
################################
LOGGER.info('read in the arguments')
resource = archiveextract(resource=[res.file for res in request.inputs['resource']])
# resources = self.getInputValues(identifier='resources')
season = request.inputs['season'][0].data
LOGGER.info('season %s', season)
if 'bbox' in request.inputs:
bbox = request.inputs['bbox'][0].data
bbox = [-80, 20, 50, 70]
else:
bbox = [-80, 20, 50, 70]
period = request.inputs['period'][0].data
LOGGER.info('period %s', period)
anualcycle = request.inputs['anualcycle'][0].data
kappa = request.inputs['kappa'][0].data
LOGGER.info('kappa %s', kappa)
start = dt.strptime(period.split('-')[0], '%Y%m%d')
end = dt.strptime(period.split('-')[1], '%Y%m%d')
LOGGER.debug('start: %s , end: %s ', start, end)
LOGGER.info('bbox %s', bbox)
LOGGER.info('period %s', period)
LOGGER.info('season %s', season)
except Exception as e:
msg = 'failed to read in the arguments'
LOGGER.exception(msg)
raise Exception(msg)
############################################################
# get the required bbox and time region from resource data
############################################################
response.update_status('start subsetting', 17)
# from flyingpigeon.weatherregimes import get_level
from flyingpigeon.ocgis_module import call
from flyingpigeon.utils import get_variable
time_range = [start, end]
variable = get_variable(resource)
model_subset = call(
resource=resource, variable=variable,
geom=bbox, spatial_wrapping='wrap', time_range=time_range, # conform_units_to=conform_units_to
)
LOGGER.info('Dataset subset done: %s ' % model_subset)
response.update_status('dataset subsetted', 19)
#####################
# computing anomalies
#####################
response.update_status('computing anomalies ', 19)
cycst = anualcycle.split('-')[0]
cycen = anualcycle.split('-')[0]
reference = [dt.strptime(cycst, '%Y%m%d'), dt.strptime(cycen, '%Y%m%d')]
model_anomal = wr.get_anomalies(model_subset, reference=reference)
###################
# extracting season
####################
model_season = wr.get_season(model_anomal, season=season)
response.update_status('values normalized', 20)
####################
# call the R scripts
####################
response.update_status('Start weather regime clustering ', 25)
import shlex
import subprocess
from flyingpigeon import config
from os.path import curdir, exists, join
try:
rworkspace = curdir
Rsrc = config.Rsrc_dir()
Rfile = 'weatherregimes_model.R'
infile = model_season # model_subset #model_ponderate
modelname = 'MODEL'
yr1 = start.year
yr2 = end.year
ip, output_graphics = mkstemp(dir=curdir, suffix='.pdf')
ip, file_pca = mkstemp(dir=curdir, suffix='.txt')
ip, file_class = mkstemp(dir=curdir, suffix='.Rdat')
args = ['Rscript', join(Rsrc, Rfile), '%s/' % curdir,
'%s/' % Rsrc, '%s' % infile, '%s' % variable,
'%s' % output_graphics, '%s' % file_pca,
'%s' % file_class, '%s' % season,
'%s' % start.year, '%s' % end.year,
'%s' % 'MODEL', '%s' % kappa]
LOGGER.info('Rcall builded')
except Exception as e:
msg = 'failed to build the R command %s' % e
LOGGER.error(msg)
raise Exception(msg)
try:
output, error = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
# ,shell=True
LOGGER.info('R outlog info:\n %s ' % output)
LOGGER.debug('R outlog errors:\n %s ' % error)
if len(output) > 0:
response.update_status('**** weatherregime in R suceeded', 90)
else:
LOGGER.error('NO! output returned from R call')
except Exception as e:
msg = 'weatherregime in R %s ' % e
LOGGER.error(msg)
raise Exception(msg)
response.update_status('Weather regime clustering done ', 90)
############################################
# set the outputs
############################################
response.update_status('Set the process outputs ', 95)
response.outputs['Routput_graphic'].file = output_graphics
response.outputs['output_pca'].file = file_pca
response.outputs['output_classification'].file = file_class
response.outputs['output_netcdf'].file = model_season
response.update_status('done', 100)
return response
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
response.update_status('execution started at : {}'.format(dt.now()), 5)
################################
# reading in the input arguments
################################
try:
LOGGER.info('read in the arguments')
# resources = self.getInputValues(identifier='resources')
season = request.inputs['season'][0].data
LOGGER.info('season %s', season)
period = request.inputs['period'][0].data
LOGGER.info('period %s', period)
anualcycle = request.inputs['anualcycle'][0].data
start = dt.strptime(period.split('-')[0], '%Y%m%d')
end = dt.strptime(period.split('-')[1], '%Y%m%d')
LOGGER.debug('start: %s , end: %s ' % (start, end))
resource = archiveextract(resource=rename_complexinputs(request.inputs['resource']))
# resource = archiveextract(resource=[res.file for res in request.inputs['resource']])
url_Rdat = request.inputs['Rdat'][0].data
url_dat = request.inputs['dat'][0].data
url_ref_file = request.inputs['netCDF'][0].data # can be None
# season = self.getInputValues(identifier='season')[0]
# period = self.getInputValues(identifier='period')[0]
# anualcycle = self.getInputValues(identifier='anualcycle')[0]
LOGGER.info('period %s' % str(period))
LOGGER.info('season %s' % str(season))
LOGGER.info('read in the arguments')
LOGGER.info('url_ref_file: %s' % url_ref_file)
LOGGER.info('url_Rdat: %s' % url_Rdat)
LOGGER.info('url_dat: %s' % url_dat)
except Exception as e:
LOGGER.debug('failed to convert arguments %s ' % e)
############################
# fetching trainging data
############################
try:
dat = abspath(download(url_dat))
Rdat = abspath(download(url_Rdat))
LOGGER.info('training data fetched')
except Exception as e:
LOGGER.error('failed to fetch training data %s' % e)
##########################################################
# get the required bbox and time region from resource data
##########################################################
# from flyingpigeon.weatherregimes import get_level
try:
from flyingpigeon.ocgis_module import call
from flyingpigeon.utils import get_variable
time_range = [start, end]
variable = get_variable(resource)
if len(url_ref_file) > 0:
ref_file = download(url_ref_file)
model_subset = call(
resource=resource, variable=variable,
time_range=time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap',
regrid_destination=ref_file, regrid_options='bil')
LOGGER.info('Dataset subset with regridding done: %s ' % model_subset)
else:
model_subset = call(
resource=resource, variable=variable,
time_range=time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap',
)
LOGGER.info('Dataset time period extracted: %s ' % model_subset)
except:
LOGGER.exception('failed to make a data subset ')
#######################
# computing anomalies
#######################
try:
cycst = anualcycle.split('-')[0]
cycen = anualcycle.split('-')[0]
reference = [dt.strptime(cycst, '%Y%m%d'), dt.strptime(cycen, '%Y%m%d')]
model_anomal = wr.get_anomalies(model_subset, reference=reference)
#####################
# extracting season
#####################
model_season = wr.get_season(model_anomal, season=season)
except:
LOGGER.exception('failed to compute anualcycle or seasons')
#######################
# call the R scripts
#######################
import shlex
import subprocess
from flyingpigeon import config
from os.path import curdir, exists, join
try:
rworkspace = curdir
Rsrc = config.Rsrc_dir()
Rfile = 'weatherregimes_projection.R'
yr1 = start.year
yr2 = end.year
time = get_time(model_season, format='%Y%m%d')
# ip, output_graphics = mkstemp(dir=curdir ,suffix='.pdf')
ip, file_pca = mkstemp(dir=curdir, suffix='.txt')
ip, file_class = mkstemp(dir=curdir, suffix='.Rdat')
ip, output_frec = mkstemp(dir=curdir, suffix='.txt')
args = ['Rscript', join(Rsrc, Rfile), '%s/' % curdir,
'%s/' % Rsrc,
'%s' % model_season,
'%s' % variable,
'%s' % str(time).strip("[]").replace("'", "").replace(" ", ""),
# '%s' % output_graphics,
'%s' % dat,
'%s' % Rdat,
'%s' % file_pca,
'%s' % file_class,
'%s' % output_frec,
'%s' % season,
'%s' % start.year,
'%s' % end.year,
'%s' % 'MODEL']
LOGGER.info('Rcall builded')
except Exception as e:
msg = 'failed to build the R command %s' % e
LOGGER.error(msg)
raise Exception(msg)
try:
output, error = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
# , shell=True
LOGGER.info('R outlog info:\n %s ' % output)
LOGGER.debug('R outlog errors:\n %s ' % error)
if len(output) > 0:
response.update_status('**** weatherregime in R suceeded', 90)
else:
LOGGER.error('NO! output returned from R call')
except Exception as e:
msg = 'weatherregime in R %s ' % e
LOGGER.error(msg)
raise Exception(msg)
#################
# set the outputs
#################
response.update_status('Set the process outputs ', 95)
response.outputs['output_pca'].file = file_pca
response.outputs['output_classification'].file = file_class
response.outputs['output_netcdf'].file = model_season
response.outputs['output_frequency'].file = output_frec
response.update_status('done', 100)
return response
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
LOGGER.info('Start process')
response.update_status('execution started at : {}'.format(dt.now()), 5)
################################
# reading in the input arguments
################################
try:
response.update_status(
'execution started at : {}'.format(dt.now()), 5)
################################
# reading in the input arguments
################################
LOGGER.info('read in the arguments')
resource = archiveextract(
resource=[res.file for res in request.inputs['resource']])
# If files are from different datasets.
# i.e. files: ...output1/slp.1999.nc and ...output2/slp.1997.nc will not be sorted with just .sort()
# So:
if type(resource) == list:
resource = sorted(
resource, key=lambda i: path.splitext(path.basename(i))[0])
else:
resource = [resource]
# resources = self.getInputValues(identifier='resources')
season = request.inputs['season'][0].data
LOGGER.info('season {}'.format(season))
# if 'bbox' in request.inputs:
# bbox = request.inputs['bbox'][0].data
# bbox = [-80, 20, 50, 70]
# else:
# bbox = [-80, 20, 50, 70]
bbox = []
bboxStr = request.inputs['BBox'][0].data
bboxStr = bboxStr.split(',')
bbox.append(float(bboxStr[0]))
bbox.append(float(bboxStr[2]))
bbox.append(float(bboxStr[1]))
bbox.append(float(bboxStr[3]))
LOGGER.debug('BBOX for ocgis: {}'.format(bbox))
LOGGER.debug('BBOX original: {}'.format(bboxStr))
period = request.inputs['period'][0].data
LOGGER.info('period: {}'.format(period))
anualcycle = request.inputs['anualcycle'][0].data
kappa = request.inputs['kappa'][0].data
LOGGER.info('kappa: {}'.format(kappa))
method = request.inputs['method'][0].data
LOGGER.info('Calc annual cycle with {}'.format(method))
sseas = request.inputs['sseas'][0].data
LOGGER.info('Annual cycle calc with {}'.format(sseas))
start = dt.strptime(period.split('-')[0], '%Y%m%d')
end = dt.strptime(period.split('-')[1], '%Y%m%d')
# OCGIS for models workaround - to catch 31 of Dec
start = dt.combine(start, dt_time(12, 0))
end = dt.combine(end, dt_time(12, 0))
cycst = anualcycle.split('-')[0]
cycen = anualcycle.split('-')[1]
reference = [
dt.strptime(cycst, '%Y%m%d'),
dt.strptime(cycen, '%Y%m%d')
]
LOGGER.debug('Reference start: {0}, end: {1}'.format(
reference[0], reference[1]))
reference[0] = dt.combine(reference[0], dt_time(12, 0))
reference[1] = dt.combine(reference[1], dt_time(12, 0))
LOGGER.debug('New Reference start: {0}, end: {1}'.format(
reference[0], reference[1]))
# Check if 360_day calendar (all months are exactly 30 days):
try:
if type(resource) is not list: resource = [resource]
modcal, calunits = get_calendar(resource[0])
if '360_day' in modcal:
if start.day == 31:
start = start.replace(day=30)
LOGGER.debug(
'Date has been changed for: {}'.format(start))
if end.day == 31:
end = end.replace(day=30)
LOGGER.debug(
'Date has been changed for: {}'.format(end))
if reference[0].day == 31:
reference[0] = reference[0].replace(day=30)
LOGGER.debug('Date has been changed for: {}'.format(
reference[0]))
if reference[1].day == 31:
reference[1] = reference[1].replace(day=30)
LOGGER.debug('Date has been changed for: {}'.format(
reference[1]))
except Exception as ex:
msg = 'Could not detect calendar: {}'.format(ex)
LOGGER.debug(msg)
raise Exception(msg)
LOGGER.debug('start: {0}, end: {1}'.format(start, end))
LOGGER.info('bbox: {}'.format(bbox))
LOGGER.info('period {}'.format(period))
LOGGER.info('season {}'.format(season))
except Exception as ex:
msg = 'failed to read in the arguments: {}'.format(ex)
LOGGER.exception(msg)
raise Exception(msg)
############################################################
# get the required bbox and time region from resource data
############################################################
response.update_status('start subsetting', 17)
# from flyingpigeon.weatherregimes import get_level
from flyingpigeon.ocgis_module import call
from flyingpigeon.utils import get_variable, get_timerange
time_range = [start, end]
tmp_resource = []
for re in resource:
s, e = get_timerange(re)
tmpSt = dt.strptime(s, '%Y%m%d')
tmpEn = dt.strptime(e, '%Y%m%d')
if (tmpSt <= end) and (tmpEn >= start):
tmp_resource.append(re)
LOGGER.debug('Selected file: {}'.format(re))
resource = tmp_resource
# Here start trick with z... levels and regriding...
# Otherwise call will give memory error for hires models with geop
# TODO: Add level and domain selection as in wps_analogs_model for 4D var.
variable = get_variable(resource)
model_subset = call(
resource=resource,
variable=variable,
geom=bbox,
spatial_wrapping='wrap',
time_range=time_range, # conform_units_to=conform_units_to
)
LOGGER.info('Dataset subset done: {}'.format(model_subset))
response.update_status('dataset subsetted', 18)
#####################
# computing anomalies
#####################
response.update_status('computing anomalies ', 19)
model_anomal = wr.get_anomalies(model_subset,
reference=reference,
method=method,
sseas=sseas)
###################
# extracting season
####################
model_season = wr.get_season(model_anomal, season=season)
response.update_status('values normalized', 20)
####################
# call the R scripts
####################
response.update_status('Start weather regime clustering ', 50)
import subprocess
from flyingpigeon import config
from os.path import curdir, join
try:
rworkspace = curdir
Rsrc = config.Rsrc_dir()
Rfile = 'weatherregimes_model.R'
infile = model_season # model_subset #model_ponderate
modelname = 'MODEL'
yr1 = start.year
yr2 = end.year
ip, output_graphics = mkstemp(dir=curdir, suffix='.pdf')
ip, file_pca = mkstemp(dir=curdir, suffix='.txt')
ip, file_class = mkstemp(dir=curdir, suffix='.Rdat')
# TODO: Rewrite this using os.path.join or pathlib libraries
args = [
'Rscript',
join(Rsrc, Rfile),
'%s/' % curdir,
'%s/' % Rsrc,
'%s' % infile,
'%s' % variable,
'%s' % output_graphics,
'%s' % file_pca,
'%s' % file_class,
'%s' % season,
'%s' % start.year,
'%s' % end.year,
'%s' % 'MODEL',
'%s' % kappa
]
LOGGER.info('Rcall builded')
LOGGER.debug('ARGS: {}'.format(args))
except Exception as ex:
msg = 'failed to build the R command {}'.format(ex)
LOGGER.error(msg)
raise Exception(msg)
try:
output, error = subprocess.Popen(
args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
# ,shell=True
LOGGER.info('R outlog info:\n {}'.format(output))
LOGGER.debug('R outlog errors:\n {}'.format(error))
if len(output) > 0:
response.update_status('**** weatherregime in R suceeded', 90)
else:
LOGGER.error('NO! output returned from R call')
except Exception as ex:
msg = 'failed to run the R weatherregime: {}'.format(ex)
LOGGER.exception(msg)
raise Exception(msg)
response.update_status('Weather regime clustering done ', 92)
############################################
# set the outputs
############################################
response.update_status('Set the process outputs ', 95)
# bla=bla
response.outputs['Routput_graphic'].file = output_graphics
response.outputs['output_pca'].file = file_pca
response.outputs['output_classification'].file = file_class
response.outputs['output_netcdf'].file = model_season
response.update_status('done', 100)
return response