def _handler(self, request, response):
# LOGGER.debug('CURDIR XXXX : %s ' % (abspath(curdir)))
# LOGGER.debug('WORKDIR XXXX : %s ' % (self.workdir))
os.chdir(self.workdir)
# LOGGER.debug('CURDIR XXXX : %s ' % (abspath(curdir)))
init_process_logger('log.txt')
# init_process_logger(os.path.join(self.workdir, 'log.txt'))
# response.outputs['output_log'].file = 'log.txt'
LOGGER.info('Start process')
response.update_status('execution started at : {}'.format(dt.now()), 5)
process_start_time = time.time() # measure process execution time ...
start_time = time.time() # measure init ...
################################
# reading in the input arguments
################################
try:
response.update_status('read input parameter : %s ' % dt.now(), 6)
refSt = request.inputs['refSt'][0].data
refEn = request.inputs['refEn'][0].data
dateSt = request.inputs['dateSt'][0].data
dateEn = request.inputs['dateEn'][0].data
seasonwin = request.inputs['seasonwin'][0].data
nanalog = request.inputs['nanalog'][0].data
timres = request.inputs['timeres'][0].data
bboxDef = '-20,40,30,70' # in general format
bbox = []
bboxStr = request.inputs['BBox'][0].data
LOGGER.debug('BBOX selected by user: %s ' % (bboxStr))
bboxStr = bboxStr.split(',')
# Checking for wrong cordinates and apply default if nesessary
if (abs(float(bboxStr[0])) > 180 or
abs(float(bboxStr[1]) > 180) or
abs(float(bboxStr[2]) > 90) or
abs(float(bboxStr[3])) > 90):
bboxStr = bboxDef # request.inputs['BBox'].default # .default doesn't work anymore!!!
LOGGER.debug('BBOX is out of the range, using default instead: %s ' % (bboxStr))
bboxStr = bboxStr.split(',')
bbox.append(float(bboxStr[0]))
bbox.append(float(bboxStr[2]))
bbox.append(float(bboxStr[1]))
bbox.append(float(bboxStr[3]))
LOGGER.debug('BBOX for ocgis: %s ' % (bbox))
LOGGER.debug('BBOX original: %s ' % (bboxStr))
normalize = request.inputs['normalize'][0].data
detrend = request.inputs['detrend'][0].data
plot = request.inputs['plot'][0].data
distance = request.inputs['dist'][0].data
outformat = request.inputs['outformat'][0].data
timewin = request.inputs['timewin'][0].data
model_var = request.inputs['reanalyses'][0].data
model, var = model_var.split('_')
LOGGER.info('input parameters set')
response.update_status('Read in and convert the arguments', 7)
except Exception as e:
msg = 'failed to read input prameter %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
######################################
# convert types and set environment
######################################
try:
response.update_status('Preparing enviroment converting arguments', 8)
LOGGER.debug('date: %s %s %s %s ' % (type(refSt), refEn, dateSt, dateSt))
start = min(refSt, dateSt)
end = max(refEn, dateEn)
if normalize == 'None':
seacyc = False
else:
seacyc = True
if outformat == 'ascii':
outformat = '.txt'
elif outformat == 'netCDF':
outformat = '.nc'
else:
LOGGER.exception('output format not valid')
except Exception as e:
msg = 'failed to set environment %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
###########################
# set the environment
###########################
response.update_status('fetching data from archive', 9)
try:
if model == 'NCEP':
getlevel = False
if 'z' in var:
level = var.strip('z')
# conform_units_to = None
else:
level = None
if var == 'precip':
var = 'pr_wtr'
# conform_units_to = 'hPa'
elif '20CRV2' in model:
getlevel = False
if 'z' in var:
level = var.strip('z')
# conform_units_to = None
else:
level = None
# conform_units_to = 'hPa'
else:
LOGGER.exception('Reanalyses dataset not known')
LOGGER.info('environment set for model: %s' % model)
except Exception:
msg = 'failed to set environment'
LOGGER.exception(msg)
raise Exception(msg)
##########################################
# fetch Data from original data archive
##########################################
# NOTE: If ref is say 1950 - 1990, and sim is just 1 week in 2017:
# - ALL the data will be downloaded, 1950 - 2017
try:
model_nc = rl(start=start.year,
end=end.year,
dataset=model,
variable=var, timres=timres, getlevel=getlevel)
LOGGER.info('reanalyses data fetched')
except Exception:
msg = 'failed to get reanalyses data'
LOGGER.exception(msg)
raise Exception(msg)
response.update_status('subsetting region of interest', 10)
LOGGER.debug("start and end time: %s - %s" % (start, end))
time_range = [start, end]
# Checking memory and dataset size
model_size = get_files_size(model_nc)
memory_avail = psutil.virtual_memory().available
thrs = 0.3 # 30%
if (model_size >= thrs * memory_avail):
ser_r = True
else:
ser_r = False
# ################################
# For 20CRV2 geopotential height, daily dataset for 100 years is about 50 Gb
# So it makes sense, to operate it step-by-step
# TODO: need to create dictionary for such datasets (for models as well)
# TODO: benchmark the method bellow for NCEP z500 for 60 years
# if ('20CRV2' in model) and ('z' in var):
if ('z' in var):
tmp_total = []
origvar = get_variable(model_nc)
for z in model_nc:
# tmp_n = 'tmp_%s' % (uuid.uuid1())
b0 = call(resource=z, variable=origvar, level_range=[int(level), int(level)], geom=bbox,
spatial_wrapping='wrap', prefix='levdom_' + os.path.basename(z)[0:-3])
tmp_total.append(b0)
tmp_total = sorted(tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0])
inter_subset_tmp = call(resource=tmp_total, variable=origvar, time_range=time_range)
# Clean
for i in tmp_total:
tbr = 'rm -f %s' % (i)
os.system(tbr)
# Create new variable
ds = Dataset(inter_subset_tmp, mode='a')
z_var = ds.variables.pop(origvar)
dims = z_var.dimensions
new_var = ds.createVariable('z%s' % level, z_var.dtype, dimensions=(dims[0], dims[2], dims[3]))
new_var[:, :, :] = squeeze(z_var[:, 0, :, :])
# new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()})
ds.close()
model_subset_tmp = call(inter_subset_tmp, variable='z%s' % level)
else:
if ser_r:
LOGGER.debug('Process reanalysis step-by-step')
tmp_total = []
for z in model_nc:
# tmp_n = 'tmp_%s' % (uuid.uuid1())
b0 = call(resource=z, variable=var, geom=bbox, spatial_wrapping='wrap',
prefix='Rdom_' + os.path.basename(z)[0:-3])
tmp_total.append(b0)
tmp_total = sorted(tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0])
model_subset_tmp = call(resource=tmp_total, variable=var, time_range=time_range)
else:
LOGGER.debug('Using whole dataset at once')
model_subset_tmp = call(resource=model_nc, variable=var,
geom=bbox, spatial_wrapping='wrap', time_range=time_range,
)
# If dataset is 20CRV2 the 6 hourly file should be converted to daily.
# Option to use previously 6h data from cache (if any) and not download daily files.
# Disabled for now
# if '20CRV2' in model:
# if timres == '6h':
# from cdo import Cdo
# cdo = Cdo(env=os.environ)
# model_subset = '%s.nc' % uuid.uuid1()
# tmp_f = '%s.nc' % uuid.uuid1()
# cdo_op = getattr(cdo, 'daymean')
# cdo_op(input=model_subset_tmp, output=tmp_f)
# sti = '00:00:00'
# cdo_op = getattr(cdo, 'settime')
# cdo_op(sti, input=tmp_f, output=model_subset)
# LOGGER.debug('File Converted from: %s to daily' % (timres))
# else:
# model_subset = model_subset_tmp
# else:
# model_subset = model_subset_tmp
# Remove \/\/\/ if work with 6h data...
model_subset = model_subset_tmp
LOGGER.info('Dataset subset done: %s ', model_subset)
response.update_status('dataset subsetted', 15)
# BLOCK OF DETRENDING of model_subset !
# Original model subset kept to further visualisaion if needed
# Now is issue with SLP:
# TODO 1 Keep trend as separate file
# TODO 2 Think how to add options to plot abomalies AND original data...
# May be do archive and simulation = call.. over NOT detrended data and keep it as well
# TODO 3 Check with faster smoother add removing trend of each grid
if detrend == 'None':
orig_model_subset = model_subset
else:
orig_model_subset = remove_mean_trend(model_subset, varname=var)
# ======================================
LOGGER.debug("get_input_subset_dataset took %s seconds.",
time.time() - start_time)
response.update_status('**** Input data fetched', 20)
########################
# input data preperation
########################
response.update_status('Start preparing input data', 30)
start_time = time.time() # measure data preperation ...
try:
# Construct descriptive filenames for the three files
# listed in config file
# refDatesString = dt.strftime(refSt, '%Y-%m-%d') + "_" + dt.strftime(refEn, '%Y-%m-%d')
# simDatesString = dt.strftime(dateSt, '%Y-%m-%d') + "_" + dt.strftime(dateEn, '%Y-%m-%d')
# Fix < 1900 issue...
refDatesString = refSt.isoformat().strip().split("T")[0] + "_" + refEn.isoformat().strip().split("T")[0]
simDatesString = dateSt.isoformat().strip().split("T")[0] + "_" + dateEn.isoformat().strip().split("T")[0]
archiveNameString = "base_" + var + "_" + refDatesString + '_%.1f_%.1f_%.1f_%.1f' \
% (bbox[0], bbox[2], bbox[1], bbox[3])
simNameString = "sim_" + var + "_" + simDatesString + '_%.1f_%.1f_%.1f_%.1f' \
% (bbox[0], bbox[2], bbox[1], bbox[3])
archive = call(resource=model_subset,
time_range=[refSt, refEn],
prefix=archiveNameString)
simulation = call(resource=model_subset, time_range=[dateSt, dateEn],
prefix=simNameString)
LOGGER.info('archive and simulation files generated: %s, %s'
% (archive, simulation))
except Exception as e:
msg = 'failed to prepare archive and simulation files %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
try:
if seacyc is True:
LOGGER.info('normalization function with method: %s '
% normalize)
seasoncyc_base, seasoncyc_sim = analogs.seacyc(
archive,
simulation,
method=normalize)
else:
seasoncyc_base = seasoncyc_sim = None
except Exception as e:
msg = 'failed to generate normalization files %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
output_file = 'output.txt'
files = [os.path.abspath(archive), os.path.abspath(simulation), output_file]
LOGGER.debug("Data preperation took %s seconds.",
time.time() - start_time)
############################
# generate the config file
############################
config_file = analogs.get_configfile(
files=files,
seasoncyc_base=seasoncyc_base,
seasoncyc_sim=seasoncyc_sim,
base_id=model,
sim_id=model,
timewin=timewin,
varname=var,
seacyc=seacyc,
cycsmooth=91,
nanalog=nanalog,
seasonwin=seasonwin,
distfun=distance,
outformat=outformat,
calccor=True,
silent=False,
# period=[dt.strftime(refSt, '%Y-%m-%d'), dt.strftime(refEn, '%Y-%m-%d')],
period=[refSt.isoformat().strip().split("T")[0], refEn.isoformat().strip().split("T")[0]],
bbox="{0[0]},{0[2]},{0[1]},{0[3]}".format(bbox))
response.update_status('generated config file', 40)
#######################
# CASTf90 call
#######################
start_time = time.time() # measure call castf90
# -----------------------
try:
import ctypes
# TODO: This lib is for linux
mkl_rt = ctypes.CDLL('libmkl_rt.so')
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('Current number of threads: %s' % (nth))
mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64)))
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('NEW number of threads: %s' % (nth))
# TODO: Does it \/\/\/ work with default shell=False in subprocess... (?)
os.environ['MKL_NUM_THREADS'] = str(nth)
os.environ['OMP_NUM_THREADS'] = str(nth)
except Exception as e:
msg = 'Failed to set THREADS %s ' % e
LOGGER.debug(msg)
# -----------------------
# ##### TEMPORAL WORKAROUND! With instaled hdf5-1.8.18 in anaconda ###############
# ##### MUST be removed after castf90 recompiled with the latest hdf version
# ##### NOT safe
os.environ['HDF5_DISABLE_VERSION_CHECK'] = '1'
# hdflib = os.path.expanduser("~") + '/anaconda/lib'
# hdflib = os.getenv("HOME") + '/anaconda/lib'
import pwd
hdflib = pwd.getpwuid(os.getuid()).pw_dir + '/anaconda/lib'
os.environ['LD_LIBRARY_PATH'] = hdflib
# ################################################################################
response.update_status('Start CASTf90 call', 50)
try:
# response.update_status('execution of CASTf90', 50)
cmd = ['analogue.out', config_file]
LOGGER.debug("castf90 command: %s", cmd)
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
LOGGER.info('analogue output:\n %s', output)
response.update_status('**** CASTf90 suceeded', 60)
except CalledProcessError as e:
msg = 'CASTf90 failed:\n{0}'.format(e.output)
LOGGER.exception(msg)
raise Exception(msg)
LOGGER.debug("castf90 took %s seconds.", time.time() - start_time)
# TODO: Add try - except for pdfs
if plot == 'Yes':
analogs_pdf = analogs.plot_analogs(configfile=config_file)
else:
analogs_pdf = 'dummy_plot.pdf'
with open(analogs_pdf, 'a'): os.utime(analogs_pdf, None)
response.update_status('preparing output', 70)
response.outputs['analog_pdf'].file = analogs_pdf
response.outputs['config'].file = config_file
response.outputs['analogs'].file = output_file
response.outputs['output_netcdf'].file = simulation
response.outputs['target_netcdf'].file = archive
if seacyc is True:
response.outputs['base_netcdf'].file = seasoncyc_base
response.outputs['sim_netcdf'].file = seasoncyc_sim
else:
# TODO: Still unclear how to overpass unknown number of outputs
dummy_base = 'dummy_base.nc'
dummy_sim = 'dummy_sim.nc'
with open(dummy_base, 'a'): os.utime(dummy_base, None)
with open(dummy_sim, 'a'): os.utime(dummy_sim, None)
response.outputs['base_netcdf'].file = dummy_base
response.outputs['sim_netcdf'].file = dummy_sim
########################
# generate analog viewer
########################
formated_analogs_file = analogs.reformat_analogs(output_file)
# response.outputs['formated_analogs'].storage = FileStorage()
response.outputs['formated_analogs'].file = formated_analogs_file
LOGGER.info('analogs reformated')
response.update_status('reformatted analog file', 80)
viewer_html = analogs.render_viewer(
# configfile=response.outputs['config'].get_url(),
configfile=config_file,
# datafile=response.outputs['formated_analogs'].get_url())
datafile=formated_analogs_file)
response.outputs['output'].file = viewer_html
response.update_status('Successfully generated analogs viewer', 90)
LOGGER.info('rendered pages: %s ', viewer_html)
response.update_status('execution ended', 100)
LOGGER.debug("total execution took %s seconds.",
time.time() - process_start_time)
response.outputs['output_log'].file = 'log.txt'
return response
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
LOGGER.info('Start process')
response.update_status('execution started at : {}'.format(dt.now()), 5)
process_start_time = time.time() # measure process execution time ...
start_time = time.time() # measure init ...
################################
# reading in the input arguments
################################
try:
response.update_status('read input parameter : %s ' % dt.now(), 6)
dateSt = request.inputs['dateSt'][0].data
dateEn = request.inputs['dateEn'][0].data
# timres = request.inputs['timeres'][0].data
timres = 'day'
season = request.inputs['season'][0].data
bboxDef = '-20,40,30,70' # in general format
bbox = []
bboxStr = request.inputs['BBox'][0].data
LOGGER.debug('BBOX selected by user: %s ' % (bboxStr))
bboxStr = bboxStr.split(',')
# Checking for wrong cordinates and apply default if nesessary
if (abs(float(bboxStr[0])) > 180 or
abs(float(bboxStr[1]) > 180) or
abs(float(bboxStr[2]) > 90) or
abs(float(bboxStr[3])) > 90):
bboxStr = bboxDef # request.inputs['BBox'].default # .default doesn't work anymore!!!
LOGGER.debug('BBOX is out of the range, using default instead: %s ' % (bboxStr))
bboxStr = bboxStr.split(',')
bbox.append(float(bboxStr[0]))
bbox.append(float(bboxStr[2]))
bbox.append(float(bboxStr[1]))
bbox.append(float(bboxStr[3]))
LOGGER.debug('BBOX for ocgis: %s ' % (bbox))
LOGGER.debug('BBOX original: %s ' % (bboxStr))
distance = request.inputs['dist'][0].data
method = request.inputs['method'][0].data
model_var = request.inputs['reanalyses'][0].data
model, var = model_var.split('_')
LOGGER.info('input parameters set')
response.update_status('Read in and convert the arguments', 7)
except Exception as e:
msg = 'failed to read input prameter %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
######################################
# convert types and set environment
######################################
start = dateSt
end = dateEn
###########################
# set the environment
###########################
response.update_status('fetching data from archive', 9)
try:
if model == 'NCEP':
getlevel = False
if 'z' in var:
level = var.strip('z')
# conform_units_to = None
else:
level = None
if var == 'precip':
var = 'pr_wtr'
# conform_units_to = 'hPa'
elif '20CRV2' in model:
getlevel = False
if 'z' in var:
level = var.strip('z')
# conform_units_to = None
else:
level = None
# conform_units_to = 'hPa'
else:
LOGGER.exception('Reanalyses dataset not known')
LOGGER.info('environment set for model: %s' % model)
except Exception:
msg = 'failed to set environment'
LOGGER.exception(msg)
raise Exception(msg)
##########################################
# fetch Data from original data archive
##########################################
# NOTE: If ref is say 1950 - 1990, and sim is just 1 week in 2017:
# - ALL the data will be downloaded, 1950 - 2017
try:
model_nc = rl(start=start.year,
end=end.year,
dataset=model,
variable=var, timres=timres, getlevel=getlevel)
LOGGER.info('reanalyses data fetched')
except Exception:
msg = 'failed to get reanalyses data'
LOGGER.exception(msg)
raise Exception(msg)
response.update_status('subsetting region of interest', 10)
# from flyingpigeon.weatherregimes import get_level
LOGGER.debug("start and end time: %s - %s" % (start, end))
time_range = [start, end]
# Checking memory and dataset size
model_size = get_files_size(model_nc)
memory_avail = psutil.virtual_memory().available
thrs = 0.5 # 50%
if (model_size >= thrs * memory_avail):
ser_r = True
else:
ser_r = False
# ################################
# For 20CRV2 geopotential height, daily dataset for 100 years is about 50 Gb
# So it makes sense, to operate it step-by-step
# TODO: need to create dictionary for such datasets (for models as well)
# TODO: benchmark the method bellow for NCEP z500 for 60 years
# if ('20CRV2' in model) and ('z' in var):
if ('z' in var):
tmp_total = []
origvar = get_variable(model_nc)
for z in model_nc:
# tmp_n = 'tmp_%s' % (uuid.uuid1())
b0 = call(resource=z, variable=origvar, level_range=[int(level), int(level)], geom=bbox,
spatial_wrapping='wrap', prefix='levdom_' + os.path.basename(z)[0:-3])
tmp_total.append(b0)
tmp_total = sorted(tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0])
inter_subset_tmp = call(resource=tmp_total, variable=origvar, time_range=time_range)
# Clean
for i in tmp_total:
tbr = 'rm -f %s' % (i)
os.system(tbr)
# Create new variable
ds = Dataset(inter_subset_tmp, mode='a')
z_var = ds.variables.pop(origvar)
dims = z_var.dimensions
new_var = ds.createVariable('z%s' % level, z_var.dtype, dimensions=(dims[0], dims[2], dims[3]))
new_var[:, :, :] = squeeze(z_var[:, 0, :, :])
# new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()})
ds.close()
model_subset_tmp = call(inter_subset_tmp, variable='z%s' % level)
else:
if ser_r:
LOGGER.debug('Process reanalysis step-by-step')
tmp_total = []
for z in model_nc:
# tmp_n = 'tmp_%s' % (uuid.uuid1())
b0 = call(resource=z, variable=var, geom=bbox, spatial_wrapping='wrap',
prefix='Rdom_' + os.path.basename(z)[0:-3])
tmp_total.append(b0)
tmp_total = sorted(tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0])
model_subset_tmp = call(resource=tmp_total, variable=var, time_range=time_range)
else:
LOGGER.debug('Using whole dataset at once')
model_subset_tmp = call(resource=model_nc, variable=var,
geom=bbox, spatial_wrapping='wrap', time_range=time_range,
)
# If dataset is 20CRV2 the 6 hourly file should be converted to daily.
# Option to use previously 6h data from cache (if any) and not download daily files.
if '20CRV2' in model:
if timres == '6h':
from cdo import Cdo
cdo = Cdo(env=os.environ)
model_subset = '%s.nc' % uuid.uuid1()
tmp_f = '%s.nc' % uuid.uuid1()
cdo_op = getattr(cdo, 'daymean')
cdo_op(input=model_subset_tmp, output=tmp_f)
sti = '00:00:00'
cdo_op = getattr(cdo, 'settime')
cdo_op(sti, input=tmp_f, output=model_subset)
LOGGER.debug('File Converted from: %s to daily' % (timres))
else:
model_subset = model_subset_tmp
else:
model_subset = model_subset_tmp
LOGGER.info('Dataset subset done: %s ', model_subset)
response.update_status('dataset subsetted', 15)
# ======================================
LOGGER.debug("get_input_subset_dataset took %s seconds.",
time.time() - start_time)
response.update_status('**** Input data fetched', 20)
########################
# input data preperation
########################
response.update_status('Start preparing input data', 30)
start_time = time.time() # measure data preperation ...
# -----------------------
# try:
# import ctypes
# # TODO: This lib is for linux
# mkl_rt = ctypes.CDLL('libmkl_rt.so')
# nth = mkl_rt.mkl_get_max_threads()
# LOGGER.debug('Current number of threads: %s' % (nth))
# mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64)))
# nth = mkl_rt.mkl_get_max_threads()
# LOGGER.debug('NEW number of threads: %s' % (nth))
# # TODO: Does it \/\/\/ work with default shell=False in subprocess... (?)
# os.environ['MKL_NUM_THREADS'] = str(nth)
# os.environ['OMP_NUM_THREADS'] = str(nth)
# except Exception as e:
# msg = 'Failed to set THREADS %s ' % e
# LOGGER.debug(msg)
# -----------------------
response.update_status('Start DIM calc', 50)
# Calculation of Local Dimentsions ==================
LOGGER.debug('Calculation of the distances using: %s metric' % (distance))
LOGGER.debug('Calculation of the dims with: %s' % (method))
dim_filename = '%s.txt' % model
tmp_dim_fn = '%s.txt' % uuid.uuid1()
Rsrc = config.Rsrc_dir()
if (method == 'Python'):
try:
l_dist, l_theta = localdims(resource=model_subset, variable=var, distance=str(distance))
response.update_status('**** Dims with Python suceeded', 60)
except:
LOGGER.exception('NO! output returned from Python call')
if (method == 'Python_wrap'):
try:
l_dist, l_theta = localdims_par(resource=model_subset, variable=var, distance=str(distance))
response.update_status('**** Dims with Python suceeded', 60)
except:
LOGGER.exception('NO! output returned from Python call')
if (method == 'R'):
# from os.path import join
Rfile = 'localdimension_persistence_fullD.R'
args = ['Rscript', os.path.join(Rsrc, Rfile),
'%s' % model_subset, '%s' % var,
'%s' % tmp_dim_fn]
LOGGER.info('Rcall builded')
LOGGER.debug('ARGS: %s' % (args))
try:
output, error = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
LOGGER.info('R outlog info:\n %s ' % output)
LOGGER.exception('R outlog errors:\n %s ' % error)
if len(output) > 0:
response.update_status('**** Dims with R suceeded', 60)
else:
LOGGER.exception('NO! output returned from R call')
# HERE READ DATA FROM TEXT FILES
R_resdim = loadtxt(fname=tmp_dim_fn, delimiter=',')
l_theta = R_resdim[:, 0]
l_dist = R_resdim[:, 1]
except:
msg = 'Dim with R'
LOGGER.exception(msg)
raise Exception(msg)
if (method == 'R_wrap'):
# from os.path import join
Rfile = 'localdimension_persistence_serrD.R'
args = ['Rscript', os.path.join(Rsrc, Rfile),
'%s' % model_subset, '%s' % var,
'%s' % tmp_dim_fn]
LOGGER.info('Rcall builded')
LOGGER.debug('ARGS: %s' % (args))
try:
output, error = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
LOGGER.info('R outlog info:\n %s ' % output)
LOGGER.exception('R outlog errors:\n %s ' % error)
if len(output) > 0:
response.update_status('**** Dims with R_wrap suceeded', 60)
else:
LOGGER.exception('NO! output returned from R call')
# HERE READ DATA FROM TEXT FILES
R_resdim = loadtxt(fname=tmp_dim_fn, delimiter=',')
l_theta = R_resdim[:, 0]
l_dist = R_resdim[:, 1]
except:
msg = 'Dim with R_wrap'
LOGGER.exception(msg)
raise Exception(msg)
try:
res_times = get_time(model_subset)
except:
LOGGER.debug('Not standard calendar')
res_times = analogs.get_time_nc(model_subset)
# plot 1
ld_pdf = analogs.pdf_from_ld(x=l_dist, y=l_theta)
#
res_times=[res_times[i].isoformat().strip().split("T")[0].replace('-','') for i in range(len(res_times))]
# concatenation of values
concat_vals = column_stack([res_times, l_theta, l_dist])
savetxt(dim_filename, concat_vals, fmt='%s', delimiter=',')
# output season
try:
seas = _TIMEREGIONS_[season]['month'] # [12, 1, 2]
LOGGER.info('Season to grep from TIMEREGIONS: %s ' % season)
LOGGER.info('Season N to grep from TIMEREGIONS: %s ' % seas)
except:
LOGGER.info('No months in TIMEREGIONS, moving to months')
try:
seas = _MONTHS_[season]['month'] # [1] or [2] or ...
LOGGER.info('Season to grep from MONTHS: %s ' % season)
LOGGER.info('Season N to grep from MONTHS: %s ' % seas)
except:
seas = [1,2,3,4,5,6,7,8,9,10,11,12]
ind = []
# TODO: change concat_vals[i][0][4:6] to dt_obj.month !!!
for i in range(len(res_times)):
if (int(concat_vals[i][0][4:6]) in seas[:]):
ind.append(i)
sf = column_stack([concat_vals[i] for i in ind]).T
seas_dim_filename = season + '_' + dim_filename
savetxt(seas_dim_filename, sf, fmt='%s', delimiter=',')
# -------------------------- plot with R ---------------
R_plot_file = 'plot_csv.R'
ld2_pdf = 'local_dims.pdf'
ld2_seas_pdf = season + '_local_dims.pdf'
args = ['Rscript', os.path.join(Rsrc, R_plot_file),
'%s' % dim_filename,
'%s' % ld2_pdf]
try:
output, error = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
LOGGER.info('R outlog info:\n %s ' % output)
LOGGER.exception('R outlog errors:\n %s ' % error)
except:
msg = 'Could not produce plot'
LOGGER.exception(msg)
# TODO: Here need produce empty pdf(s) to pass to output
args = ['Rscript', os.path.join(Rsrc, R_plot_file),
'%s' % seas_dim_filename,
'%s' % ld2_seas_pdf]
try:
output, error = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
LOGGER.info('R outlog info:\n %s ' % output)
LOGGER.exception('R outlog errors:\n %s ' % error)
except:
msg = 'Could not produce plot'
LOGGER.exception(msg)
# TODO: Here need produce empty pdf(s) to pass to output
#
# ====================================================
response.update_status('preparing output', 80)
response.outputs['ldist'].file = dim_filename
response.outputs['ldist_seas'].file = seas_dim_filename
response.outputs['ld_pdf'].file = ld_pdf
response.outputs['ld2_pdf'].file = ld2_pdf
response.outputs['ld2_seas_pdf'].file = ld2_seas_pdf
response.update_status('execution ended', 100)
LOGGER.debug("total execution took %s seconds.",
time.time() - process_start_time)
return response
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
LOGGER.info('Start process')
response.update_status('execution started at : {}'.format(dt.now()), 5)
process_start_time = time.time() # measure process execution time ...
start_time = time.time() # measure init ...
################################
# reading in the input arguments
################################
try:
response.update_status('read input parameter : %s ' % dt.now(), 7)
refSt = request.inputs['refSt'][0].data
#refEn = request.inputs['refEn'][0].data
refEn = dt.strptime(
'%s%s%s' % (dt.now().year, dt.now().month, dt.now().day),
'%Y%m%d')
refEn = refEn - timedelta(days=3)
dateSt = request.inputs['dateSt'][0].data
#dateEn = request.inputs['dateEn'][0].data
dateEn = dt.strptime(
'%s%s%s' % (dt.now().year, dt.now().month, dt.now().day),
'%Y%m%d')
dateEn = dateEn - timedelta(days=3)
seasonwin = request.inputs['seasonwin'][0].data
nanalog = request.inputs['nanalog'][0].data
bboxDef = '-80,50,20,70' # in general format
bbox = []
bboxStr = request.inputs['BBox'][0].data
LOGGER.debug('BBOX selected by user: %s ' % (bboxStr))
bboxStr = bboxStr.split(',')
# Checking for wrong cordinates and apply default if nesessary
if (abs(float(bboxStr[0])) > 180 or abs(float(bboxStr[1]) > 180)
or abs(float(bboxStr[2]) > 90)
or abs(float(bboxStr[3])) > 90):
bboxStr = bboxDef # request.inputs['BBox'].default # .default doesn't work anymore!!!
LOGGER.debug(
'BBOX is out of the range, using default instead: %s ' %
(bboxStr))
bboxStr = bboxStr.split(',')
bbox.append(float(bboxStr[0]))
bbox.append(float(bboxStr[2]))
bbox.append(float(bboxStr[1]))
bbox.append(float(bboxStr[3]))
LOGGER.debug('BBOX for ocgis: %s ' % (bbox))
LOGGER.debug('BBOX original: %s ' % (bboxStr))
normalize = request.inputs['normalize'][0].data
detrend = request.inputs['detrend'][0].data
distance = request.inputs['dist'][0].data
outformat = request.inputs['outformat'][0].data
timewin = request.inputs['timewin'][0].data
model_var = request.inputs['reanalyses'][0].data
model, var = model_var.split('_')
LOGGER.info('input parameters set')
response.update_status('Read in and convert the arguments', 8)
except Exception as e:
msg = 'failed to read input prameter %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
######################################
# convert types and set environment
######################################
try:
response.update_status('Preparing enviroment converting arguments',
9)
LOGGER.debug('date: %s %s %s %s ' %
(type(refSt), refEn, dateSt, dateSt))
start = min(refSt, dateSt)
end = max(refEn, dateEn)
if normalize == 'None':
seacyc = False
else:
seacyc = True
if outformat == 'ascii':
outformat = '.txt'
elif outformat == 'netCDF':
outformat = '.nc'
else:
LOGGER.exception('output format not valid')
except Exception as e:
msg = 'failed to set environment %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
###########################
# set the environment
###########################
response.update_status('fetching data from archive', 10)
# We work only with NCEP
getlevel = False
if 'z' in var:
level = var.strip('z')
else:
level = None
##########################################
# fetch Data from original data archive
##########################################
try:
model_nc = rl(start=start.year,
end=end.year,
dataset=model,
variable=var,
getlevel=getlevel)
LOGGER.info('reanalyses data fetched')
except Exception:
msg = 'failed to get reanalyses data'
LOGGER.exception(msg)
raise Exception(msg)
response.update_status('subsetting region of interest', 17)
LOGGER.debug("start and end time: %s - %s" % (start, end))
time_range = [start, end]
# Checking memory and dataset size
m_size = get_files_size(model_nc)
memory_avail = psutil.virtual_memory().available
thrs = 0.2 # 20%
if (m_size >= thrs * memory_avail):
ser_r = True
else:
ser_r = False
LOGGER.debug('Available Memory: %s ' % (memory_avail))
LOGGER.debug('Dataset size: %s ' % (m_size))
LOGGER.debug('Threshold: %s ' % (thrs * memory_avail))
LOGGER.debug('Serial or at once: %s ' % (ser_r))
# if ('20CRV2' in model) and ('z' in var):
if ('z' in var):
tmp_total = []
origvar = get_variable(model_nc)
for z in model_nc:
tmp_n = 'tmp_%s' % (uuid.uuid1())
b0 = call(resource=z,
variable=origvar,
level_range=[int(level), int(level)],
geom=bbox,
spatial_wrapping='wrap',
prefix='levdom_' + os.path.basename(z)[0:-3])
tmp_total.append(b0)
tmp_total = sorted(
tmp_total,
key=lambda i: os.path.splitext(os.path.basename(i))[0])
inter_subset_tmp = call(resource=tmp_total,
variable=origvar,
time_range=time_range)
# Clean
for i in tmp_total:
tbr = 'rm -f %s' % (i)
os.system(tbr)
# Create new variable
ds = Dataset(inter_subset_tmp, mode='a')
z_var = ds.variables.pop(origvar)
dims = z_var.dimensions
new_var = ds.createVariable('z%s' % level,
z_var.dtype,
dimensions=(dims[0], dims[2], dims[3]))
new_var[:, :, :] = squeeze(z_var[:, 0, :, :])
# new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()})
ds.close()
model_subset_tmp = call(inter_subset_tmp, variable='z%s' % level)
else:
if ser_r:
LOGGER.debug('Process reanalysis step-by-step')
tmp_total = []
for z in model_nc:
tmp_n = 'tmp_%s' % (uuid.uuid1())
b0 = call(resource=z,
variable=var,
geom=bbox,
spatial_wrapping='wrap',
prefix='Rdom_' + os.path.basename(z)[0:-3])
tmp_total.append(b0)
tmp_total = sorted(
tmp_total,
key=lambda i: os.path.splitext(os.path.basename(i))[0])
model_subset_tmp = call(resource=tmp_total,
variable=var,
time_range=time_range)
# Clean
for i in tmp_total:
tbr = 'rm -f %s' % (i)
os.system(tbr)
else:
LOGGER.debug('Using whole dataset at once')
model_subset_tmp = call(
resource=model_nc,
variable=var,
geom=bbox,
spatial_wrapping='wrap',
time_range=time_range,
)
# Rest from 20CRV...
model_subset = model_subset_tmp
LOGGER.info('Dataset subset done: %s ', model_subset)
response.update_status('dataset subsetted', 19)
# BLOCK OF DETRENDING of model_subset !
# Original model subset kept to further visualisaion if needed
# Now is issue with SLP:
# TODO 1 Keep trend as separate file
# TODO 2 Think how to add options to plot abomalies AND original data...
# May be do archive and simulation = call.. over NOT detrended data and keep it as well
# TODO 3 Check with faster smoother add removing trend of each grid
if detrend == 'None':
orig_model_subset = model_subset
else:
orig_model_subset = remove_mean_trend(model_subset, varname=var)
# ======================================
LOGGER.debug("get_input_subset_dataset took %s seconds.",
time.time() - start_time)
response.update_status('**** Input data fetched', 20)
########################
# input data preperation
########################
response.update_status('Start preparing input data', 22)
start_time = time.time() # measure data preperation ...
try:
# Construct descriptive filenames for the three files
# listed in config file
# TODO check strftime for years <1900 (!)
refDatesString = dt.strftime(
refSt, '%Y-%m-%d') + "_" + dt.strftime(refEn, '%Y-%m-%d')
simDatesString = dt.strftime(
dateSt, '%Y-%m-%d') + "_" + dt.strftime(dateEn, '%Y-%m-%d')
archiveNameString = "base_" + var + "_" + refDatesString + '_%.1f_%.1f_%.1f_%.1f' \
% (bbox[0], bbox[2], bbox[1], bbox[3])
simNameString = "sim_" + var + "_" + simDatesString + '_%.1f_%.1f_%.1f_%.1f' \
% (bbox[0], bbox[2], bbox[1], bbox[3])
archive = call(resource=model_subset,
time_range=[refSt, refEn],
prefix=archiveNameString)
simulation = call(resource=model_subset,
time_range=[dateSt, dateEn],
prefix=simNameString)
LOGGER.info('archive and simulation files generated: %s, %s' %
(archive, simulation))
except Exception as e:
msg = 'failed to prepare archive and simulation files %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
try:
if seacyc is True:
LOGGER.info('normalization function with method: %s ' %
normalize)
seasoncyc_base, seasoncyc_sim = analogs.seacyc(
archive, simulation, method=normalize)
else:
seasoncyc_base = seasoncyc_sim = None
except Exception as e:
msg = 'failed to generate normalization files %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
output_file = 'output.txt'
files = [
os.path.abspath(archive),
os.path.abspath(simulation), output_file
]
LOGGER.debug("Data preperation took %s seconds.",
time.time() - start_time)
############################
# generate the config file
############################
config_file = analogs.get_configfile(
files=files,
seasoncyc_base=seasoncyc_base,
seasoncyc_sim=seasoncyc_sim,
base_id=model,
sim_id=model,
timewin=timewin,
varname=var,
seacyc=seacyc,
cycsmooth=91,
nanalog=nanalog,
seasonwin=seasonwin,
distfun=distance,
outformat=outformat,
calccor=True,
silent=False,
period=[
dt.strftime(refSt, '%Y-%m-%d'),
dt.strftime(refEn, '%Y-%m-%d')
],
bbox="{0[0]},{0[2]},{0[1]},{0[3]}".format(bbox))
response.update_status('generated config file', 25)
#######################
# CASTf90 call
#######################
start_time = time.time() # measure call castf90
#-----------------------
try:
import ctypes
# TODO: This lib is for linux
mkl_rt = ctypes.CDLL('libmkl_rt.so')
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('Current number of threads: %s' % (nth))
mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64)))
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('NEW number of threads: %s' % (nth))
# TODO: Does it \/\/\/ work with default shell=False in subprocess... (?)
os.environ['MKL_NUM_THREADS'] = str(nth)
os.environ['OMP_NUM_THREADS'] = str(nth)
except Exception as e:
msg = 'Failed to set THREADS %s ' % e
LOGGER.debug(msg)
#-----------------------
# ##### TEMPORAL WORKAROUND! With instaled hdf5-1.8.18 in anaconda ###############
# ##### MUST be removed after castf90 recompiled with the latest hdf version
# ##### NOT safe
os.environ['HDF5_DISABLE_VERSION_CHECK'] = '1'
#hdflib = os.path.expanduser("~") + '/anaconda/lib'
#hdflib = os.getenv("HOME") + '/anaconda/lib'
import pwd
hdflib = pwd.getpwuid(os.getuid()).pw_dir + '/anaconda/lib'
os.environ['LD_LIBRARY_PATH'] = hdflib
# ################################################################################
response.update_status('Start CASTf90 call', 30)
try:
# response.update_status('execution of CASTf90', 50)
cmd = ['analogue.out', config_file]
LOGGER.debug("castf90 command: %s", cmd)
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
LOGGER.info('analogue output:\n %s', output)
response.update_status('**** CASTf90 suceeded', 40)
except CalledProcessError as e:
msg = 'CASTf90 failed:\n{0}'.format(e.output)
LOGGER.exception(msg)
raise Exception(msg)
LOGGER.debug("castf90 took %s seconds.", time.time() - start_time)
# --------------- R cont analogs calcs -----------------------------------
#######################
# call the R scripts
#######################
response.update_status(
'Start calculation of the stats and Return Periods ', 50)
import shlex
# import subprocess
from blackswan import config
from blackswan.visualisation import pdfmerge
from os.path import curdir, exists, join
try:
#rworkspace = curdir
Rsrc = config.Rsrc_dir()
Rfile = 'analogs_diags-prox.R'
Rdatfile = 'analogs_RT.Rdat'
probs_c = 0.7
probs_n = 0.3
args = [
'Rscript',
join(Rsrc, Rfile),
'%s' % output_file,
'%s' % probs_c,
'%s' % probs_n,
'%s' % Rdatfile
]
LOGGER.info('Rcall builded')
LOGGER.debug('ARGS: %s' % (args))
except:
msg = 'failed to build the R command'
LOGGER.exception(msg)
raise Exception(msg)
try:
output, error = subprocess.Popen(
args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
LOGGER.info('R outlog info:\n %s ' % output)
LOGGER.exception('R outlog errors:\n %s ' % error)
if len(output) > 0:
response.update_status('**** Return Periods with R suceeded',
60)
else:
LOGGER.exception('NO! output returned from R call')
analogs_pdf = pdfmerge(
['analogs_score-diags_new.pdf', 'analogs_RP-diags_new.pdf'])
except:
msg = 'ReturnPeriods in R'
LOGGER.exception(msg)
raise Exception(msg)
response.update_status('Calculation of Return Periods done ', 70)
# --------------- END of R cont analogs calcs ----------------------------
response.update_status('preparing output', 75)
response.outputs['analog_pdf'].file = analogs_pdf
response.outputs['config'].file = config_file
response.outputs['analogs'].file = output_file
response.outputs['output_netcdf'].file = simulation
response.outputs['target_netcdf'].file = archive
if seacyc is True:
response.outputs['base_netcdf'].file = seasoncyc_base
response.outputs['sim_netcdf'].file = seasoncyc_sim
else:
# TODO: Still unclear how to overpass unknown number of outputs
dummy_base = 'dummy_base.nc'
dummy_sim = 'dummy_sim.nc'
with open(dummy_base, 'a'):
os.utime(dummy_base, None)
with open(dummy_sim, 'a'):
os.utime(dummy_sim, None)
response.outputs['base_netcdf'].file = dummy_base
response.outputs['sim_netcdf'].file = dummy_sim
########################
# generate analog viewer
########################
formated_analogs_file = analogs.reformat_analogs(output_file)
# response.outputs['formated_analogs'].storage = FileStorage()
response.outputs['formated_analogs'].file = formated_analogs_file
LOGGER.info('analogs reformated')
response.update_status('reformatted analog file', 80)
viewer_html = analogs.render_viewer(
# configfile=response.outputs['config'].get_url(),
configfile=config_file,
# datafile=response.outputs['formated_analogs'].get_url())
datafile=formated_analogs_file)
response.outputs['output'].file = viewer_html
response.update_status('Successfully generated analogs viewer', 90)
LOGGER.info('rendered pages: %s ', viewer_html)
response.update_status('execution ended', 100)
LOGGER.debug("total execution took %s seconds.",
time.time() - process_start_time)
return response
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
LOGGER.info('Start process')
from datetime import datetime as dt
from blackswan import weatherregimes as wr
from tempfile import mkstemp
response.update_status('execution started at : {}'.format(dt.now()), 5)
################################
# reading in the input arguments
################################
LOGGER.info('read in the arguments')
# resources = self.getInputValues(identifier='resources')
season = request.inputs['season'][0].data
LOGGER.info('season %s', season)
bboxDef = '-80,50,20,70' # in general format
bbox = []
bboxStr = request.inputs['BBox'][0].data
LOGGER.debug('BBOX selected by user: %s ' % (bboxStr))
bboxStr = bboxStr.split(',')
# Checking for wrong cordinates and apply default if nesessary
if (abs(float(bboxStr[0])) > 180 or abs(float(bboxStr[1]) > 180)
or abs(float(bboxStr[2]) > 90) or abs(float(bboxStr[3])) > 90):
bboxStr = bboxDef # request.inputs['BBox'].default # .default doesn't work anymore!!!
LOGGER.debug(
'BBOX is out of the range, using default instead: %s ' %
(bboxStr))
bboxStr = bboxStr.split(',')
bbox.append(float(bboxStr[0]))
bbox.append(float(bboxStr[2]))
bbox.append(float(bboxStr[1]))
bbox.append(float(bboxStr[3]))
LOGGER.debug('BBOX for ocgis: %s ' % (bbox))
LOGGER.debug('BBOX original: %s ' % (bboxStr))
model_var = request.inputs['reanalyses'][0].data
model, variable = model_var.split('_')
period = request.inputs['period'][0].data
LOGGER.info('period %s', period)
anualcycle = request.inputs['anualcycle'][0].data
kappa = request.inputs['kappa'][0].data
LOGGER.info('kappa %s', kappa)
method = request.inputs['method'][0].data
LOGGER.info('Calc annual cycle with %s', method)
sseas = request.inputs['sseas'][0].data
LOGGER.info('Annual cycle calc with %s', sseas)
start = dt.strptime(period.split('-')[0], '%Y%m%d')
end = dt.strptime(period.split('-')[1], '%Y%m%d')
LOGGER.debug('start: %s , end: %s ' % (start, end))
###########################
# set the environment
###########################
response.update_status('fetching data from archive', 10)
try:
if model == 'NCEP':
getlevel = False
if 'z' in variable:
level = variable.strip('z')
# conform_units_to = None
else:
level = None
# conform_units_to = 'hPa'
elif '20CRV2' in model:
getlevel = False
if 'z' in variable:
level = variable.strip('z')
# conform_units_to = None
else:
level = None
# conform_units_to = 'hPa'
else:
LOGGER.exception('Reanalyses dataset not known')
LOGGER.info('environment set for model: %s' % model)
except:
msg = 'failed to set environment'
LOGGER.exception(msg)
raise Exception(msg)
##########################################
# fetch Data from original data archive
##########################################
from blackswan.datafetch import reanalyses as rl
from blackswan.utils import get_variable
# from os.path import basename, splitext
from os import system
from netCDF4 import Dataset
from numpy import squeeze
try:
model_nc = rl(start=start.year,
end=end.year,
dataset=model,
variable=variable,
getlevel=getlevel)
LOGGER.info('reanalyses data fetched')
except:
msg = 'failed to get reanalyses data'
LOGGER.exception(msg)
raise Exception(msg)
response.update_status('fetching data done', 20)
############################################################
# get the required bbox and time region from resource data
############################################################
response.update_status('subsetting region of interest', 30)
time_range = [start, end]
############################################################
# Block of level and domain selection for geop huge dataset
############################################################
LevMulti = False
# ===========================================================================================
# Temporary add step-by-step also for pressure... for slow VM machine...
if ('z' in variable) or ('p' in variable):
tmp_total = []
origvar = get_variable(model_nc)
if ('z' in variable):
level_range = [int(level), int(level)]
else:
level_range = None
if (LevMulti == False):
for z in model_nc:
b0 = call(resource=z,
variable=origvar,
level_range=level_range,
geom=bbox,
spatial_wrapping='wrap',
prefix='levdom_' + basename(z)[0:-3])
tmp_total.append(b0)
else:
# multiproc - no inprovements yet, need to check in hi perf machine...
# -----------------------
try:
import ctypes
import os
# TODO: This lib is for linux
mkl_rt = ctypes.CDLL('libmkl_rt.so')
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('Current number of threads: %s' % (nth))
mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64)))
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('NEW number of threads: %s' % (nth))
# TODO: Does it \/\/\/ work with default shell=False in subprocess... (?)
os.environ['MKL_NUM_THREADS'] = str(nth)
os.environ['OMP_NUM_THREADS'] = str(nth)
except Exception as e:
msg = 'Failed to set THREADS %s ' % e
LOGGER.debug(msg)
# -----------------------
from multiprocessing import Pool
pool = Pool()
# from multiprocessing.dummy import Pool as ThreadPool
# pool = ThreadPool()
tup_var = [origvar] * len(model_nc)
tup_lev = [level] * len(model_nc)
tup_bbox = [bbox] * len(model_nc)
tup_args = zip(model_nc, tup_var, tup_lev, tup_bbox)
tmp_total = pool.map(ocgis_call_wrap, tup_args)
pool.close()
pool.join()
LOGGER.debug('Temporal subset files: %s' % (tmp_total))
tmp_total = sorted(tmp_total,
key=lambda i: splitext(basename(i))[0])
inter_subset_tmp = call(resource=tmp_total,
variable=origvar,
time_range=time_range)
# Clean
for i in tmp_total:
tbr = 'rm -f %s' % (i)
system(tbr)
if ('z' in variable):
# Create new variable for Z geop
ds = Dataset(inter_subset_tmp, mode='a')
z_var = ds.variables.pop(origvar)
dims = z_var.dimensions
new_var = ds.createVariable('z%s' % level,
z_var.dtype,
dimensions=(dims[0], dims[2],
dims[3]))
new_var[:, :, :] = squeeze(z_var[:, 0, :, :])
# new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()})
ds.close()
model_subset = call(inter_subset_tmp, variable='z%s' % level)
else:
model_subset = inter_subset_tmp
else:
model_subset = call(
resource=model_nc,
variable=variable,
geom=bbox,
spatial_wrapping='wrap',
time_range=time_range,
# conform_units_to=conform_units_to
)
# =============================================================================================
LOGGER.info('Dataset subset done: %s ', model_subset)
response.update_status('dataset subsetted', 40)
##############################################
# computing anomalies
##############################################
response.update_status('computing anomalies ', 50)
cycst = anualcycle.split('-')[0]
cycen = anualcycle.split('-')[1]
reference = [
dt.strptime(cycst, '%Y%m%d'),
dt.strptime(cycen, '%Y%m%d')
]
LOGGER.info('reference time: %s', reference)
model_anomal = wr.get_anomalies(model_subset,
reference=reference,
method=method,
sseas=sseas) # , variable=variable)
#####################
# extracting season
#####################
response.update_status('normalizing data', 60)
model_season = wr.get_season(model_anomal, season=season)
response.update_status('anomalies computed and normalized', 70)
#######################
# call the R scripts
#######################
response.update_status('Start weather regime clustering ', 80)
import shlex
import subprocess
from blackswan import config
from os.path import curdir, exists, join
try:
# rworkspace = curdir
Rsrc = config.Rsrc_dir()
Rfile = 'weatherregimes_model.R'
infile = model_season # model_subset #model_ponderate
# modelname = model
# yr1 = start.year
# yr2 = end.year
ip, output_graphics = mkstemp(dir=curdir, suffix='.pdf')
ip, file_pca = mkstemp(dir=curdir, suffix='.txt')
ip, file_class = mkstemp(dir=curdir, suffix='.Rdat')
args = [
'Rscript',
join(Rsrc, Rfile),
'%s/' % curdir,
'%s/' % Rsrc,
'%s' % infile,
'%s' % variable,
'%s' % output_graphics,
'%s' % file_pca,
'%s' % file_class,
'%s' % season,
'%s' % start.year,
'%s' % end.year,
'%s' % model_var,
'%s' % kappa
]
LOGGER.info('Rcall builded')
LOGGER.debug('ARGS: %s' % (args))
except:
msg = 'failed to build the R command'
LOGGER.exception(msg)
raise Exception(msg)
try:
output, error = subprocess.Popen(
args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
LOGGER.info('R outlog info:\n %s ' % output)
LOGGER.exception('R outlog errors:\n %s ' % error)
if len(output) > 0:
response.update_status('**** weatherregime in R suceeded', 90)
else:
LOGGER.exception('NO! output returned from R call')
except:
msg = 'weatherregime in R'
LOGGER.exception(msg)
raise Exception(msg)
response.update_status('Weather regime clustering done ', 95)
############################################
# set the outputs
############################################
# response.update_status('Set the process outputs ', 96)
response.outputs['Routput_graphic'].file = output_graphics
response.outputs['output_pca'].file = file_pca
response.outputs['output_classification'].file = file_class
response.outputs['output_netcdf'].file = model_subset
response.update_status('done', 100)
return response
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
LOGGER.info('Start process')
response.update_status('execution started at : {}'.format(dt.now()), 5)
process_start_time = time.time() # measure process execution time ...
start_time = time.time() # measure init ...
################################
# reading in the input arguments
################################
try:
response.update_status('read input parameter : %s ' % dt.now(), 6)
refSt = request.inputs['refSt'][0].data
refEn = request.inputs['refEn'][0].data
dateSt = request.inputs['dateSt'][0].data
dateEn = request.inputs['dateEn'][0].data
seasonwin = request.inputs['seasonwin'][0].data
nanalog = request.inputs['nanalog'][0].data
bboxDef = '-20,40,30,70' # in general format
bbox = []
bboxStr = request.inputs['BBox'][0].data
LOGGER.debug('BBOX selected by user: %s ' % (bboxStr))
bboxStr = bboxStr.split(',')
# Checking for wrong cordinates and apply default if nesessary
if (abs(float(bboxStr[0])) > 180 or
abs(float(bboxStr[1]) > 180) or
abs(float(bboxStr[2]) > 90) or
abs(float(bboxStr[3])) > 90):
bboxStr = bboxDef # request.inputs['BBox'].default # .default doesn't work anymore!!!
LOGGER.debug('BBOX is out of the range, using default instead: %s ' % (bboxStr))
bboxStr = bboxStr.split(',')
bbox.append(float(bboxStr[0]))
bbox.append(float(bboxStr[2]))
bbox.append(float(bboxStr[1]))
bbox.append(float(bboxStr[3]))
LOGGER.debug('BBOX for ocgis: %s ' % (bbox))
LOGGER.debug('BBOX original: %s ' % (bboxStr))
plot = request.inputs['plot'][0].data
distance = request.inputs['dist'][0].data
outformat = request.inputs['outformat'][0].data
timewin = request.inputs['timewin'][0].data
model_var = request.inputs['reanalyses'][0].data
model, var = model_var.split('_')
ref_model_var = request.inputs['Refreanalyses'][0].data
ref_model, ref_var = ref_model_var.split('_')
LOGGER.info('input parameters set')
response.update_status('Read in and convert the arguments', 7)
except Exception as e:
msg = 'failed to read input prameter %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
######################################
# convert types and set environment
######################################
try:
response.update_status('Preparing enviroment converting arguments', 8)
LOGGER.debug('date: %s %s %s %s ' % (type(refSt), refEn, dateSt, dateSt))
# normalize == 'None':
seacyc = False
if outformat == 'ascii':
outformat = '.txt'
elif outformat == 'netCDF':
outformat = '.nc'
else:
LOGGER.exception('output format not valid')
except Exception as e:
msg = 'failed to set environment %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
###########################
# set the environment
###########################
response.update_status('fetching data from archive', 9)
getlevel = False
if 'z' in var:
level = var.strip('z')
else:
level = None
##########################################
# fetch Data from original data archive
##########################################
try:
model_nc = rl(start=dateSt.year, end=dateEn.year,
dataset=model, variable=var,
getlevel=getlevel)
ref_model_nc = rl(start=refSt.year, end=refEn.year,
dataset=ref_model, variable=ref_var,
getlevel=getlevel)
LOGGER.info('reanalyses data fetched')
except Exception:
msg = 'failed to get reanalyses data'
LOGGER.exception(msg)
raise Exception(msg)
response.update_status('subsetting region of interest', 10)
# Checking memory and dataset size
model_size = get_files_size(model_nc)
ref_model_size = get_files_size(ref_model_nc)
m_size = max(model_size, ref_model_size)
memory_avail = psutil.virtual_memory().available
thrs = 0.2 # 20%
if (m_size >= thrs * memory_avail):
ser_r = True
else:
ser_r = False
LOGGER.debug('Available Memory: %s ' % (memory_avail))
LOGGER.debug('Dataset size: %s ' % (m_size))
LOGGER.debug('Threshold: %s ' % (thrs * memory_avail))
LOGGER.debug('Serial or at once: %s ' % (ser_r))
# #####################################################
# Construct descriptive filenames for the three files #
# listed in config file #
# TODO check strftime for years <1900 (!) #
# #####################################################
# refDatesString = dt.strftime(refSt, '%Y-%m-%d') + "_" + dt.strftime(refEn, '%Y-%m-%d')
# simDatesString = dt.strftime(dateSt, '%Y-%m-%d') + "_" + dt.strftime(dateEn, '%Y-%m-%d')
# Fix < 1900 issue...
refDatesString = refSt.isoformat().strip().split("T")[0] + "_" + refEn.isoformat().strip().split("T")[0]
simDatesString = dateSt.isoformat().strip().split("T")[0] + "_" + dateEn.isoformat().strip().split("T")[0]
archiveNameString = "base_" + var + "_" + refDatesString + '_%.1f_%.1f_%.1f_%.1f' \
% (bbox[0], bbox[2], bbox[1], bbox[3])
simNameString = "sim_" + var + "_" + simDatesString + '_%.1f_%.1f_%.1f_%.1f' \
% (bbox[0], bbox[2], bbox[1], bbox[3])
if ('z' in var):
# ------------------ NCEP -------------------
tmp_total = []
origvar = get_variable(model_nc)
for z in model_nc:
b0 = call(resource=z, variable=origvar, level_range=[int(level), int(level)], geom=bbox,
spatial_wrapping='wrap', prefix='levdom_' + os.path.basename(z)[0:-3])
tmp_total.append(b0)
time_range = [dateSt, dateEn]
tmp_total = sorted(tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0])
inter_subset_tmp = call(resource=tmp_total, variable=origvar, time_range=time_range)
# Clean
for i in tmp_total:
tbr = 'rm -f %s' % (i)
os.system(tbr)
# Create new variable
ds = Dataset(inter_subset_tmp, mode='a')
z_var = ds.variables.pop(origvar)
dims = z_var.dimensions
new_var = ds.createVariable('z%s' % level, z_var.dtype, dimensions=(dims[0], dims[2], dims[3]))
new_var[:, :, :] = squeeze(z_var[:, 0, :, :])
ds.close()
simulation = call(inter_subset_tmp, variable='z%s' % level, prefix=simNameString)
# ------------------ 20CRV2c -------------------
tmp_total = []
origvar = get_variable(ref_model_nc)
for z in ref_model_nc:
tmp_n = 'tmp_%s' % (uuid.uuid1())
# select level and regrid
b0 = call(resource=z, variable=origvar, level_range=[int(level), int(level)],
spatial_wrapping='wrap', cdover='system',
regrid_destination=model_nc[0], regrid_options='bil', prefix=tmp_n)
# select domain
b01 = call(resource=b0, variable=origvar, geom=bbox, spatial_wrapping='wrap', prefix='levregr_' + os.path.basename(z)[0:-3])
tbr = 'rm -f %s' % (b0)
os.system(tbr)
tbr = 'rm -f %s.nc' % (tmp_n)
os.system(tbr)
tmp_total.append(b01)
time_range = [refSt, refEn]
tmp_total = sorted(tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0])
ref_inter_subset_tmp = call(resource=tmp_total, variable=origvar, time_range=time_range)
# Clean
for i in tmp_total:
tbr = 'rm -f %s' % (i)
os.system(tbr)
# Create new variable
ds = Dataset(ref_inter_subset_tmp, mode='a')
z_var = ds.variables.pop(origvar)
dims = z_var.dimensions
new_var = ds.createVariable('z%s' % level, z_var.dtype, dimensions=(dims[0], dims[2], dims[3]))
new_var[:, :, :] = squeeze(z_var[:, 0, :, :])
ds.close()
archive = call(ref_inter_subset_tmp, variable='z%s' % level, prefix=archiveNameString)
else:
if ser_r:
LOGGER.debug('Process reanalysis step-by-step')
# ----- NCEP ------
tmp_total = []
for z in model_nc:
b0 = call(resource=z, variable=var, geom=bbox, spatial_wrapping='wrap',
prefix='Rdom_' + os.path.basename(z)[0:-3])
tmp_total.append(b0)
tmp_total = sorted(tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0])
simulation = call(resource=tmp_total, variable=var, time_range=[dateSt, dateEn], prefix=simNameString)
# Clean
for i in tmp_total:
tbr = 'rm -f %s' % (i)
os.system(tbr)
# ----- 20CRV2c ------
tmp_n = 'tmp_%s' % (uuid.uuid1())
tmp_total = []
for z in ref_model_nc:
# regrid
b0 = call(resource=z, variable=ref_var, spatial_wrapping='wrap', cdover='system',
regrid_destination=model_nc[0], regrid_options='bil', prefix=tmp_n)
# select domain
b01 = call(resource=b0, variable=ref_var, geom=bbox, spatial_wrapping='wrap',
prefix='ref_Rdom_' + os.path.basename(z)[0:-3])
tbr = 'rm -f %s' % (b0)
os.system(tbr)
tbr = 'rm -f %s.nc' % (tmp_n)
os.system(tbr)
tmp_total.append(b01)
tmp_total = sorted(tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0])
archive = call(resource=tmp_total, variable=ref_var, time_range=[refSt, refEn], prefix=archiveNameString)
# Clean
for i in tmp_total:
tbr = 'rm -f %s' % (i)
os.system(tbr)
else:
LOGGER.debug('Using whole dataset at once')
simulation = call(resource=model_nc, variable=var,
geom=bbox, spatial_wrapping='wrap', time_range=[dateSt, dateEn], prefix=simNameString)
ref_inter_subset_tmp = call(resource=ref_model_nc, variable=ref_var, spatial_wrapping='wrap',
cdover='system', regrid_destination=model_nc[0], regrid_options='bil')
archive = call(resource=ref_inter_subset_tmp, geom=bbox, spatial_wrapping='wrap', time_range=[refSt, refEn], prefix=archiveNameString)
response.update_status('datasets subsetted', 15)
LOGGER.debug("get_input_subset_dataset took %s seconds.",
time.time() - start_time)
response.update_status('**** Input data fetched', 20)
########################
# input data preperation
########################
response.update_status('Start preparing input data', 30)
start_time = time.time() # measure data preperation ...
LOGGER.info('archive and simulation files generated: %s, %s'
% (archive, simulation))
# Rename variable (TODO: For this specific process we know names: slp and prmsl...)
try:
if level is not None:
out_var = 'z%s' % level
else:
var_archive = get_variable(archive)
var_simulation = get_variable(simulation)
if var_archive != var_simulation:
rename_variable(archive, oldname=var_archive, newname=var_simulation)
out_var = var_simulation
LOGGER.info('varname %s in netCDF renamed to %s' % (var_archive, var_simulation))
except:
msg = 'failed to rename variable in target files'
LOGGER.exception(msg)
raise Exception(msg)
# seacyc is False:
seasoncyc_base = seasoncyc_sim = None
output_file = 'output.txt'
files = [os.path.abspath(archive), os.path.abspath(simulation), output_file]
LOGGER.debug("Data preperation took %s seconds.",
time.time() - start_time)
############################
# generate the config file
############################
config_file = analogs.get_configfile(
files=files,
seasoncyc_base=seasoncyc_base,
seasoncyc_sim=seasoncyc_sim,
base_id=ref_model,
sim_id=model,
timewin=timewin,
varname=out_var,
seacyc=seacyc,
cycsmooth=91,
nanalog=nanalog,
seasonwin=seasonwin,
distfun=distance,
outformat=outformat,
calccor=True,
silent=False,
# period=[dt.strftime(refSt, '%Y-%m-%d'), dt.strftime(refEn, '%Y-%m-%d')],
period=[refSt.isoformat().strip().split("T")[0], refEn.isoformat().strip().split("T")[0]],
bbox="{0[0]},{0[2]},{0[1]},{0[3]}".format(bbox))
response.update_status('generated config file', 40)
#######################
# CASTf90 call
#######################
start_time = time.time() # measure call castf90
# -----------------------
try:
import ctypes
# TODO: This lib is for linux
mkl_rt = ctypes.CDLL('libmkl_rt.so')
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('Current number of threads: %s' % (nth))
mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64)))
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('NEW number of threads: %s' % (nth))
# TODO: Does it \/\/\/ work with default shell=False in subprocess... (?)
os.environ['MKL_NUM_THREADS'] = str(nth)
os.environ['OMP_NUM_THREADS'] = str(nth)
except Exception as e:
msg = 'Failed to set THREADS %s ' % e
LOGGER.debug(msg)
# -----------------------
# ##### TEMPORAL WORKAROUND! With instaled hdf5-1.8.18 in anaconda ###############
# ##### MUST be removed after castf90 recompiled with the latest hdf version
# ##### NOT safe
os.environ['HDF5_DISABLE_VERSION_CHECK'] = '1'
# hdflib = os.path.expanduser("~") + '/anaconda/lib'
# hdflib = os.getenv("HOME") + '/anaconda/lib'
import pwd
hdflib = pwd.getpwuid(os.getuid()).pw_dir + '/anaconda/lib'
os.environ['LD_LIBRARY_PATH'] = hdflib
# ################################################################################
response.update_status('Start CASTf90 call', 50)
try:
# response.update_status('execution of CASTf90', 50)
cmd = ['analogue.out', config_file]
LOGGER.debug("castf90 command: %s", cmd)
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
LOGGER.info('analogue output:\n %s', output)
response.update_status('**** CASTf90 suceeded', 60)
except CalledProcessError as e:
msg = 'CASTf90 failed:\n{0}'.format(e.output)
LOGGER.exception(msg)
raise Exception(msg)
LOGGER.debug("castf90 took %s seconds.", time.time() - start_time)
# TODO: Add try - except for pdfs
if plot == 'Yes':
analogs_pdf = analogs.plot_analogs(configfile=config_file)
else:
analogs_pdf = 'dummy_plot.pdf'
with open(analogs_pdf, 'a'): os.utime(analogs_pdf, None)
response.update_status('preparing output', 70)
response.outputs['analog_pdf'].file = analogs_pdf
response.outputs['config'].file = config_file
response.outputs['analogs'].file = output_file
########################
# generate analog viewer
########################
formated_analogs_file = analogs.reformat_analogs(output_file)
# response.outputs['formated_analogs'].storage = FileStorage()
response.outputs['formated_analogs'].file = formated_analogs_file
LOGGER.info('analogs reformated')
response.update_status('reformatted analog file', 80)
viewer_html = analogs.render_viewer(
# configfile=response.outputs['config'].get_url(),
configfile=config_file,
# datafile=response.outputs['formated_analogs'].get_url())
datafile=formated_analogs_file)
response.outputs['output'].file = viewer_html
response.update_status('Successfully generated analogs viewer', 90)
LOGGER.info('rendered pages: %s ', viewer_html)
response.update_status('execution ended', 100)
LOGGER.debug("total execution took %s seconds.",
time.time() - process_start_time)
return response