def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
###########################################
# reorganize analog txt file for javascript
# and find associated config file
###########################################
# Reformat data file output by the analogs detection process so that
# it can be read by the analogues viewer template.
try:
# Get the output csv file of analogs process (input by user in
# text box)
analogs = rename_complexinputs(request.inputs['analog_result'])[0]
# analogs = request.inputs['analog_result'][0]
LOGGER.info("analogs file path %s ", analogs)
configfile = "dummy.txt" # anlg.get_viewer_configfile(analogs)
analogs_mod = anlg.reformat_analogs(analogs)
response.outputs['output_txt'].file = analogs_mod # output_data
LOGGER.info("analogs for visualisation prepared")
except Exception:
msg = 'Failed to reformat analogs file'
LOGGER.exception(msg)
raise Exception(msg)
try:
output_av = anlg.render_viewer(configfile=configfile,
datafile=analogs_mod)
LOGGER.info('Viewer html page generated')
response.update_status(
'Successfully generated analogs viewer html page', 90)
response.outputs['output_html'].file = output_av
LOGGER.info('output_av: %s ', output_av)
except Exception:
msg = 'Failed to generate viewer'
LOGGER.exception(msg)
raise Exception(msg)
return response
def _handler(self, request, response):
# LOGGER.debug('CURDIR XXXX : %s ' % (abspath(curdir)))
# LOGGER.debug('WORKDIR XXXX : %s ' % (self.workdir))
os.chdir(self.workdir)
# LOGGER.debug('CURDIR XXXX : %s ' % (abspath(curdir)))
init_process_logger('log.txt')
# init_process_logger(os.path.join(self.workdir, 'log.txt'))
# response.outputs['output_log'].file = 'log.txt'
LOGGER.info('Start process')
response.update_status('execution started at : {}'.format(dt.now()), 5)
process_start_time = time.time() # measure process execution time ...
start_time = time.time() # measure init ...
################################
# reading in the input arguments
################################
try:
response.update_status('read input parameter : %s ' % dt.now(), 6)
refSt = request.inputs['refSt'][0].data
refEn = request.inputs['refEn'][0].data
dateSt = request.inputs['dateSt'][0].data
dateEn = request.inputs['dateEn'][0].data
seasonwin = request.inputs['seasonwin'][0].data
nanalog = request.inputs['nanalog'][0].data
timres = request.inputs['timeres'][0].data
bboxDef = '-20,40,30,70' # in general format
bbox = []
bboxStr = request.inputs['BBox'][0].data
LOGGER.debug('BBOX selected by user: %s ' % (bboxStr))
bboxStr = bboxStr.split(',')
# Checking for wrong cordinates and apply default if nesessary
if (abs(float(bboxStr[0])) > 180 or
abs(float(bboxStr[1]) > 180) or
abs(float(bboxStr[2]) > 90) or
abs(float(bboxStr[3])) > 90):
bboxStr = bboxDef # request.inputs['BBox'].default # .default doesn't work anymore!!!
LOGGER.debug('BBOX is out of the range, using default instead: %s ' % (bboxStr))
bboxStr = bboxStr.split(',')
bbox.append(float(bboxStr[0]))
bbox.append(float(bboxStr[2]))
bbox.append(float(bboxStr[1]))
bbox.append(float(bboxStr[3]))
LOGGER.debug('BBOX for ocgis: %s ' % (bbox))
LOGGER.debug('BBOX original: %s ' % (bboxStr))
normalize = request.inputs['normalize'][0].data
detrend = request.inputs['detrend'][0].data
plot = request.inputs['plot'][0].data
distance = request.inputs['dist'][0].data
outformat = request.inputs['outformat'][0].data
timewin = request.inputs['timewin'][0].data
model_var = request.inputs['reanalyses'][0].data
model, var = model_var.split('_')
LOGGER.info('input parameters set')
response.update_status('Read in and convert the arguments', 7)
except Exception as e:
msg = 'failed to read input prameter %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
######################################
# convert types and set environment
######################################
try:
response.update_status('Preparing enviroment converting arguments', 8)
LOGGER.debug('date: %s %s %s %s ' % (type(refSt), refEn, dateSt, dateSt))
start = min(refSt, dateSt)
end = max(refEn, dateEn)
if normalize == 'None':
seacyc = False
else:
seacyc = True
if outformat == 'ascii':
outformat = '.txt'
elif outformat == 'netCDF':
outformat = '.nc'
else:
LOGGER.exception('output format not valid')
except Exception as e:
msg = 'failed to set environment %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
###########################
# set the environment
###########################
response.update_status('fetching data from archive', 9)
try:
if model == 'NCEP':
getlevel = False
if 'z' in var:
level = var.strip('z')
# conform_units_to = None
else:
level = None
if var == 'precip':
var = 'pr_wtr'
# conform_units_to = 'hPa'
elif '20CRV2' in model:
getlevel = False
if 'z' in var:
level = var.strip('z')
# conform_units_to = None
else:
level = None
# conform_units_to = 'hPa'
else:
LOGGER.exception('Reanalyses dataset not known')
LOGGER.info('environment set for model: %s' % model)
except Exception:
msg = 'failed to set environment'
LOGGER.exception(msg)
raise Exception(msg)
##########################################
# fetch Data from original data archive
##########################################
# NOTE: If ref is say 1950 - 1990, and sim is just 1 week in 2017:
# - ALL the data will be downloaded, 1950 - 2017
try:
model_nc = rl(start=start.year,
end=end.year,
dataset=model,
variable=var, timres=timres, getlevel=getlevel)
LOGGER.info('reanalyses data fetched')
except Exception:
msg = 'failed to get reanalyses data'
LOGGER.exception(msg)
raise Exception(msg)
response.update_status('subsetting region of interest', 10)
LOGGER.debug("start and end time: %s - %s" % (start, end))
time_range = [start, end]
# Checking memory and dataset size
model_size = get_files_size(model_nc)
memory_avail = psutil.virtual_memory().available
thrs = 0.3 # 30%
if (model_size >= thrs * memory_avail):
ser_r = True
else:
ser_r = False
# ################################
# For 20CRV2 geopotential height, daily dataset for 100 years is about 50 Gb
# So it makes sense, to operate it step-by-step
# TODO: need to create dictionary for such datasets (for models as well)
# TODO: benchmark the method bellow for NCEP z500 for 60 years
# if ('20CRV2' in model) and ('z' in var):
if ('z' in var):
tmp_total = []
origvar = get_variable(model_nc)
for z in model_nc:
# tmp_n = 'tmp_%s' % (uuid.uuid1())
b0 = call(resource=z, variable=origvar, level_range=[int(level), int(level)], geom=bbox,
spatial_wrapping='wrap', prefix='levdom_' + os.path.basename(z)[0:-3])
tmp_total.append(b0)
tmp_total = sorted(tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0])
inter_subset_tmp = call(resource=tmp_total, variable=origvar, time_range=time_range)
# Clean
for i in tmp_total:
tbr = 'rm -f %s' % (i)
os.system(tbr)
# Create new variable
ds = Dataset(inter_subset_tmp, mode='a')
z_var = ds.variables.pop(origvar)
dims = z_var.dimensions
new_var = ds.createVariable('z%s' % level, z_var.dtype, dimensions=(dims[0], dims[2], dims[3]))
new_var[:, :, :] = squeeze(z_var[:, 0, :, :])
# new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()})
ds.close()
model_subset_tmp = call(inter_subset_tmp, variable='z%s' % level)
else:
if ser_r:
LOGGER.debug('Process reanalysis step-by-step')
tmp_total = []
for z in model_nc:
# tmp_n = 'tmp_%s' % (uuid.uuid1())
b0 = call(resource=z, variable=var, geom=bbox, spatial_wrapping='wrap',
prefix='Rdom_' + os.path.basename(z)[0:-3])
tmp_total.append(b0)
tmp_total = sorted(tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0])
model_subset_tmp = call(resource=tmp_total, variable=var, time_range=time_range)
else:
LOGGER.debug('Using whole dataset at once')
model_subset_tmp = call(resource=model_nc, variable=var,
geom=bbox, spatial_wrapping='wrap', time_range=time_range,
)
# If dataset is 20CRV2 the 6 hourly file should be converted to daily.
# Option to use previously 6h data from cache (if any) and not download daily files.
# Disabled for now
# if '20CRV2' in model:
# if timres == '6h':
# from cdo import Cdo
# cdo = Cdo(env=os.environ)
# model_subset = '%s.nc' % uuid.uuid1()
# tmp_f = '%s.nc' % uuid.uuid1()
# cdo_op = getattr(cdo, 'daymean')
# cdo_op(input=model_subset_tmp, output=tmp_f)
# sti = '00:00:00'
# cdo_op = getattr(cdo, 'settime')
# cdo_op(sti, input=tmp_f, output=model_subset)
# LOGGER.debug('File Converted from: %s to daily' % (timres))
# else:
# model_subset = model_subset_tmp
# else:
# model_subset = model_subset_tmp
# Remove \/\/\/ if work with 6h data...
model_subset = model_subset_tmp
LOGGER.info('Dataset subset done: %s ', model_subset)
response.update_status('dataset subsetted', 15)
# BLOCK OF DETRENDING of model_subset !
# Original model subset kept to further visualisaion if needed
# Now is issue with SLP:
# TODO 1 Keep trend as separate file
# TODO 2 Think how to add options to plot abomalies AND original data...
# May be do archive and simulation = call.. over NOT detrended data and keep it as well
# TODO 3 Check with faster smoother add removing trend of each grid
if detrend == 'None':
orig_model_subset = model_subset
else:
orig_model_subset = remove_mean_trend(model_subset, varname=var)
# ======================================
LOGGER.debug("get_input_subset_dataset took %s seconds.",
time.time() - start_time)
response.update_status('**** Input data fetched', 20)
########################
# input data preperation
########################
response.update_status('Start preparing input data', 30)
start_time = time.time() # measure data preperation ...
try:
# Construct descriptive filenames for the three files
# listed in config file
# refDatesString = dt.strftime(refSt, '%Y-%m-%d') + "_" + dt.strftime(refEn, '%Y-%m-%d')
# simDatesString = dt.strftime(dateSt, '%Y-%m-%d') + "_" + dt.strftime(dateEn, '%Y-%m-%d')
# Fix < 1900 issue...
refDatesString = refSt.isoformat().strip().split("T")[0] + "_" + refEn.isoformat().strip().split("T")[0]
simDatesString = dateSt.isoformat().strip().split("T")[0] + "_" + dateEn.isoformat().strip().split("T")[0]
archiveNameString = "base_" + var + "_" + refDatesString + '_%.1f_%.1f_%.1f_%.1f' \
% (bbox[0], bbox[2], bbox[1], bbox[3])
simNameString = "sim_" + var + "_" + simDatesString + '_%.1f_%.1f_%.1f_%.1f' \
% (bbox[0], bbox[2], bbox[1], bbox[3])
archive = call(resource=model_subset,
time_range=[refSt, refEn],
prefix=archiveNameString)
simulation = call(resource=model_subset, time_range=[dateSt, dateEn],
prefix=simNameString)
LOGGER.info('archive and simulation files generated: %s, %s'
% (archive, simulation))
except Exception as e:
msg = 'failed to prepare archive and simulation files %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
try:
if seacyc is True:
LOGGER.info('normalization function with method: %s '
% normalize)
seasoncyc_base, seasoncyc_sim = analogs.seacyc(
archive,
simulation,
method=normalize)
else:
seasoncyc_base = seasoncyc_sim = None
except Exception as e:
msg = 'failed to generate normalization files %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
output_file = 'output.txt'
files = [os.path.abspath(archive), os.path.abspath(simulation), output_file]
LOGGER.debug("Data preperation took %s seconds.",
time.time() - start_time)
############################
# generate the config file
############################
config_file = analogs.get_configfile(
files=files,
seasoncyc_base=seasoncyc_base,
seasoncyc_sim=seasoncyc_sim,
base_id=model,
sim_id=model,
timewin=timewin,
varname=var,
seacyc=seacyc,
cycsmooth=91,
nanalog=nanalog,
seasonwin=seasonwin,
distfun=distance,
outformat=outformat,
calccor=True,
silent=False,
# period=[dt.strftime(refSt, '%Y-%m-%d'), dt.strftime(refEn, '%Y-%m-%d')],
period=[refSt.isoformat().strip().split("T")[0], refEn.isoformat().strip().split("T")[0]],
bbox="{0[0]},{0[2]},{0[1]},{0[3]}".format(bbox))
response.update_status('generated config file', 40)
#######################
# CASTf90 call
#######################
start_time = time.time() # measure call castf90
# -----------------------
try:
import ctypes
# TODO: This lib is for linux
mkl_rt = ctypes.CDLL('libmkl_rt.so')
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('Current number of threads: %s' % (nth))
mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64)))
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('NEW number of threads: %s' % (nth))
# TODO: Does it \/\/\/ work with default shell=False in subprocess... (?)
os.environ['MKL_NUM_THREADS'] = str(nth)
os.environ['OMP_NUM_THREADS'] = str(nth)
except Exception as e:
msg = 'Failed to set THREADS %s ' % e
LOGGER.debug(msg)
# -----------------------
# ##### TEMPORAL WORKAROUND! With instaled hdf5-1.8.18 in anaconda ###############
# ##### MUST be removed after castf90 recompiled with the latest hdf version
# ##### NOT safe
os.environ['HDF5_DISABLE_VERSION_CHECK'] = '1'
# hdflib = os.path.expanduser("~") + '/anaconda/lib'
# hdflib = os.getenv("HOME") + '/anaconda/lib'
import pwd
hdflib = pwd.getpwuid(os.getuid()).pw_dir + '/anaconda/lib'
os.environ['LD_LIBRARY_PATH'] = hdflib
# ################################################################################
response.update_status('Start CASTf90 call', 50)
try:
# response.update_status('execution of CASTf90', 50)
cmd = ['analogue.out', config_file]
LOGGER.debug("castf90 command: %s", cmd)
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
LOGGER.info('analogue output:\n %s', output)
response.update_status('**** CASTf90 suceeded', 60)
except CalledProcessError as e:
msg = 'CASTf90 failed:\n{0}'.format(e.output)
LOGGER.exception(msg)
raise Exception(msg)
LOGGER.debug("castf90 took %s seconds.", time.time() - start_time)
# TODO: Add try - except for pdfs
if plot == 'Yes':
analogs_pdf = analogs.plot_analogs(configfile=config_file)
else:
analogs_pdf = 'dummy_plot.pdf'
with open(analogs_pdf, 'a'): os.utime(analogs_pdf, None)
response.update_status('preparing output', 70)
response.outputs['analog_pdf'].file = analogs_pdf
response.outputs['config'].file = config_file
response.outputs['analogs'].file = output_file
response.outputs['output_netcdf'].file = simulation
response.outputs['target_netcdf'].file = archive
if seacyc is True:
response.outputs['base_netcdf'].file = seasoncyc_base
response.outputs['sim_netcdf'].file = seasoncyc_sim
else:
# TODO: Still unclear how to overpass unknown number of outputs
dummy_base = 'dummy_base.nc'
dummy_sim = 'dummy_sim.nc'
with open(dummy_base, 'a'): os.utime(dummy_base, None)
with open(dummy_sim, 'a'): os.utime(dummy_sim, None)
response.outputs['base_netcdf'].file = dummy_base
response.outputs['sim_netcdf'].file = dummy_sim
########################
# generate analog viewer
########################
formated_analogs_file = analogs.reformat_analogs(output_file)
# response.outputs['formated_analogs'].storage = FileStorage()
response.outputs['formated_analogs'].file = formated_analogs_file
LOGGER.info('analogs reformated')
response.update_status('reformatted analog file', 80)
viewer_html = analogs.render_viewer(
# configfile=response.outputs['config'].get_url(),
configfile=config_file,
# datafile=response.outputs['formated_analogs'].get_url())
datafile=formated_analogs_file)
response.outputs['output'].file = viewer_html
response.update_status('Successfully generated analogs viewer', 90)
LOGGER.info('rendered pages: %s ', viewer_html)
response.update_status('execution ended', 100)
LOGGER.debug("total execution took %s seconds.",
time.time() - process_start_time)
response.outputs['output_log'].file = 'log.txt'
return response
def _handler(self, request, response):
chdir(self.workdir)
init_process_logger('log.txt')
process_start_time = time.time() # measure process execution time ...
response.update_status('execution started at : %s ' % dt.now(), 5)
start_time = time.time() # measure init ...
resource = archiveextract(
resource=rename_complexinputs(request.inputs['resource']))
# Filter resource:
if type(resource) == list:
resource = sorted(resource,
key=lambda i: path.splitext(path.basename(i))[0])
else:
resource = [resource]
refSt = request.inputs['refSt'][0].data
refEn = request.inputs['refEn'][0].data
dateSt = request.inputs['dateSt'][0].data
dateEn = request.inputs['dateEn'][0].data
regrset = request.inputs['regrset'][0].data
direction = request.inputs['direction'][0].data
# Check if model has 360_day calendar:
try:
modcal, calunits = get_calendar(resource[0])
LOGGER.debug('CALENDAR: %s' % (modcal))
if '360_day' in modcal:
if direction == 're2mo':
if refSt.day == 31:
refSt = refSt.replace(day=30)
LOGGER.debug('Date has been changed for: %s' % (refSt))
if refEn.day == 31:
refEn = refEn.replace(day=30)
LOGGER.debug('Date has been changed for: %s' % (refEn))
else: # mo2re
if dateSt.day == 31:
dateSt = dateSt.replace(day=30)
LOGGER.debug('Date has been changed for: %s' %
(dateSt))
if dateEn.day == 31:
dateEn = dateEn.replace(day=30)
LOGGER.debug('Date has been changed for: %s' %
(dateEn))
except:
LOGGER.debug('Could not detect calendar')
seasonwin = request.inputs['seasonwin'][0].data
nanalog = request.inputs['nanalog'][0].data
bboxDef = '-20,40,30,70' # in general format
bbox = []
bboxStr = request.inputs['BBox'][0].data
LOGGER.debug('BBOX selected by user: %s ' % (bboxStr))
bboxStr = bboxStr.split(',')
# Checking for wrong cordinates and apply default if nesessary
if (abs(float(bboxStr[0])) > 180 or abs(float(bboxStr[1]) > 180)
or abs(float(bboxStr[2]) > 90) or abs(float(bboxStr[3])) > 90):
bboxStr = bboxDef # request.inputs['BBox'].default # .default doesn't work anymore!!!
LOGGER.debug(
'BBOX is out of the range, using default instead: %s ' %
(bboxStr))
bboxStr = bboxStr.split(',')
bbox.append(float(bboxStr[0]))
bbox.append(float(bboxStr[2]))
bbox.append(float(bboxStr[1]))
bbox.append(float(bboxStr[3]))
normalize = request.inputs['normalize'][0].data
plot = request.inputs['plot'][0].data
distance = request.inputs['dist'][0].data
outformat = request.inputs['outformat'][0].data
timewin = request.inputs['timewin'][0].data
model_var = request.inputs['reanalyses'][0].data
model, var = model_var.split('_')
try:
if direction == 're2mo':
anaSt = dt.combine(dateSt, dt_time(
0, 0)) # dt.strptime(dateSt[0], '%Y-%m-%d')
anaEn = dt.combine(dateEn, dt_time(
0, 0)) # dt.strptime(dateEn[0], '%Y-%m-%d')
refSt = dt.combine(refSt, dt_time(
12, 0)) # dt.strptime(refSt[0], '%Y-%m-%d')
refEn = dt.combine(refEn, dt_time(
12, 0)) # dt.strptime(refEn[0], '%Y-%m-%d')
r_time_range = [anaSt, anaEn]
m_time_range = [refSt, refEn]
elif direction == 'mo2re':
anaSt = dt.combine(dateSt, dt_time(
12, 0)) # dt.strptime(refSt[0], '%Y-%m-%d')
anaEn = dt.combine(dateEn, dt_time(
12, 0)) # dt.strptime(refEn[0], '%Y-%m-%d')
refSt = dt.combine(refSt, dt_time(
0, 0)) # dt.strptime(dateSt[0], '%Y-%m-%d')
refEn = dt.combine(refEn, dt_time(
0, 0)) # dt.strptime(dateEn[0], '%Y-%m-%d')
r_time_range = [refSt, refEn]
m_time_range = [anaSt, anaEn]
else:
LOGGER.exception(
'failed to find time periods for comparison direction')
except:
msg = 'failed to put simulation and reference time in order'
LOGGER.exception(msg)
raise Exception(msg)
if normalize == 'None':
seacyc = False
else:
seacyc = True
if outformat == 'ascii':
outformat = '.txt'
elif outformat == 'netCDF':
outformat = '.nc'
else:
LOGGER.exception('output format not valid')
try:
if model == 'NCEP':
# getlevel = True
getlevel = False
if 'z' in var:
level = var.strip('z')
variable = 'hgt'
# conform_units_to='hPa'
else:
variable = 'slp'
level = None
# conform_units_to='hPa'
elif '20CRV2' in model:
getlevel = False
if 'z' in var:
variable = 'hgt'
level = var.strip('z')
# conform_units_to=None
else:
variable = 'prmsl'
level = None
# conform_units_to='hPa'
else:
LOGGER.exception('Reanalyses model not known')
LOGGER.info('environment set')
except:
msg = 'failed to set environment'
LOGGER.exception(msg)
raise Exception(msg)
# LOGGER.exception("init took %s seconds.", time.time() - start_time)
response.update_status('Read in the arguments', 10)
#################
# get input data
#################
# TODO: do not forget to select years
start_time = time.time() # measure get_input_data ...
response.update_status('fetching input data', 20)
try:
if direction == 're2mo':
nc_reanalyses = reanalyses(start=anaSt.year,
end=anaEn.year,
variable=var,
dataset=model,
getlevel=getlevel)
else:
nc_reanalyses = reanalyses(start=refSt.year,
end=refEn.year,
variable=var,
dataset=model,
getlevel=getlevel)
if type(nc_reanalyses) == list:
nc_reanalyses = sorted(
nc_reanalyses,
key=lambda i: path.splitext(path.basename(i))[0])
else:
nc_reanalyses = [nc_reanalyses]
# For 20CRV2 geopotential height, daily dataset for 100 years is about 50 Gb
# So it makes sense, to operate it step-by-step
# TODO: need to create dictionary for such datasets (for models as well)
# TODO: benchmark the method bellow for NCEP z500 for 60 years, may be use the same (!)
# TODO Now everything regrid to the reanalysis
# if ('20CRV2' in model) and ('z' in var):
if ('z' in var):
tmp_total = []
origvar = get_variable(nc_reanalyses[0])
for z in nc_reanalyses:
# tmp_n = 'tmp_%s' % (uuid.uuid1())
b0 = call(resource=z,
variable=origvar,
level_range=[int(level), int(level)],
geom=bbox,
spatial_wrapping='wrap',
prefix='levdom_' + path.basename(z)[0:-3])
tmp_total.append(b0)
tmp_total = sorted(
tmp_total,
key=lambda i: path.splitext(path.basename(i))[0])
inter_subset_tmp = call(resource=tmp_total,
variable=origvar,
time_range=r_time_range)
# Create new variable
ds = Dataset(inter_subset_tmp, mode='a')
z_var = ds.variables.pop(origvar)
dims = z_var.dimensions
new_var = ds.createVariable('z%s' % level,
z_var.dtype,
dimensions=(dims[0], dims[2],
dims[3]))
new_var[:, :, :] = squeeze(z_var[:, 0, :, :])
# new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()})
ds.close()
nc_subset = call(inter_subset_tmp, variable='z%s' % level)
# Clean
for i in tmp_total:
tbr = 'rm -f %s' % (i)
system(tbr)
# for i in inter_subset_tmp
tbr = 'rm -f %s' % (inter_subset_tmp)
system(tbr)
else:
# TODO: ADD HERE serial as well as in analogs reanalysis process!!
nc_subset = call(
resource=nc_reanalyses,
variable=var,
geom=bbox,
spatial_wrapping='wrap',
time_range=r_time_range,
)
response.update_status('**** Input reanalyses data fetched', 30)
except:
msg = 'failed to fetch or subset input files'
LOGGER.exception(msg)
raise Exception(msg)
########################
# input data preperation
########################
response.update_status('Start preparing input data', 40)
m_start = m_time_range[0]
m_end = m_time_range[1]
# ===============================================================
# REMOVE resources from the list which are out of interest from the list
# (years > and < than requested for calculation)
tmp_resource = []
for re in resource:
s, e = get_timerange(re)
tmpSt = dt.strptime(s, '%Y%m%d')
tmpEn = dt.strptime(e, '%Y%m%d')
if ((tmpSt <= m_end) and (tmpEn >= m_start)):
tmp_resource.append(re)
LOGGER.debug('Selected file: %s ' % (re))
resource = tmp_resource
start_time = time.time() # mesure data preperation ...
# TODO: Check the callendars ! for model vs reanalyses.
# TODO: Check the units! model vs reanalyses.
try:
m_total = []
modvar = get_variable(resource)
# resource properties
ds = Dataset(resource[0])
m_var = ds.variables[modvar]
dims = list(m_var.dimensions)
dimlen = len(dims)
try:
model_id = ds.getncattr('model_id')
except AttributeError:
model_id = 'Unknown model'
LOGGER.debug('MODEL: %s ' % (model_id))
lev_units = 'hPa'
if (dimlen > 3):
lev = ds.variables[dims[1]]
# TODO: actually index [1] need to be detected... assuming zg(time, plev, lat, lon)
lev_units = lev.units
if (lev_units == 'Pa'):
m_level = str(int(level) * 100)
else:
m_level = level
else:
m_level = None
if level == None:
level_range = None
else:
level_range = [int(m_level), int(m_level)]
ds.close()
for z in resource:
tmp_n = 'tmp_%s' % (uuid.uuid1())
# TODO: Important! if only 1 file - select time period from that first!
# select level and regrid
# \/\/ working version 19Feb2019
# b0 = call(resource=z, variable=modvar, level_range=level_range,
# spatial_wrapping='wrap', cdover='system',
# regrid_destination=nc_reanalyses[0], regrid_options='bil', prefix=tmp_n)
try:
b0 = call(resource=z,
variable=modvar,
level_range=level_range,
spatial_wrapping='wrap',
cdover='system',
regrid_destination=nc_subset,
regrid_options='bil',
prefix=tmp_n)
except:
b0 = call(resource=z,
variable=modvar,
level_range=level_range,
spatial_wrapping='wrap',
cdover='system',
regrid_destination=nc_reanalyses[0],
regrid_options='bil',
prefix=tmp_n)
# select domain (already selected in fact, if regrided to 'nc_subset')
b01 = call(resource=b0,
geom=bbox,
spatial_wrapping='wrap',
prefix='levregr_' + path.basename(z)[0:-3])
# TODO: REPLACE rm -f by os.remove() !
tbr = 'rm -f %s' % (b0)
system(tbr)
tbr = 'rm -f %s.nc' % (tmp_n)
system(tbr)
# get full resource
m_total.append(b01)
model_subset = call(m_total, time_range=m_time_range)
for i in m_total:
tbr = 'rm -f %s' % (i)
system(tbr)
if m_level is not None:
# Create new variable in model set
ds = Dataset(model_subset, mode='a')
mod_var = ds.variables.pop(modvar)
dims = mod_var.dimensions
new_modvar = ds.createVariable('z%s' % level,
mod_var.dtype,
dimensions=(dims[0], dims[2],
dims[3]))
new_modvar[:, :, :] = squeeze(mod_var[:, 0, :, :])
# new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()})
ds.close()
mod_subset = call(model_subset, variable='z%s' % level)
else:
mod_subset = model_subset
except:
msg = 'failed to subset simulation or reference data'
LOGGER.exception(msg)
raise Exception(msg)
# --------------------------------------------
try:
if direction == 'mo2re':
simulation = mod_subset
archive = nc_subset
base_id = model
sim_id = model_id
elif direction == 're2mo':
simulation = nc_subset
archive = mod_subset
base_id = model_id
sim_id = model
else:
LOGGER.exception('direction not valid: %s ' % direction)
except:
msg = 'failed to find comparison direction'
LOGGER.exception(msg)
raise Exception(msg)
try:
if level is not None:
out_var = 'z%s' % level
else:
var_archive = get_variable(archive)
var_simulation = get_variable(simulation)
if var_archive != var_simulation:
rename_variable(archive,
oldname=var_archive,
newname=var_simulation)
out_var = var_simulation
LOGGER.info('varname %s in netCDF renamed to %s' %
(var_archive, var_simulation))
except:
msg = 'failed to rename variable in target files'
LOGGER.exception(msg)
raise Exception(msg)
try:
if seacyc is True:
seasoncyc_base, seasoncyc_sim = analogs.seacyc(
archive, simulation, method=normalize)
else:
seasoncyc_base = None
seasoncyc_sim = None
except:
msg = 'failed to prepare seasonal cycle reference files'
LOGGER.exception(msg)
raise Exception(msg)
# ip, output = mkstemp(dir='.', suffix='.txt')
# output_file = path.abspath(output)
output_file = 'output.txt'
################################
# Prepare names for config.txt #
################################
# refDatesString = dt.strftime(refSt, '%Y-%m-%d') + "_" + dt.strftime(refEn, '%Y-%m-%d')
# simDatesString = dt.strftime(dateSt, '%Y-%m-%d') + "_" + dt.strftime(dateEn, '%Y-%m-%d')
# Fix < 1900 issue...
refDatesString = refSt.isoformat().strip().split(
"T")[0] + "_" + refEn.isoformat().strip().split("T")[0]
simDatesString = dateSt.isoformat().strip().split(
"T")[0] + "_" + dateEn.isoformat().strip().split("T")[0]
archiveNameString = "base_" + out_var + "_" + refDatesString + '_%.1f_%.1f_%.1f_%.1f' \
% (bbox[0], bbox[2], bbox[1], bbox[3]) + '.nc'
simNameString = "sim_" + out_var + "_" + simDatesString + '_%.1f_%.1f_%.1f_%.1f' \
% (bbox[0], bbox[2], bbox[1], bbox[3]) + '.nc'
move(archive, archiveNameString)
move(simulation, simNameString)
archive = archiveNameString
simulation = simNameString
files = [path.abspath(archive), path.abspath(simulation), output_file]
############################
# generating the config file
############################
response.update_status('writing config file', 50)
start_time = time.time() # measure write config ...
try:
config_file = analogs.get_configfile(
files=files,
seasoncyc_base=seasoncyc_base,
seasoncyc_sim=seasoncyc_sim,
base_id=base_id,
sim_id=sim_id,
timewin=timewin,
# varname=var,
varname=out_var,
seacyc=seacyc,
cycsmooth=91,
nanalog=nanalog,
seasonwin=seasonwin,
distfun=distance,
outformat=outformat,
calccor=True,
silent=False,
# period=[dt.strftime(refSt, '%Y-%m-%d'), dt.strftime(refEn, '%Y-%m-%d')],
period=[
refSt.isoformat().strip().split("T")[0],
refEn.isoformat().strip().split("T")[0]
],
bbox="%s,%s,%s,%s" % (bbox[0], bbox[2], bbox[1], bbox[3]))
except:
msg = 'failed to generate config file'
LOGGER.exception(msg)
raise Exception(msg)
#######################
# CASTf90 call
#######################
import subprocess
import shlex
start_time = time.time() # measure call castf90
response.update_status('Start CASTf90 call', 60)
# -----------------------
try:
import ctypes
# TODO: This lib is for linux
mkl_rt = ctypes.CDLL('libmkl_rt.so')
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('Current number of threads: %s' % (nth))
mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64)))
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('NEW number of threads: %s' % (nth))
# TODO: Does it \/\/\/ work with default shell=False in subprocess... (?)
environ['MKL_NUM_THREADS'] = str(nth)
environ['OMP_NUM_THREADS'] = str(nth)
except Exception as e:
msg = 'Failed to set THREADS %s ' % e
LOGGER.debug(msg)
# -----------------------
# ##### TEMPORAL WORKAROUND! With instaled hdf5-1.8.18 in anaconda ###############
# ##### MUST be removed after castf90 recompiled with the latest hdf version
# ##### NOT safe
environ['HDF5_DISABLE_VERSION_CHECK'] = '1'
# hdflib = os.path.expanduser("~") + '/anaconda/lib'
# hdflib = os.getenv("HOME") + '/anaconda/lib'
import pwd
hdflib = pwd.getpwuid(getuid()).pw_dir + '/anaconda/lib'
environ['LD_LIBRARY_PATH'] = hdflib
# ################################################################################
try:
response.update_status('execution of CASTf90', 70)
cmd = 'analogue.out %s' % path.relpath(config_file)
# system(cmd)
args = shlex.split(cmd)
output, error = subprocess.Popen(
args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
LOGGER.info('analogue.out info:\n %s ' % output)
LOGGER.exception('analogue.out errors:\n %s ' % error)
response.update_status('**** CASTf90 suceeded', 80)
except:
msg = 'CASTf90 failed'
LOGGER.exception(msg)
raise Exception(msg)
LOGGER.debug("castf90 took %s seconds.", time.time() - start_time)
# TODO: Add try - except for pdfs
if plot == 'Yes':
analogs_pdf = analogs.plot_analogs(configfile=config_file)
else:
analogs_pdf = 'dummy_plot.pdf'
with open(analogs_pdf, 'a'):
utime(analogs_pdf, None)
response.update_status('preparting output', 90)
# Stopper to keep twitcher results, for debug
# dummy=dummy
response.outputs['analog_pdf'].file = analogs_pdf
response.outputs['config'].file = config_file
response.outputs['analogs'].file = output_file
response.outputs['output_netcdf'].file = simulation
response.outputs['target_netcdf'].file = archive
########################
# generate analog viewer
########################
formated_analogs_file = analogs.reformat_analogs(output_file)
response.outputs['formated_analogs'].file = formated_analogs_file
LOGGER.info('analogs reformated')
# response.update_status('reformatted analog file', 95)
viewer_html = analogs.render_viewer(
# configfile=response.outputs['config'].get_url(),
configfile=config_file,
# datafile=response.outputs['formated_analogs'].get_url())
datafile=formated_analogs_file)
response.outputs['output'].file = viewer_html
response.update_status('Successfully generated analogs viewer', 95)
LOGGER.info('rendered pages: %s ', viewer_html)
response.update_status('execution ended', 100)
LOGGER.debug("total execution took %s seconds.",
time.time() - process_start_time)
response.outputs['output_log'].file = 'log.txt'
return response
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
LOGGER.info('Start process')
response.update_status('execution started at : {}'.format(dt.now()), 5)
process_start_time = time.time() # measure process execution time ...
start_time = time.time() # measure init ...
################################
# reading in the input arguments
################################
# response.update_status('execution started at : %s ' % dt.now(), 5)
# start_time = time.time() # measure init ...
################################
# reading in the input arguments
################################
try:
response.update_status('read input parameter : %s ' % dt.now(), 10)
resource = archiveextract(resource=rename_complexinputs(request.inputs['resource']))
refSt = request.inputs['refSt'][0].data
refEn = request.inputs['refEn'][0].data
dateSt = request.inputs['dateSt'][0].data
dateEn = request.inputs['dateEn'][0].data
seasonwin = request.inputs['seasonwin'][0].data
nanalog = request.inputs['nanalog'][0].data
bboxDef = '-20,40,30,70' # in general format
# level = 500
level = request.inputs['level'][0].data
if (level == 500):
dummylevel = 1000 # dummy workaround for cdo sellevel
else:
dummylevel = 500
LOGGER.debug('LEVEL selected: %s hPa' % (level))
bbox = []
bboxStr = request.inputs['BBox'][0].data
LOGGER.debug('BBOX selected by user: %s ' % (bboxStr))
bboxStr = bboxStr.split(',')
# Checking for wrong cordinates and apply default if nesessary
if (abs(float(bboxStr[0])) > 180 or
abs(float(bboxStr[1]) > 180) or
abs(float(bboxStr[2]) > 90) or
abs(float(bboxStr[3])) > 90):
bboxStr = bboxDef # request.inputs['BBox'].default # .default doesn't work anymore!!!
LOGGER.debug('BBOX is out of the range, using default instead: %s ' % (bboxStr))
bboxStr = bboxStr.split(',')
# for i in bboxStr: bbox.append(int(i))
bbox.append(float(bboxStr[0]))
bbox.append(float(bboxStr[2]))
bbox.append(float(bboxStr[1]))
bbox.append(float(bboxStr[3]))
LOGGER.debug('BBOX for ocgis: %s ' % (bbox))
LOGGER.debug('BBOX original: %s ' % (bboxStr))
normalize = request.inputs['normalize'][0].data
plot = request.inputs['plot'][0].data
distance = request.inputs['dist'][0].data
outformat = request.inputs['outformat'][0].data
timewin = request.inputs['timewin'][0].data
detrend = request.inputs['detrend'][0].data
LOGGER.info('input parameters set')
response.update_status('Read in and convert the arguments', 20)
except Exception as e:
msg = 'failed to read input prameter %s ' % e
LOGGER.error(msg)
raise Exception(msg)
######################################
# convert types and set environment
######################################
try:
# not nesessary if fix ocgis_module.py
refSt = dt.combine(refSt, dt_time(12, 0))
refEn = dt.combine(refEn, dt_time(12, 0))
dateSt = dt.combine(dateSt, dt_time(12, 0))
dateEn = dt.combine(dateEn, dt_time(12, 0))
# Check if 360_day calendar:
try:
if type(resource) is not list:
resource = [resource]
modcal, calunits = get_calendar(resource[0])
if '360_day' in modcal:
if refSt.day == 31:
refSt = refSt.replace(day=30)
LOGGER.debug('Date has been changed for: %s' % (refSt))
if refEn.day == 31:
refEn = refEn.replace(day=30)
LOGGER.debug('Date has been changed for: %s' % (refEn))
if dateSt.day == 31:
dateSt = dateSt.replace(day=30)
LOGGER.debug('Date has been changed for: %s' % (dateSt))
if dateEn.day == 31:
dateEn = dateEn.replace(day=30)
LOGGER.debug('Date has been changed for: %s' % (dateEn))
except:
LOGGER.debug('Could not detect calendar')
if normalize == 'None':
seacyc = False
else:
seacyc = True
if outformat == 'ascii':
outformat = '.txt'
elif outformat == 'netCDF':
outformat = '.nc'
else:
LOGGER.error('output format not valid')
start = min(refSt, dateSt)
end = max(refEn, dateEn)
LOGGER.info('environment set')
except Exception as e:
msg = 'failed to set environment %s ' % e
LOGGER.error(msg)
raise Exception(msg)
LOGGER.debug("init took %s seconds.", time.time() - start_time)
response.update_status('Read in and convert the arguments', 30)
########################
# input data preperation
########################
# TODO: Check if files containing more than one dataset
response.update_status('Start preparing input data', 40)
start_time = time.time() # mesure data preperation ...
try:
# TODO: Add selection of the level. maybe bellow in call(..., level_range=[...,...])
if type(resource) == list:
# resource.sort()
resource = sorted(resource, key=lambda i: path.splitext(path.basename(i))[0])
else:
resource = [resource]
# ===============================================================
# REMOVE resources which are out of interest from the list
# (years > and < than requested for calculation)
tmp_resource = []
for re in resource:
s,e = get_timerange(re)
tmpSt = dt.strptime(s, '%Y%m%d')
tmpEn = dt.strptime(e, '%Y%m%d')
if ((tmpSt <= end) and (tmpEn >= start)):
tmp_resource.append(re)
LOGGER.debug('Selected file: %s ' % (re))
resource = tmp_resource
# Try to fix memory issue... (ocgis call for files like 20-30 gb... )
# IF 4D - select pressure level before domain cut
#
# resource properties
ds = Dataset(resource[0])
variable = get_variable(resource[0])
var = ds.variables[variable]
dims = list(var.dimensions)
dimlen = len(dims)
try:
model_id = ds.getncattr('model_id')
except AttributeError:
model_id = 'Unknown model'
LOGGER.debug('MODEL: %s ' % (model_id))
lev_units = 'hPa'
if (dimlen > 3):
lev = ds.variables[dims[1]]
# actually index [1] need to be detected... assuming zg(time, plev, lat, lon)
lev_units = lev.units
if (lev_units == 'Pa'):
level = level * 100
dummylevel = dummylevel * 100
# TODO: OR check the NAME and units of vertical level and find 200 , 300, or 500 mbar in it
# Not just level = level * 100.
# Get Levels
from cdo import Cdo
cdo = Cdo(env=environ)
lev_res = []
if(dimlen > 3):
for res_fn in resource:
tmp_f = 'lev_' + path.basename(res_fn)
try:
tmp_f = call(resource=res_fn, variable=variable, spatial_wrapping='wrap',
level_range=[int(level), int(level)], prefix=tmp_f[0:-3])
except:
comcdo = '%s,%s' % (level, dummylevel)
cdo.sellevel(comcdo, input=res_fn, output=tmp_f)
lev_res.append(tmp_f)
else:
lev_res = resource
# ===============================================================
# TODO: Before domain, Regrid to selected grid! (???) if no rean.
# ================================================================
# Get domain
regr_res = []
for res_fn in lev_res:
tmp_f = 'dom_' + path.basename(res_fn)
comcdo = '%s,%s,%s,%s' % (bbox[0], bbox[2], bbox[1], bbox[3])
try:
tmp_f = call(resource=res_fn, geom=bbox, spatial_wrapping='wrap', prefix=tmp_f[0:-3])
except:
cdo.sellonlatbox(comcdo, input=res_fn, output=tmp_f)
regr_res.append(tmp_f)
# ============================
# Block to Detrend data
# TODO 1 Keep trend as separate file
# TODO 2 Think how to add options to plot abomalies AND original data...
# May be do archive and simulation = call.. over NOT detrended data and keep it as well
if (dimlen > 3):
res_tmp = get_level(regr_res, level=level)
variable = 'z%s' % level
else:
res_tmp = call(resource=regr_res, spatial_wrapping='wrap')
if detrend == 'None':
orig_model_subset = res_tmp
else:
orig_model_subset = remove_mean_trend(res_tmp, varname=variable)
# ============================
# archive_tmp = call(resource=regr_res, time_range=[refSt, refEn], spatial_wrapping='wrap')
# simulation_tmp = call(resource=regr_res, time_range=[dateSt, dateEn], spatial_wrapping='wrap')
################################
# Prepare names for config.txt #
################################
# refDatesString = dt.strftime(refSt, '%Y-%m-%d') + "_" + dt.strftime(refEn, '%Y-%m-%d')
# simDatesString = dt.strftime(dateSt, '%Y-%m-%d') + "_" + dt.strftime(dateEn, '%Y-%m-%d')
# Fix < 1900 issue...
refDatesString = refSt.isoformat().strip().split("T")[0] + "_" + refEn.isoformat().strip().split("T")[0]
simDatesString = dateSt.isoformat().strip().split("T")[0] + "_" + dateEn.isoformat().strip().split("T")[0]
archiveNameString = "base_" + variable + "_" + refDatesString + '_%.1f_%.1f_%.1f_%.1f' \
% (bbox[0], bbox[2], bbox[1], bbox[3])
simNameString = "sim_" + variable + "_" + simDatesString + '_%.1f_%.1f_%.1f_%.1f' \
% (bbox[0], bbox[2], bbox[1], bbox[3])
archive = call(resource=res_tmp, time_range=[refSt, refEn], spatial_wrapping='wrap', prefix=archiveNameString)
simulation = call(resource=res_tmp, time_range=[dateSt, dateEn], spatial_wrapping='wrap', prefix=simNameString)
#######################################################################################
if seacyc is True:
seasoncyc_base, seasoncyc_sim = analogs.seacyc(archive, simulation, method=normalize)
else:
seasoncyc_base = None
seasoncyc_sim = None
except Exception as e:
msg = 'failed to prepare archive and simulation files %s ' % e
LOGGER.debug(msg)
raise Exception(msg)
ip, output = mkstemp(dir='.', suffix='.txt')
output_file = path.abspath(output)
files = [path.abspath(archive), path.abspath(simulation), output_file]
LOGGER.debug("data preperation took %s seconds.", time.time() - start_time)
############################
# generating the config file
############################
# TODO: add MODEL name as argument
response.update_status('writing config file', 50)
start_time = time.time() # measure write config ...
try:
config_file = analogs.get_configfile(
files=files,
seasoncyc_base=seasoncyc_base,
seasoncyc_sim=seasoncyc_sim,
base_id=model_id,
sim_id=model_id,
timewin=timewin,
varname=variable,
seacyc=seacyc,
cycsmooth=91,
nanalog=nanalog,
seasonwin=seasonwin,
distfun=distance,
outformat=outformat,
calccor=True,
silent=False,
# period=[dt.strftime(refSt, '%Y-%m-%d'), dt.strftime(refEn, '%Y-%m-%d')],
period=[refSt.isoformat().strip().split("T")[0], refEn.isoformat().strip().split("T")[0]],
bbox="%s,%s,%s,%s" % (bbox[0], bbox[2], bbox[1], bbox[3]))
except Exception as e:
msg = 'failed to generate config file %s ' % e
LOGGER.debug(msg)
raise Exception(msg)
LOGGER.debug("write_config took %s seconds.", time.time() - start_time)
##############
# CASTf90 call
##############
import subprocess
import shlex
start_time = time.time() # measure call castf90
response.update_status('Start CASTf90 call', 60)
# -----------------------
try:
import ctypes
# TODO: This lib is for linux
mkl_rt = ctypes.CDLL('libmkl_rt.so')
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('Current number of threads: %s' % (nth))
mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64)))
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('NEW number of threads: %s' % (nth))
# TODO: Does it \/\/\/ work with default shell=False in subprocess... (?)
environ['MKL_NUM_THREADS'] = str(nth)
environ['OMP_NUM_THREADS'] = str(nth)
except Exception as e:
msg = 'Failed to set THREADS %s ' % e
LOGGER.debug(msg)
# -----------------------
# ##### TEMPORAL WORKAROUND! With instaled hdf5-1.8.18 in anaconda ###############
# ##### MUST be removed after castf90 recompiled with the latest hdf version
# ##### NOT safe
environ['HDF5_DISABLE_VERSION_CHECK'] = '1'
# hdflib = os.path.expanduser("~") + '/anaconda/lib'
# hdflib = os.getenv("HOME") + '/anaconda/lib'
import pwd
hdflib = pwd.getpwuid(getuid()).pw_dir + '/anaconda/lib'
environ['LD_LIBRARY_PATH'] = hdflib
# ################################################################################
try:
# response.update_status('execution of CASTf90', 50)
cmd = 'analogue.out %s' % path.relpath(config_file)
# system(cmd)
args = shlex.split(cmd)
output, error = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
LOGGER.info('analogue.out info:\n %s ' % output)
LOGGER.debug('analogue.out errors:\n %s ' % error)
response.update_status('**** CASTf90 suceeded', 70)
except Exception as e:
msg = 'CASTf90 failed %s ' % e
LOGGER.error(msg)
raise Exception(msg)
LOGGER.debug("castf90 took %s seconds.", time.time() - start_time)
# TODO: Add try - except for pdfs
if plot == 'Yes':
analogs_pdf = analogs.plot_analogs(configfile=config_file)
else:
analogs_pdf = 'dummy_plot.pdf'
with open(analogs_pdf, 'a'): utime(analogs_pdf, None)
response.update_status('preparing output', 80)
response.outputs['analog_pdf'].file = analogs_pdf
response.outputs['config'].file = config_file
response.outputs['analogs'].file = output_file
response.outputs['output_netcdf'].file = simulation
response.outputs['target_netcdf'].file = archive
if seacyc is True:
response.outputs['base_netcdf'].file = seasoncyc_base
response.outputs['sim_netcdf'].file = seasoncyc_sim
else:
# TODO: Still unclear how to overpass unknown number of outputs
dummy_base = 'dummy_base.nc'
dummy_sim = 'dummy_sim.nc'
with open(dummy_base, 'a'): utime(dummy_base, None)
with open(dummy_sim, 'a'): utime(dummy_sim, None)
response.outputs['base_netcdf'].file = dummy_base
response.outputs['sim_netcdf'].file = dummy_sim
########################
# generate analog viewer
########################
formated_analogs_file = analogs.reformat_analogs(output_file)
# response.outputs['formated_analogs'].storage = FileStorage()
response.outputs['formated_analogs'].file = formated_analogs_file
LOGGER.info('analogs reformated')
response.update_status('reformatted analog file', 90)
viewer_html = analogs.render_viewer(
# configfile=response.outputs['config'].get_url(),
configfile=config_file,
# datafile=response.outputs['formated_analogs'].get_url())
datafile=formated_analogs_file)
response.outputs['output'].file = viewer_html
response.update_status('Successfully generated analogs viewer', 95)
LOGGER.info('rendered pages: %s ', viewer_html)
response.update_status('execution ended', 100)
LOGGER.debug("total execution took %s seconds.",
time.time() - process_start_time)
return response
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
LOGGER.info('Start process')
response.update_status('execution started at : {}'.format(dt.now()), 5)
process_start_time = time.time() # measure process execution time ...
start_time = time.time() # measure init ...
################################
# reading in the input arguments
################################
try:
response.update_status('read input parameter : %s ' % dt.now(), 7)
refSt = request.inputs['refSt'][0].data
#refEn = request.inputs['refEn'][0].data
refEn = dt.strptime(
'%s%s%s' % (dt.now().year, dt.now().month, dt.now().day),
'%Y%m%d')
refEn = refEn - timedelta(days=3)
dateSt = request.inputs['dateSt'][0].data
#dateEn = request.inputs['dateEn'][0].data
dateEn = dt.strptime(
'%s%s%s' % (dt.now().year, dt.now().month, dt.now().day),
'%Y%m%d')
dateEn = dateEn - timedelta(days=3)
seasonwin = request.inputs['seasonwin'][0].data
nanalog = request.inputs['nanalog'][0].data
bboxDef = '-80,50,20,70' # in general format
bbox = []
bboxStr = request.inputs['BBox'][0].data
LOGGER.debug('BBOX selected by user: %s ' % (bboxStr))
bboxStr = bboxStr.split(',')
# Checking for wrong cordinates and apply default if nesessary
if (abs(float(bboxStr[0])) > 180 or abs(float(bboxStr[1]) > 180)
or abs(float(bboxStr[2]) > 90)
or abs(float(bboxStr[3])) > 90):
bboxStr = bboxDef # request.inputs['BBox'].default # .default doesn't work anymore!!!
LOGGER.debug(
'BBOX is out of the range, using default instead: %s ' %
(bboxStr))
bboxStr = bboxStr.split(',')
bbox.append(float(bboxStr[0]))
bbox.append(float(bboxStr[2]))
bbox.append(float(bboxStr[1]))
bbox.append(float(bboxStr[3]))
LOGGER.debug('BBOX for ocgis: %s ' % (bbox))
LOGGER.debug('BBOX original: %s ' % (bboxStr))
normalize = request.inputs['normalize'][0].data
detrend = request.inputs['detrend'][0].data
distance = request.inputs['dist'][0].data
outformat = request.inputs['outformat'][0].data
timewin = request.inputs['timewin'][0].data
model_var = request.inputs['reanalyses'][0].data
model, var = model_var.split('_')
LOGGER.info('input parameters set')
response.update_status('Read in and convert the arguments', 8)
except Exception as e:
msg = 'failed to read input prameter %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
######################################
# convert types and set environment
######################################
try:
response.update_status('Preparing enviroment converting arguments',
9)
LOGGER.debug('date: %s %s %s %s ' %
(type(refSt), refEn, dateSt, dateSt))
start = min(refSt, dateSt)
end = max(refEn, dateEn)
if normalize == 'None':
seacyc = False
else:
seacyc = True
if outformat == 'ascii':
outformat = '.txt'
elif outformat == 'netCDF':
outformat = '.nc'
else:
LOGGER.exception('output format not valid')
except Exception as e:
msg = 'failed to set environment %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
###########################
# set the environment
###########################
response.update_status('fetching data from archive', 10)
# We work only with NCEP
getlevel = False
if 'z' in var:
level = var.strip('z')
else:
level = None
##########################################
# fetch Data from original data archive
##########################################
try:
model_nc = rl(start=start.year,
end=end.year,
dataset=model,
variable=var,
getlevel=getlevel)
LOGGER.info('reanalyses data fetched')
except Exception:
msg = 'failed to get reanalyses data'
LOGGER.exception(msg)
raise Exception(msg)
response.update_status('subsetting region of interest', 17)
LOGGER.debug("start and end time: %s - %s" % (start, end))
time_range = [start, end]
# Checking memory and dataset size
m_size = get_files_size(model_nc)
memory_avail = psutil.virtual_memory().available
thrs = 0.2 # 20%
if (m_size >= thrs * memory_avail):
ser_r = True
else:
ser_r = False
LOGGER.debug('Available Memory: %s ' % (memory_avail))
LOGGER.debug('Dataset size: %s ' % (m_size))
LOGGER.debug('Threshold: %s ' % (thrs * memory_avail))
LOGGER.debug('Serial or at once: %s ' % (ser_r))
# if ('20CRV2' in model) and ('z' in var):
if ('z' in var):
tmp_total = []
origvar = get_variable(model_nc)
for z in model_nc:
tmp_n = 'tmp_%s' % (uuid.uuid1())
b0 = call(resource=z,
variable=origvar,
level_range=[int(level), int(level)],
geom=bbox,
spatial_wrapping='wrap',
prefix='levdom_' + os.path.basename(z)[0:-3])
tmp_total.append(b0)
tmp_total = sorted(
tmp_total,
key=lambda i: os.path.splitext(os.path.basename(i))[0])
inter_subset_tmp = call(resource=tmp_total,
variable=origvar,
time_range=time_range)
# Clean
for i in tmp_total:
tbr = 'rm -f %s' % (i)
os.system(tbr)
# Create new variable
ds = Dataset(inter_subset_tmp, mode='a')
z_var = ds.variables.pop(origvar)
dims = z_var.dimensions
new_var = ds.createVariable('z%s' % level,
z_var.dtype,
dimensions=(dims[0], dims[2], dims[3]))
new_var[:, :, :] = squeeze(z_var[:, 0, :, :])
# new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()})
ds.close()
model_subset_tmp = call(inter_subset_tmp, variable='z%s' % level)
else:
if ser_r:
LOGGER.debug('Process reanalysis step-by-step')
tmp_total = []
for z in model_nc:
tmp_n = 'tmp_%s' % (uuid.uuid1())
b0 = call(resource=z,
variable=var,
geom=bbox,
spatial_wrapping='wrap',
prefix='Rdom_' + os.path.basename(z)[0:-3])
tmp_total.append(b0)
tmp_total = sorted(
tmp_total,
key=lambda i: os.path.splitext(os.path.basename(i))[0])
model_subset_tmp = call(resource=tmp_total,
variable=var,
time_range=time_range)
# Clean
for i in tmp_total:
tbr = 'rm -f %s' % (i)
os.system(tbr)
else:
LOGGER.debug('Using whole dataset at once')
model_subset_tmp = call(
resource=model_nc,
variable=var,
geom=bbox,
spatial_wrapping='wrap',
time_range=time_range,
)
# Rest from 20CRV...
model_subset = model_subset_tmp
LOGGER.info('Dataset subset done: %s ', model_subset)
response.update_status('dataset subsetted', 19)
# BLOCK OF DETRENDING of model_subset !
# Original model subset kept to further visualisaion if needed
# Now is issue with SLP:
# TODO 1 Keep trend as separate file
# TODO 2 Think how to add options to plot abomalies AND original data...
# May be do archive and simulation = call.. over NOT detrended data and keep it as well
# TODO 3 Check with faster smoother add removing trend of each grid
if detrend == 'None':
orig_model_subset = model_subset
else:
orig_model_subset = remove_mean_trend(model_subset, varname=var)
# ======================================
LOGGER.debug("get_input_subset_dataset took %s seconds.",
time.time() - start_time)
response.update_status('**** Input data fetched', 20)
########################
# input data preperation
########################
response.update_status('Start preparing input data', 22)
start_time = time.time() # measure data preperation ...
try:
# Construct descriptive filenames for the three files
# listed in config file
# TODO check strftime for years <1900 (!)
refDatesString = dt.strftime(
refSt, '%Y-%m-%d') + "_" + dt.strftime(refEn, '%Y-%m-%d')
simDatesString = dt.strftime(
dateSt, '%Y-%m-%d') + "_" + dt.strftime(dateEn, '%Y-%m-%d')
archiveNameString = "base_" + var + "_" + refDatesString + '_%.1f_%.1f_%.1f_%.1f' \
% (bbox[0], bbox[2], bbox[1], bbox[3])
simNameString = "sim_" + var + "_" + simDatesString + '_%.1f_%.1f_%.1f_%.1f' \
% (bbox[0], bbox[2], bbox[1], bbox[3])
archive = call(resource=model_subset,
time_range=[refSt, refEn],
prefix=archiveNameString)
simulation = call(resource=model_subset,
time_range=[dateSt, dateEn],
prefix=simNameString)
LOGGER.info('archive and simulation files generated: %s, %s' %
(archive, simulation))
except Exception as e:
msg = 'failed to prepare archive and simulation files %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
try:
if seacyc is True:
LOGGER.info('normalization function with method: %s ' %
normalize)
seasoncyc_base, seasoncyc_sim = analogs.seacyc(
archive, simulation, method=normalize)
else:
seasoncyc_base = seasoncyc_sim = None
except Exception as e:
msg = 'failed to generate normalization files %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
output_file = 'output.txt'
files = [
os.path.abspath(archive),
os.path.abspath(simulation), output_file
]
LOGGER.debug("Data preperation took %s seconds.",
time.time() - start_time)
############################
# generate the config file
############################
config_file = analogs.get_configfile(
files=files,
seasoncyc_base=seasoncyc_base,
seasoncyc_sim=seasoncyc_sim,
base_id=model,
sim_id=model,
timewin=timewin,
varname=var,
seacyc=seacyc,
cycsmooth=91,
nanalog=nanalog,
seasonwin=seasonwin,
distfun=distance,
outformat=outformat,
calccor=True,
silent=False,
period=[
dt.strftime(refSt, '%Y-%m-%d'),
dt.strftime(refEn, '%Y-%m-%d')
],
bbox="{0[0]},{0[2]},{0[1]},{0[3]}".format(bbox))
response.update_status('generated config file', 25)
#######################
# CASTf90 call
#######################
start_time = time.time() # measure call castf90
#-----------------------
try:
import ctypes
# TODO: This lib is for linux
mkl_rt = ctypes.CDLL('libmkl_rt.so')
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('Current number of threads: %s' % (nth))
mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64)))
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('NEW number of threads: %s' % (nth))
# TODO: Does it \/\/\/ work with default shell=False in subprocess... (?)
os.environ['MKL_NUM_THREADS'] = str(nth)
os.environ['OMP_NUM_THREADS'] = str(nth)
except Exception as e:
msg = 'Failed to set THREADS %s ' % e
LOGGER.debug(msg)
#-----------------------
# ##### TEMPORAL WORKAROUND! With instaled hdf5-1.8.18 in anaconda ###############
# ##### MUST be removed after castf90 recompiled with the latest hdf version
# ##### NOT safe
os.environ['HDF5_DISABLE_VERSION_CHECK'] = '1'
#hdflib = os.path.expanduser("~") + '/anaconda/lib'
#hdflib = os.getenv("HOME") + '/anaconda/lib'
import pwd
hdflib = pwd.getpwuid(os.getuid()).pw_dir + '/anaconda/lib'
os.environ['LD_LIBRARY_PATH'] = hdflib
# ################################################################################
response.update_status('Start CASTf90 call', 30)
try:
# response.update_status('execution of CASTf90', 50)
cmd = ['analogue.out', config_file]
LOGGER.debug("castf90 command: %s", cmd)
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
LOGGER.info('analogue output:\n %s', output)
response.update_status('**** CASTf90 suceeded', 40)
except CalledProcessError as e:
msg = 'CASTf90 failed:\n{0}'.format(e.output)
LOGGER.exception(msg)
raise Exception(msg)
LOGGER.debug("castf90 took %s seconds.", time.time() - start_time)
# --------------- R cont analogs calcs -----------------------------------
#######################
# call the R scripts
#######################
response.update_status(
'Start calculation of the stats and Return Periods ', 50)
import shlex
# import subprocess
from blackswan import config
from blackswan.visualisation import pdfmerge
from os.path import curdir, exists, join
try:
#rworkspace = curdir
Rsrc = config.Rsrc_dir()
Rfile = 'analogs_diags-prox.R'
Rdatfile = 'analogs_RT.Rdat'
probs_c = 0.7
probs_n = 0.3
args = [
'Rscript',
join(Rsrc, Rfile),
'%s' % output_file,
'%s' % probs_c,
'%s' % probs_n,
'%s' % Rdatfile
]
LOGGER.info('Rcall builded')
LOGGER.debug('ARGS: %s' % (args))
except:
msg = 'failed to build the R command'
LOGGER.exception(msg)
raise Exception(msg)
try:
output, error = subprocess.Popen(
args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
LOGGER.info('R outlog info:\n %s ' % output)
LOGGER.exception('R outlog errors:\n %s ' % error)
if len(output) > 0:
response.update_status('**** Return Periods with R suceeded',
60)
else:
LOGGER.exception('NO! output returned from R call')
analogs_pdf = pdfmerge(
['analogs_score-diags_new.pdf', 'analogs_RP-diags_new.pdf'])
except:
msg = 'ReturnPeriods in R'
LOGGER.exception(msg)
raise Exception(msg)
response.update_status('Calculation of Return Periods done ', 70)
# --------------- END of R cont analogs calcs ----------------------------
response.update_status('preparing output', 75)
response.outputs['analog_pdf'].file = analogs_pdf
response.outputs['config'].file = config_file
response.outputs['analogs'].file = output_file
response.outputs['output_netcdf'].file = simulation
response.outputs['target_netcdf'].file = archive
if seacyc is True:
response.outputs['base_netcdf'].file = seasoncyc_base
response.outputs['sim_netcdf'].file = seasoncyc_sim
else:
# TODO: Still unclear how to overpass unknown number of outputs
dummy_base = 'dummy_base.nc'
dummy_sim = 'dummy_sim.nc'
with open(dummy_base, 'a'):
os.utime(dummy_base, None)
with open(dummy_sim, 'a'):
os.utime(dummy_sim, None)
response.outputs['base_netcdf'].file = dummy_base
response.outputs['sim_netcdf'].file = dummy_sim
########################
# generate analog viewer
########################
formated_analogs_file = analogs.reformat_analogs(output_file)
# response.outputs['formated_analogs'].storage = FileStorage()
response.outputs['formated_analogs'].file = formated_analogs_file
LOGGER.info('analogs reformated')
response.update_status('reformatted analog file', 80)
viewer_html = analogs.render_viewer(
# configfile=response.outputs['config'].get_url(),
configfile=config_file,
# datafile=response.outputs['formated_analogs'].get_url())
datafile=formated_analogs_file)
response.outputs['output'].file = viewer_html
response.update_status('Successfully generated analogs viewer', 90)
LOGGER.info('rendered pages: %s ', viewer_html)
response.update_status('execution ended', 100)
LOGGER.debug("total execution took %s seconds.",
time.time() - process_start_time)
return response
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
LOGGER.info('Start process')
response.update_status('execution started at : {}'.format(dt.now()), 5)
process_start_time = time.time() # measure process execution time ...
start_time = time.time() # measure init ...
################################
# reading in the input arguments
################################
try:
response.update_status('read input parameter : %s ' % dt.now(), 6)
refSt = request.inputs['refSt'][0].data
refEn = request.inputs['refEn'][0].data
dateSt = request.inputs['dateSt'][0].data
dateEn = request.inputs['dateEn'][0].data
seasonwin = request.inputs['seasonwin'][0].data
nanalog = request.inputs['nanalog'][0].data
bboxDef = '-20,40,30,70' # in general format
bbox = []
bboxStr = request.inputs['BBox'][0].data
LOGGER.debug('BBOX selected by user: %s ' % (bboxStr))
bboxStr = bboxStr.split(',')
# Checking for wrong cordinates and apply default if nesessary
if (abs(float(bboxStr[0])) > 180 or
abs(float(bboxStr[1]) > 180) or
abs(float(bboxStr[2]) > 90) or
abs(float(bboxStr[3])) > 90):
bboxStr = bboxDef # request.inputs['BBox'].default # .default doesn't work anymore!!!
LOGGER.debug('BBOX is out of the range, using default instead: %s ' % (bboxStr))
bboxStr = bboxStr.split(',')
bbox.append(float(bboxStr[0]))
bbox.append(float(bboxStr[2]))
bbox.append(float(bboxStr[1]))
bbox.append(float(bboxStr[3]))
LOGGER.debug('BBOX for ocgis: %s ' % (bbox))
LOGGER.debug('BBOX original: %s ' % (bboxStr))
plot = request.inputs['plot'][0].data
distance = request.inputs['dist'][0].data
outformat = request.inputs['outformat'][0].data
timewin = request.inputs['timewin'][0].data
model_var = request.inputs['reanalyses'][0].data
model, var = model_var.split('_')
ref_model_var = request.inputs['Refreanalyses'][0].data
ref_model, ref_var = ref_model_var.split('_')
LOGGER.info('input parameters set')
response.update_status('Read in and convert the arguments', 7)
except Exception as e:
msg = 'failed to read input prameter %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
######################################
# convert types and set environment
######################################
try:
response.update_status('Preparing enviroment converting arguments', 8)
LOGGER.debug('date: %s %s %s %s ' % (type(refSt), refEn, dateSt, dateSt))
# normalize == 'None':
seacyc = False
if outformat == 'ascii':
outformat = '.txt'
elif outformat == 'netCDF':
outformat = '.nc'
else:
LOGGER.exception('output format not valid')
except Exception as e:
msg = 'failed to set environment %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
###########################
# set the environment
###########################
response.update_status('fetching data from archive', 9)
getlevel = False
if 'z' in var:
level = var.strip('z')
else:
level = None
##########################################
# fetch Data from original data archive
##########################################
try:
model_nc = rl(start=dateSt.year, end=dateEn.year,
dataset=model, variable=var,
getlevel=getlevel)
ref_model_nc = rl(start=refSt.year, end=refEn.year,
dataset=ref_model, variable=ref_var,
getlevel=getlevel)
LOGGER.info('reanalyses data fetched')
except Exception:
msg = 'failed to get reanalyses data'
LOGGER.exception(msg)
raise Exception(msg)
response.update_status('subsetting region of interest', 10)
# Checking memory and dataset size
model_size = get_files_size(model_nc)
ref_model_size = get_files_size(ref_model_nc)
m_size = max(model_size, ref_model_size)
memory_avail = psutil.virtual_memory().available
thrs = 0.2 # 20%
if (m_size >= thrs * memory_avail):
ser_r = True
else:
ser_r = False
LOGGER.debug('Available Memory: %s ' % (memory_avail))
LOGGER.debug('Dataset size: %s ' % (m_size))
LOGGER.debug('Threshold: %s ' % (thrs * memory_avail))
LOGGER.debug('Serial or at once: %s ' % (ser_r))
# #####################################################
# Construct descriptive filenames for the three files #
# listed in config file #
# TODO check strftime for years <1900 (!) #
# #####################################################
# refDatesString = dt.strftime(refSt, '%Y-%m-%d') + "_" + dt.strftime(refEn, '%Y-%m-%d')
# simDatesString = dt.strftime(dateSt, '%Y-%m-%d') + "_" + dt.strftime(dateEn, '%Y-%m-%d')
# Fix < 1900 issue...
refDatesString = refSt.isoformat().strip().split("T")[0] + "_" + refEn.isoformat().strip().split("T")[0]
simDatesString = dateSt.isoformat().strip().split("T")[0] + "_" + dateEn.isoformat().strip().split("T")[0]
archiveNameString = "base_" + var + "_" + refDatesString + '_%.1f_%.1f_%.1f_%.1f' \
% (bbox[0], bbox[2], bbox[1], bbox[3])
simNameString = "sim_" + var + "_" + simDatesString + '_%.1f_%.1f_%.1f_%.1f' \
% (bbox[0], bbox[2], bbox[1], bbox[3])
if ('z' in var):
# ------------------ NCEP -------------------
tmp_total = []
origvar = get_variable(model_nc)
for z in model_nc:
b0 = call(resource=z, variable=origvar, level_range=[int(level), int(level)], geom=bbox,
spatial_wrapping='wrap', prefix='levdom_' + os.path.basename(z)[0:-3])
tmp_total.append(b0)
time_range = [dateSt, dateEn]
tmp_total = sorted(tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0])
inter_subset_tmp = call(resource=tmp_total, variable=origvar, time_range=time_range)
# Clean
for i in tmp_total:
tbr = 'rm -f %s' % (i)
os.system(tbr)
# Create new variable
ds = Dataset(inter_subset_tmp, mode='a')
z_var = ds.variables.pop(origvar)
dims = z_var.dimensions
new_var = ds.createVariable('z%s' % level, z_var.dtype, dimensions=(dims[0], dims[2], dims[3]))
new_var[:, :, :] = squeeze(z_var[:, 0, :, :])
ds.close()
simulation = call(inter_subset_tmp, variable='z%s' % level, prefix=simNameString)
# ------------------ 20CRV2c -------------------
tmp_total = []
origvar = get_variable(ref_model_nc)
for z in ref_model_nc:
tmp_n = 'tmp_%s' % (uuid.uuid1())
# select level and regrid
b0 = call(resource=z, variable=origvar, level_range=[int(level), int(level)],
spatial_wrapping='wrap', cdover='system',
regrid_destination=model_nc[0], regrid_options='bil', prefix=tmp_n)
# select domain
b01 = call(resource=b0, variable=origvar, geom=bbox, spatial_wrapping='wrap', prefix='levregr_' + os.path.basename(z)[0:-3])
tbr = 'rm -f %s' % (b0)
os.system(tbr)
tbr = 'rm -f %s.nc' % (tmp_n)
os.system(tbr)
tmp_total.append(b01)
time_range = [refSt, refEn]
tmp_total = sorted(tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0])
ref_inter_subset_tmp = call(resource=tmp_total, variable=origvar, time_range=time_range)
# Clean
for i in tmp_total:
tbr = 'rm -f %s' % (i)
os.system(tbr)
# Create new variable
ds = Dataset(ref_inter_subset_tmp, mode='a')
z_var = ds.variables.pop(origvar)
dims = z_var.dimensions
new_var = ds.createVariable('z%s' % level, z_var.dtype, dimensions=(dims[0], dims[2], dims[3]))
new_var[:, :, :] = squeeze(z_var[:, 0, :, :])
ds.close()
archive = call(ref_inter_subset_tmp, variable='z%s' % level, prefix=archiveNameString)
else:
if ser_r:
LOGGER.debug('Process reanalysis step-by-step')
# ----- NCEP ------
tmp_total = []
for z in model_nc:
b0 = call(resource=z, variable=var, geom=bbox, spatial_wrapping='wrap',
prefix='Rdom_' + os.path.basename(z)[0:-3])
tmp_total.append(b0)
tmp_total = sorted(tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0])
simulation = call(resource=tmp_total, variable=var, time_range=[dateSt, dateEn], prefix=simNameString)
# Clean
for i in tmp_total:
tbr = 'rm -f %s' % (i)
os.system(tbr)
# ----- 20CRV2c ------
tmp_n = 'tmp_%s' % (uuid.uuid1())
tmp_total = []
for z in ref_model_nc:
# regrid
b0 = call(resource=z, variable=ref_var, spatial_wrapping='wrap', cdover='system',
regrid_destination=model_nc[0], regrid_options='bil', prefix=tmp_n)
# select domain
b01 = call(resource=b0, variable=ref_var, geom=bbox, spatial_wrapping='wrap',
prefix='ref_Rdom_' + os.path.basename(z)[0:-3])
tbr = 'rm -f %s' % (b0)
os.system(tbr)
tbr = 'rm -f %s.nc' % (tmp_n)
os.system(tbr)
tmp_total.append(b01)
tmp_total = sorted(tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0])
archive = call(resource=tmp_total, variable=ref_var, time_range=[refSt, refEn], prefix=archiveNameString)
# Clean
for i in tmp_total:
tbr = 'rm -f %s' % (i)
os.system(tbr)
else:
LOGGER.debug('Using whole dataset at once')
simulation = call(resource=model_nc, variable=var,
geom=bbox, spatial_wrapping='wrap', time_range=[dateSt, dateEn], prefix=simNameString)
ref_inter_subset_tmp = call(resource=ref_model_nc, variable=ref_var, spatial_wrapping='wrap',
cdover='system', regrid_destination=model_nc[0], regrid_options='bil')
archive = call(resource=ref_inter_subset_tmp, geom=bbox, spatial_wrapping='wrap', time_range=[refSt, refEn], prefix=archiveNameString)
response.update_status('datasets subsetted', 15)
LOGGER.debug("get_input_subset_dataset took %s seconds.",
time.time() - start_time)
response.update_status('**** Input data fetched', 20)
########################
# input data preperation
########################
response.update_status('Start preparing input data', 30)
start_time = time.time() # measure data preperation ...
LOGGER.info('archive and simulation files generated: %s, %s'
% (archive, simulation))
# Rename variable (TODO: For this specific process we know names: slp and prmsl...)
try:
if level is not None:
out_var = 'z%s' % level
else:
var_archive = get_variable(archive)
var_simulation = get_variable(simulation)
if var_archive != var_simulation:
rename_variable(archive, oldname=var_archive, newname=var_simulation)
out_var = var_simulation
LOGGER.info('varname %s in netCDF renamed to %s' % (var_archive, var_simulation))
except:
msg = 'failed to rename variable in target files'
LOGGER.exception(msg)
raise Exception(msg)
# seacyc is False:
seasoncyc_base = seasoncyc_sim = None
output_file = 'output.txt'
files = [os.path.abspath(archive), os.path.abspath(simulation), output_file]
LOGGER.debug("Data preperation took %s seconds.",
time.time() - start_time)
############################
# generate the config file
############################
config_file = analogs.get_configfile(
files=files,
seasoncyc_base=seasoncyc_base,
seasoncyc_sim=seasoncyc_sim,
base_id=ref_model,
sim_id=model,
timewin=timewin,
varname=out_var,
seacyc=seacyc,
cycsmooth=91,
nanalog=nanalog,
seasonwin=seasonwin,
distfun=distance,
outformat=outformat,
calccor=True,
silent=False,
# period=[dt.strftime(refSt, '%Y-%m-%d'), dt.strftime(refEn, '%Y-%m-%d')],
period=[refSt.isoformat().strip().split("T")[0], refEn.isoformat().strip().split("T")[0]],
bbox="{0[0]},{0[2]},{0[1]},{0[3]}".format(bbox))
response.update_status('generated config file', 40)
#######################
# CASTf90 call
#######################
start_time = time.time() # measure call castf90
# -----------------------
try:
import ctypes
# TODO: This lib is for linux
mkl_rt = ctypes.CDLL('libmkl_rt.so')
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('Current number of threads: %s' % (nth))
mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64)))
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('NEW number of threads: %s' % (nth))
# TODO: Does it \/\/\/ work with default shell=False in subprocess... (?)
os.environ['MKL_NUM_THREADS'] = str(nth)
os.environ['OMP_NUM_THREADS'] = str(nth)
except Exception as e:
msg = 'Failed to set THREADS %s ' % e
LOGGER.debug(msg)
# -----------------------
# ##### TEMPORAL WORKAROUND! With instaled hdf5-1.8.18 in anaconda ###############
# ##### MUST be removed after castf90 recompiled with the latest hdf version
# ##### NOT safe
os.environ['HDF5_DISABLE_VERSION_CHECK'] = '1'
# hdflib = os.path.expanduser("~") + '/anaconda/lib'
# hdflib = os.getenv("HOME") + '/anaconda/lib'
import pwd
hdflib = pwd.getpwuid(os.getuid()).pw_dir + '/anaconda/lib'
os.environ['LD_LIBRARY_PATH'] = hdflib
# ################################################################################
response.update_status('Start CASTf90 call', 50)
try:
# response.update_status('execution of CASTf90', 50)
cmd = ['analogue.out', config_file]
LOGGER.debug("castf90 command: %s", cmd)
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
LOGGER.info('analogue output:\n %s', output)
response.update_status('**** CASTf90 suceeded', 60)
except CalledProcessError as e:
msg = 'CASTf90 failed:\n{0}'.format(e.output)
LOGGER.exception(msg)
raise Exception(msg)
LOGGER.debug("castf90 took %s seconds.", time.time() - start_time)
# TODO: Add try - except for pdfs
if plot == 'Yes':
analogs_pdf = analogs.plot_analogs(configfile=config_file)
else:
analogs_pdf = 'dummy_plot.pdf'
with open(analogs_pdf, 'a'): os.utime(analogs_pdf, None)
response.update_status('preparing output', 70)
response.outputs['analog_pdf'].file = analogs_pdf
response.outputs['config'].file = config_file
response.outputs['analogs'].file = output_file
########################
# generate analog viewer
########################
formated_analogs_file = analogs.reformat_analogs(output_file)
# response.outputs['formated_analogs'].storage = FileStorage()
response.outputs['formated_analogs'].file = formated_analogs_file
LOGGER.info('analogs reformated')
response.update_status('reformatted analog file', 80)
viewer_html = analogs.render_viewer(
# configfile=response.outputs['config'].get_url(),
configfile=config_file,
# datafile=response.outputs['formated_analogs'].get_url())
datafile=formated_analogs_file)
response.outputs['output'].file = viewer_html
response.update_status('Successfully generated analogs viewer', 90)
LOGGER.info('rendered pages: %s ', viewer_html)
response.update_status('execution ended', 100)
LOGGER.debug("total execution took %s seconds.",
time.time() - process_start_time)
return response