def execute(self):
######################
# start execution
######################
init_process_logger('log.txt')
self.output_log.setValue('log.txt')
from flyingpigeon import analogs as anlg
from flyingpigeon import config
from os.path import basename
###########################################
# reorganize analog txt file for javascript
# and find associated config file
###########################################
# Reformat data file output by the analogs detection process so that
# it can be read by the analogues viewer template.
try:
# Get the output csv file of analogs process (input by user in
# text box)
analogs = self.getInputValues(identifier='resource')[0]
configfile = anlg.get_viewer_configfile(analogs)
f = anlg.reformat_analogs(analogs)
logger.info('Analog file reformatted')
self.status.set('Successfully reformatted analog file', 50)
output_av = anlg.get_viewer(f, configfile)
logger.info('Viewer html page generated')
self.status.set(
'Successfully generated analogs viewer html page', 90)
outputUrl_path = config.outputUrl_path()
def _handler(self, request, response):
from flyingpigeon.log import init_process_logger
from flyingpigeon.utils import rename_complexinputs
from flyingpigeon.datafetch import write_fileinfo
import os
response.update_status("start fetching resource", 10)
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
resource = rename_complexinputs(request.inputs['resource'])
response.outputs['output'].file = write_fileinfo(resource,
filepath=True)
# filepathes = 'out.txt'
# with open(filepathes, 'w') as fp:
# fp.write('###############################################\n')
# fp.write('###############################################\n')
# fp.write('Following files are stored to your local discs: \n')
# fp.write('\n')
# for f in resources:
# fp.write('%s \n' % os.path.realpath(f))
# response.outputs['output'].file = filepathes
response.update_status("done", 100)
return response
def execute(self):
self.status.set('starting uncertainty process', 0)
init_process_logger('log.txt')
self.output_log.setValue('log.txt')
from flyingpigeon.utils import archiveextract
ncfiles = archiveextract(self.getInputValues(identifier='resource'))
start = self.start.getValue()
end = self.end.getValue()
timeslice = self.timeslice.getValue()
variable = self.variableIn.getValue()
method = self.method.getValue()
self.status.set('arguments read', 5)
if method == 'Method_A':
signal, low_agreement_mask, high_agreement_mask, graphic, text_src = erob.method_A(
resource=ncfiles,
start=start,
end=end,
timeslice=timeslice,
variable=variable) # graphic,
self.status.set('process worker done', 95)
self.output_signal.setValue(signal)
self.output_high.setValue(high_agreement_mask)
self.output_low.setValue(low_agreement_mask)
self.output_graphic.setValue(graphic)
self.output_text.setValue(text_src)
self.status.set('uncertainty process done', 100)
def _handler(self, request, response):
import uuid
import time
import json
outputpath = configuration.get_config_value('server', 'outputpath')
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
# -------------- #
# Input handling #
# -------------- #
resource = archiveextract(
resource=rename_complexinputs(request.inputs['resource']))
LOGGER.info("resource: %s " % resource)
dest = archiveextract(
resource=rename_complexinputs(request.inputs['dest']))
LOGGER.info("dest: %s " % dest)
method = request.inputs['method'][0].data
LOGGER.info("method: %s " % method)
snippet = request.inputs['snippet'][0].data
LOGGER.info("snippet: %s " % snippet)
# -------------------- #
# Regridding operation #
# -------------------- #
d = ocgis.RequestDataset(dest)
m = getattr(ESMF.RegridMethod, method.upper())
LOGGER.info('Start ocgis module call function')
# Prepare the environment
ocgis.env.OVERWRITE = True
prefix = str(uuid.uuid1())
ocgis.env.PREFIX = prefix
outputs = []
for source in resource:
s = ocgis.RequestDataset(source)
ops = ocgis.OcgOperations(dataset=s,
regrid_destination=d,
regrid_options={'regrid_method': m},
snippet=snippet,
dir_output=outputpath,
output_format='nc',
prefix=prefix)
outputs.append(ops.execute())
response.outputs['output_netcdf'].file = outputs[0]
time_str = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
output_json = "esmf_regrid_results_{}.json".format(time_str)
with open(output_json, 'w') as f:
f.write(json.dumps([actual_output_path(o) for o in outputs]))
response.outputs['output'].file = output_json
response.outputs['output'].output_format = json_format
return response
def execute(self):
from ast import literal_eval
from flyingpigeon.utils import archive, archiveextract
init_process_logger('log.txt')
self.output_log.setValue('log.txt')
ncs = archiveextract(self.getInputValues(identifier='resource'))
mosaic = self.mosaic.getValue()
regions = self.region.getValue()
# variable = self.variable.getValue()
# logger.info('regions: %s' % regions)
# dimension_map = self.dimension_map.getValue()
# if dimension_map != None:
# dimension_map = literal_eval(dimension_map)
logger.info('ncs = %s', ncs)
logger.info('regions = %s', regions)
logger.info('mosaic = %s', mosaic)
# logger.info('dimension_map = %s', dimension_map)
self.status.set('Arguments set for subset process', 0)
logger.debug('starting: regions=%s, num_files=%s' %
(len(regions), len(ncs)))
try:
results = clipping(
resource=ncs,
polygons=regions, # self.region.getValue(),
mosaic=mosaic,
spatial_wrapping='wrap',
# variable=variable,
dir_output=os.path.abspath(os.curdir),
# dimension_map=dimension_map,
)
logger.info('results %s' % results)
except Exception as e:
msg = 'clipping failed'
logger.exception(msg)
raise Exception(msg)
if not results:
raise Exception('no results produced.')
# prepare tar file
try:
tarf = archive(results)
logger.info('Tar file prepared')
except Exception as e:
msg = 'Tar file preparation failed'
logger.exception(msg)
raise Exception(msg)
self.output.setValue(tarf)
i = next((i for i, x in enumerate(results) if x), None)
self.output_netcdf.setValue(results[i])
self.status.set('done', 100)
def _handler(self, request, response):
from rpy2 import robjects
from rpy2.robjects.packages import importr
import os
import datetime as dt
tic = dt.datetime.now()
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
LOGGER.info('Start process')
response.update_status('Execution started at : {}'.format(tic), 1)
######################################
# Read inputs
######################################
try:
obs = archiveextract(
resource=rename_complexinputs(request.inputs['obs']))
ref = archiveextract(
resource=rename_complexinputs(request.inputs['ref']))
fut = archiveextract(
resource=rename_complexinputs(request.inputs['fut']))
# dedrizzle = request.inputs['dedrizzle'][0].data
# norm = request.inputs['norm'][0].data
except Exception as e:
msg = 'Failed to read input parameter {}'.format(e)
msg += "obs: " + request.inputs['obs']
msg += "ref: " + request.inputs['ref']
msg += "fut: " + request.inputs['fut']
LOGGER.error(msg)
raise Exception(msg)
response.update_status('Input parameters ingested', 2)
rp, ext = os.path.splitext(ref[0])
ref_out = rp + '_kddm-bc' + ext
fp, ext = os.path.splitext(fut[0])
fut_out = fp + '_kddm-bc' + ext
# Assuming all files share the same variable.
rd = ocgis.RequestDataset(ref)
varname = rd.variable
# Calling the R code
Rsrc = os.path.join(flyingpigeon.config.Rsrc_dir(), 'bc.kddm.R')
devtools = importr("devtools")
rfunc = robjects.r(open(Rsrc).read())
rfunc(varname, obs[0], ref[0], fut[0], ref_out, fut_out, False)
response.outputs['output_netcdf_ref'].file = ref_out
response.outputs['output_netcdf_fut'].file = fut_out
return response
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
#
#
# init_process_logger('log.txt')
# self.output_log.setValue('log.txt')
#
response.update_status('Start process', 0)
from flyingpigeon import sdm
try:
LOGGER.info('reading the arguments')
taxon_name = request.inputs['taxon_name'][0].data
bbox = [-180, -90, 180, 90]
# bbox_obj = self.BBox.getValue()
# bbox = [bbox_obj.coords[0][0],
# bbox_obj.coords[0][1],
# bbox_obj.coords[1][0],
# bbox_obj.coords[1][1]]
LOGGER.info("bbox={0}".format(bbox))
LOGGER.info("Taxon Name = %s", taxon_name)
except:
msg = 'failed to read in the arguments.'
LOGGER.exception(msg)
raise Exception(msg)
try:
response.update_status('Fetching GBIF Data', 10)
gbifdic = sdm.get_gbif(taxon_name, bbox=bbox)
except:
msg = 'failed to search gbif.'
LOGGER.exception(msg)
raise Exception(msg)
try:
response.update_status('write csv file', 70)
gbifcsv = sdm.gbifdic2csv(gbifdic)
except:
msg = 'failed to write csv file.'
LOGGER.exception(msg)
raise Exception(msg)
try:
response.update_status('plot map', 80)
from flyingpigeon.visualisation import map_gbifoccurrences
latlon = sdm.latlon_gbifdic(gbifdic)
occurence_map = map_gbifoccurrences(latlon)
except:
msg = 'failed to plot occurence map.'
LOGGER.exception(msg)
raise Exception(msg)
response.outputs['output_map'].file = occurence_map
response.outputs['output_csv'].file = gbifcsv
response.update_status('done', 100)
return response
def execute(self):
from flyingpigeon.utils import archive, archiveextract
from tempfile import mkstemp
init_process_logger('log.txt')
self.output_log.setValue('log.txt')
ncs = archiveextract(self.getInputValues(identifier='resource'))
# mosaic = self.mosaic.getValue()
regions = self.region.getValue()
self.status.set('Arguments set for subset process', 0)
logger.debug('starting: regions=%s, num_files=%s' % (len(regions), len(ncs)))
try:
from flyingpigeon.visualisation import plot_polygons
png_country = plot_polygons(regions)
except:
logger.exception('failed to plot the polygon to world map')
o1, factsheet_plot = mkstemp(dir='.', suffix='.png')
# clip the demanded polygons
from flyingpigeon.subset import clipping
subsets = clipping(resource=ncs, variable=None,
dimension_map=None,
calc=None,
output_format='nc',
calc_grouping=None,
time_range=None,
time_region=None,
historical_concatination=True,
prefix=None,
spatial_wrapping='wrap',
polygons=regions,
mosaic=True
)
try:
from flyingpigeon.visualisation import uncertainty
png_uncertainty = uncertainty(subsets)
except:
logger.exception('failed to generate the uncertainty plot')
_, png_uncertainty = mkstemp(dir='.', suffix='.png')
try:
from flyingpigeon.visualisation import spaghetti
png_spaghetti = spaghetti(subsets)
except:
logger.exception('failed to generate the spaghetti plot')
_, png_spaghetti = mkstemp(dir='.', suffix='.png')
from flyingpigeon.visualisation import factsheetbrewer
factsheet = factsheetbrewer(png_country=png_country,
png_uncertainty=png_uncertainty,
png_spaghetti=png_spaghetti)
self.output_factsheet.setValue(factsheet)
self.status.set('done', 100)
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
response.update_status('Start process', 0)
try:
LOGGER.info('reading the arguments')
resources = archiveextract(
resource=rename_complexinputs(request.inputs['resource']))
indices = [inpt.data for inpt in request.inputs['indices']]
LOGGER.debug("indices = %s", indices)
archive_format = request.inputs['archive_format'][0].data
except:
msg = 'failed to read the arguments.'
LOGGER.exception(msg)
raise Exception(msg)
LOGGER.info('indices %s ' % indices)
#################################
# calculate the climate indices
#################################
# indices calculation
ncs_indices = None
datasets = sort_by_filename(resources, historical_concatination=True)
LOGGER.debug("datasets=%s", datasets.keys())
for ds_name in datasets:
try:
response.update_status('calculation of {}'.format(ds_name), 30)
# TODO: what is happening with the results for each ds?
ncs_indices = sdm.get_indices(resource=datasets[ds_name],
indices=indices)
except:
msg = 'indice calculation failed for {}'.format(ds_name)
LOGGER.exception(msg)
raise Exception(msg)
# archive multiple output files to one archive file
try:
archive_indices = archive(ncs_indices, format=archive_format)
LOGGER.info('indices 3D added to tarfile')
except:
msg = 'failed adding indices to tar'
LOGGER.exception(msg)
raise Exception(msg)
response.outputs['output_indices'].file = archive_indices
i = next((i for i, x in enumerate(ncs_indices) if x), None)
response.outputs['ncout'].file = ncs_indices[i]
response.update_status('done', 100)
return response
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
ncs = archiveextract(
resource=rename_complexinputs(request.inputs['resource']))
LOGGER.info('ncs: {}'.format(ncs))
coords = []
for coord in request.inputs['coords']:
coords.append(coord.data)
LOGGER.info('coords {}'.format(coords))
filenames = []
nc_exp = sort_by_filename(ncs, historical_concatination=True)
for key in nc_exp.keys():
try:
LOGGER.info('start calculation for {}'.format(key))
ncs = nc_exp[key]
times = get_time(ncs) # , format='%Y-%m-%d_%H:%M:%S')
concat_vals = times # ['%s-%02d-%02d_%02d:%02d:%02d' %
# (t.year, t.month, t.day, t.hour, t.minute, t.second) for t in times]
header = 'date_time'
filename = '{}.csv'.format(key)
filenames.append(filename)
for p in coords:
try:
response.update_status('processing point: {}'.format(p), 20)
# define the point:
p = p.split(',')
point = Point(float(p[0]), float(p[1]))
# get the values
timeseries = call(resource=ncs, geom=point, select_nearest=True)
vals = get_values(timeseries)
# concatenation of values
header = header + ',{}-{}'.format(p[0], p[1])
concat_vals = column_stack([concat_vals, vals])
except Exception as e:
LOGGER.debug('failed for point {} {}'.format(p, e))
response.update_status('*** all points processed for {0} ****'.format(key), 50)
# TODO: Ascertain whether this 'savetxt' is a valid command without string formatting argument: '%s'
savetxt(filename, concat_vals, fmt='%s', delimiter=',', header=header)
except Exception as ex:
LOGGER.debug('failed for {}: {}'.format(key, str(ex)))
# set the outputs
response.update_status('*** creating output tar archive ****', 90)
tarout_file = archive(filenames)
response.outputs['tarout'].file = tarout_file
return response
def execute(self):
init_process_logger('log.txt')
self.output_log.setValue('log.txt')
from flyingpigeon.utils import searchfile
from flyingpigeon.subset import masking
from flyingpigeon.utils import archive, archiveextract
from flyingpigeon import config
from os import path
resources = archiveextract(self.getInputValues(identifier='resource'))
masks = archiveextract(self.getInputValues(identifier='mask'))
land_area = self.land_area.getValue()
fp_cache = config.cache_path().split('/')
base_dir = '/'.join(fp_cache[0:-1]) # base dir for all birds
logger.debug('base dir of directory tree: %s' % base_dir)
ncs = []
sftlf = []
for nc in resources:
try:
basename = path.basename(nc)
bs = basename.split('_')
pattern = 'sftlf_' + '_'.join(bs[1:-2]) + '_fx.nc'
pattern = pattern.replace('historical',
'*').replace('rcp85',
'*').replace('rcp65',
'*').replace('rcp45',
'*').replace('rcp26', '*')
logger.debug('searching for %s ' % pattern)
sftlf.extend(searchfile(pattern, path.curdir))
sftlf.extend(searchfile(pattern, base_dir))
logger.debug('lenght of sftlf: %s' % len(sftlf))
if len(sftlf) >= 1:
if len(sftlf) > 1:
logger.warn(
'more than one sftlf file is found fitting to the pattern, first one will be taken %s'
% sftlf[0])
prefix = 'masked%s' % basename.replace('.nc', '')
nc_mask = masking(nc, sftlf[0], land_area=land_area, prefix=prefix)
ncs.extend([nc_mask])
logger.info('masking processed for %s' % basename)
else:
logger.warn('no masked found. Please perform a "Download Resources"\
to make sure the land_area file is in cache')
except:
logger.exception('failed to mask file: %s' % basename)
nc_archive = archive(ncs)
self.output_archive.setValue(nc_archive)
i = next((i for i, x in enumerate(ncs) if x), None)
self.output_example.setValue(ncs[i])
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
datasets = []
# append file urls
if 'dataset' in request.inputs:
datasets.extend(
archiveextract(
resource=rename_complexinputs(request.inputs['dataset'])))
# append opendap urls
if 'dataset_opendap' in request.inputs:
for dataset in request.inputs['dataset_opendap']:
datasets.append(dataset.data)
# land or sea flag
land_area_flag = request.inputs['land_or_sea'][0].data == 'land'
masked_datasets = []
count = 0
max_count = len(datasets)
for ds in datasets:
ds_name = os.path.basename(ds)
LOGGER.info('masking dataset: {}'.format(ds_name))
if 'mask' in request.inputs:
landsea_mask = request.inputs['mask'][0].data
else:
landsea_mask = search_landsea_mask_by_esgf(ds)
LOGGER.info("using landsea_mask: {}".format(landsea_mask))
prefix = 'masked_{}'.format(ds_name.replace('.nc', ''))
try:
new_ds = masking(ds,
landsea_mask,
land_area=land_area_flag,
prefix=prefix)
masked_datasets.append(new_ds)
except Exception as ex:
msg = 'Could not subset dataset {}: {}'.format(
ds_name, str(ex))
LOGGER.exception(msg)
raise Exception(msg)
count = count + 1
response.update_status(
"masked: {:d}/{:d}".format(count, max_count),
int(100.0 * count / max_count))
response.outputs['output_archive'].file = archive(masked_datasets)
response.outputs['output_example'].file = masked_datasets[0]
response.update_status("done", 100)
return response
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
ncs = archiveextract(
resource=rename_complexinputs(request.inputs['resource']))
LOGGER.info("ncs: %s " % ncs)
coords = request.inputs['coords'] # self.getInputValues(identifier='coords')
LOGGER.info("coords %s", coords)
filenames = []
nc_exp = sort_by_filename(ncs, historical_concatination=True)
for key in nc_exp.keys():
try:
LOGGER.info('start calculation for %s ' % key)
ncs = nc_exp[key]
times = get_time(ncs, format='%Y-%m-%d_%H:%M:%S')
concat_vals = times # ['%s-%02d-%02d_%02d:%02d:%02d' %
# (t.year, t.month, t.day, t.hour, t.minute, t.second) for t in times]
header = 'date_time'
filename = '%s.csv' % key
filenames.append(filename)
for p in coords:
try:
response.update_status('processing point : {0}'.format(p), 20)
# define the point:
p = p.split(',')
point = Point(float(p[0]), float(p[1]))
# get the values
timeseries = call(resource=ncs, geom=point, select_nearest=True)
vals = get_values(timeseries)
# concatenation of values
header = header + ',%s-%s' % (p[0], p[1])
concat_vals = column_stack([concat_vals, vals])
except Exception as e:
LOGGER.debug('failed for point %s %s' % (p, e))
response.update_status('*** all points processed for {0} ****'.format(key), 50)
savetxt(filename, concat_vals, fmt='%s', delimiter=',', header=header)
except Exception as e:
LOGGER.debug('failed for %s %s' % (key, e))
# set the outputs
response.update_status('*** creating output tar archive ****', 90)
tarout_file = archive(filenames)
response.outputs['tarout'].file = tarout_file
return response
def _handler(self, request, response):
from tempfile import mkstemp
tic = dt.now()
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
LOGGER.info('Start process')
response.update_status('Execution started at : {}'.format(tic), 1)
######################################
# Read inputs
######################################
try:
resource = archiveextract(
resource=rename_complexinputs(request.inputs['resource']))[0]
fmts = [e.data for e in request.inputs['fmt']]
title = request.inputs['title'][0].data
except Exception as e:
msg = 'Failed to read input parameter {}'.format(e)
LOGGER.error(msg)
raise Exception(msg)
response.update_status('Input parameters ingested', 2)
try:
fig = map_spatial_analog(resource, title=title)
output = []
for fmt in fmts:
output.append(fig2plot(fig, fmt))
except Exception as e:
msg = "Failed to create figure: {}".format(e)
LOGGER.error(msg)
raise Exception(msg)
finally:
plt.close()
if len(fmts) == 1:
output = output[0]
else:
output = archive(output)
response.outputs['output_figure'].file = output
response.update_status("done", 100)
return response
def execute(self):
init_process_logger('log.txt')
self.output_log.setValue('log.txt')
resources = self.getInputValues(identifier='resource')
filename = 'out.txt'
with open(filename, 'w') as fp:
fp.write('###############################################\n')
fp.write('###############################################\n')
fp.write('Following files are stored to your local discs: \n')
fp.write('\n')
for resource in resources:
fp.write('%s \n' % os.path.realpath(resource))
self.output.setValue(filename)
def execute(self):
from os.path import basename
from flyingpigeon import sdm
from flyingpigeon.utils import archive
init_process_logger('log.txt')
self.output_log.setValue('log.txt')
self.status.set('Start process', 0)
try:
logger.info('reading the arguments')
resources = self.getInputValues(identifier='resources')
indices = self.getInputValues(identifier='input_indices')
logger.debug("indices = %s", indices)
archive_format = self.archive_format.getValue()
except Exception as e:
logger.error('failed to read in the arguments %s ' % e)
logger.info('indices %s ' % indices)
#################################
# calculate the climate indices
#################################
# indices calculation
ncs_indices = None
try:
self.status.set(
'start calculation of climate indices for %s' % indices, 30)
ncs_indices = sdm.get_indices(resources=resources, indices=indices)
logger.info('indice calculation done')
except:
msg = 'failed to calculate indices'
logger.exception(msg)
raise Exception(msg)
# archive multiple output files to one archive file
try:
archive_indices = archive(ncs_indices, format=archive_format)
logger.info('indices 3D added to tarfile')
except:
msg = 'failed adding indices to tar'
logger.exception(msg)
raise Exception(msg)
self.output_indices.setValue(archive_indices)
self.status.set('done', 100)
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
ncfiles = archiveextract(
resource=rename_complexinputs(request.inputs['resource']))
if 'variable' in request.inputs:
var = request.inputs['variable'][0].data
else:
var = get_variable(ncfiles[0])
# var = ncfiles[0].split("_")[0]
response.update_status('plotting variable %s' % var, 10)
try:
plotout_spagetti_file = vs.spaghetti(
ncfiles,
variable=var,
title='Fieldmean of %s ' % (var),
)
LOGGER.info("spagetti plot done")
response.update_status(
'Spagetti plot for %s %s files done' % (len(ncfiles), var), 50)
response.outputs['plotout_spagetti'].file = plotout_spagetti_file
except Exception:
raise Exception("spagetti plot failed")
try:
plotout_uncertainty_file = vs.uncertainty(
ncfiles,
variable=var,
title='Ensemble uncertainty for %s ' % (var),
)
response.update_status(
'Uncertainty plot for %s %s files done' % (len(ncfiles), var),
90)
response.outputs[
'plotout_uncertainty'].file = plotout_uncertainty_file
LOGGER.info("uncertainty plot done")
except Exception as err:
raise Exception("uncertainty plot failed %s" % err.message)
response.update_status('visualisation done', 100)
return response
def execute(self):
init_process_logger('log.txt')
self.output_log.setValue('log.txt')
ncs = self.getInputValues(identifier='resource')
indices = self.indices.getValue()
polygons = self.polygons.getValue()
percentile = int(self.percentile.getValue())
groupings = self.groupings.getValue()
mosaic = self.mosaic.getValue()
refperiod = self.refperiod.getValue()
self.status.set(
'starting: indices=%s, refperiod=%s, groupings=%s, num_files=%s' %
(indices, refperiod, groupings, len(ncs)), 0)
from flyingpigeon.indices import calc_indice_percentile
results = calc_indice_percentile(
resources=ncs,
indices=indices,
percentile=percentile,
mosaic=mosaic,
polygons=polygons,
refperiod=refperiod,
groupings=groupings,
dir_output=path.curdir,
)
if not results:
raise Exception("failed to produce results")
self.status.set('num results %s' % len(results), 90)
try:
(fp_tarf, tarf) = mkstemp(dir=".", suffix='.tar')
tar = tarfile.open(tarf, "w")
for result in results:
tar.add(result, arcname=os.path.basename(result))
tar.close()
logging.info('Tar file prepared')
except Exception as e:
msg = "Tar file preparation failed."
logging.exception(msg)
raise Exception(msg)
self.output.setValue(tarf)
self.status.set('done: indice=%s, num_files=%s' % (indices, len(ncs)),
100)
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
ncfiles = archiveextract(
resource=rename_complexinputs(request.inputs['resource']))
var = request.inputs['variableIn']
if var is None:
from flyingpigeon.utils import get_variable
var = get_variable(ncfiles[0])
response.update_status('plotting variable %s' % var, 10)
try:
plotout_spagetti_file = vs.spaghetti(ncfiles,
variable=var,
title='Fieldmean of %s ' %
(var),
dir_out=None)
LOGGER.info("spagetti plot done")
response.update_status(
'Spagetti plot for %s %s files done' % (len(ncfiles), var), 50)
except:
LOGGER.exception("spagetti plot failed")
try:
plotout_uncertainty_file = vs.uncertainty(
ncfiles,
variable=var,
title='Ensemble uncertainty for %s ' % (var),
dir_out=None)
response.update_status(
'Uncertainty plot for %s %s files done' % (len(ncfiles), var),
90)
LOGGER.info("uncertainty plot done")
except:
LOGGER.exception("uncertainty plot failed")
response.outputs['plotout_spagetti'].file = plotout_spagetti_file
response.outputs['plotout_uncertainty'].file = plotout_uncertainty_file
response.update_status('visualisation done', 100)
return response
def execute(self):
init_process_logger('log.txt')
self.output_log.setValue('log.txt')
from flyingpigeon.utils import archiveextract
ncfiles = archiveextract(self.getInputValues(identifier='resource'))
var = self.variableIn.getValue()
if var is None:
from flyingpigeon.utils import get_variable
var = get_variable(ncfiles[0])
self.status.set('plotting variable %s' % var, 10)
try:
plotout_spagetti_file = vs.spaghetti(
ncfiles,
variable=var,
title='Fieldmean of %s ' % (var),
dir_out=None
)
logger.info("spagetti plot done")
self.status.set('Spagetti plot for %s %s files done' % (len(ncfiles), var), 50)
except:
logger.exception("spagetti plot failed")
try:
plotout_uncertainty_file = vs.uncertainty(
ncfiles,
variable=var,
title='Ensemble uncertainty for %s ' % (var),
dir_out=None
)
self.status.set('Uncertainty plot for %s %s files done' % (len(ncfiles), var), 90)
logger.info("uncertainty plot done")
except:
logger.exception("uncertainty plot failed")
self.plotout_spagetti.setValue(plotout_spagetti_file)
self.plotout_uncertainty.setValue(plotout_uncertainty_file)
self.status.set('visualisation done', 100)
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
###########################################
# reorganize analog txt file for javascript
# and find associated config file
###########################################
# Reformat data file output by the analogs detection process so that
# it can be read by the analogues viewer template.
try:
# Get the output csv file of analogs process (input by user in
# text box)
analogs = rename_complexinputs(request.inputs['analog_result'])[0]
# analogs = request.inputs['analog_result'][0]
LOGGER.info("analogs file path %s ", analogs)
configfile = "dummy.txt" # anlg.get_viewer_configfile(analogs)
analogs_mod = anlg.reformat_analogs(analogs)
response.outputs['output_txt'].file = analogs_mod # output_data
LOGGER.info("analogs for visualisation prepared")
except Exception:
msg = 'Failed to reformat analogs file'
LOGGER.exception(msg)
raise Exception(msg)
try:
output_av = anlg.get_viewer(configfile=basename(configfile),
datafile=basename(analogs_mod))
LOGGER.info('Viewer html page generated')
response.update_status(
'Successfully generated analogs viewer html page', 90)
response.outputs['output_html'].file = output_av
LOGGER.info('output_av: %s ', output_av)
except Exception:
msg = 'Failed to generate viewer'
LOGGER.exception(msg)
raise Exception(msg)
return response
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
######################################
# Process inputs
######################################
try:
resources = self._resource_input_handler(request)
options = self._option_input_handler(request)
extras = self._extra_input_handler(request)
except Exception as e:
msg = 'Failed to read input parameter {}'.format(e)
LOGGER.error(msg)
raise Exception(msg)
response.update_status('Input parameters ingested', 2)
######################################
# Call ocgis function
######################################
# Mapping for multivariate functions
if getattr(self, 'has_required_variables', None):
extras.update({k: k for k in resources.keys()})
output = run_op(resource=resources,
calc=[{
'func': self.identifier,
'name': self.identifier,
'kwds': extras
}],
options=options)
response.outputs['output_netcdf'].file = output
response.update_status('Execution completed', 100)
return response
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
###########################################
# reorganize analog txt file for javascript
# and find associated config file
###########################################
# Reformat data file output by the analogs detection process so that
# it can be read by the analogues viewer template.
try:
# Get the output csv file of analogs process (input by user in
# text box)
analogs = request.inputs['analog_result'][0].data
configfile = anlg.get_viewer_configfile(analogs)
f = anlg.reformat_analogs(analogs)
LOGGER.info('Analog file reformatted')
response.update_status('Successfully reformatted analog file', 50)
output_av = anlg.get_viewer(f, configfile)
LOGGER.info('Viewer html page generated')
response.update_status(
'Successfully generated analogs viewer html page', 90)
output_url = config.output_url()
output_data = output_url + '/' + basename(f)
LOGGER.info('Data url: %s ' % output_data)
LOGGER.info('output_av: %s ' % output_av)
except Exception as e:
msg = 'Failed to reformat analogs file or generate viewer%s ' % e
LOGGER.debug(msg)
################################
# set the outputs
################################
response.outputs['output_txt'] = output_data
response.outputs['output_htm'] = output_av
return response
def _handler(self, request, response):
resource = archiveextract(resource=rename_complexinputs(
request.inputs['resource']))[0]
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
LOGGER.info('Start process')
LOGGER.debug("Rank : {}".format(ocgis.vm.rank))
print("Rank : {}".format(ocgis.vm.rank))
rd = ocgis.RequestDataset(resource)
ops = ocgis.OcgOperations(dataset=rd,
calc=[{'func': 'mean', 'name': 'mean'}],
calc_grouping=['month'],
output_format='nc')
output = ops.execute()
response.outputs['output_netcdf'].file = output
response.update_status('Execution completed', 100)
return response
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
# input files
LOGGER.debug("url=%s, mime_type=%s", request.inputs['resource'][0].url,
request.inputs['resource'][0].data_format.mime_type)
ncs = archiveextract(
resource=rename_complexinputs(request.inputs['resource']))
# mime_type=request.inputs['resource'][0].data_format.mime_type)
# mosaic option
# TODO: fix defaults in pywps 4.x
if 'mosaic' in request.inputs:
mosaic = request.inputs['mosaic'][0].data
else:
mosaic = False
# regions used for subsetting
regions = [inp.data for inp in request.inputs['region']]
LOGGER.info('ncs = %s', ncs)
LOGGER.info('regions = %s', regions)
LOGGER.info('mosaic = %s', mosaic)
response.update_status("Arguments set for subset process", 0)
LOGGER.debug('starting: regions=%s, num_files=%s', len(regions),
len(ncs))
try:
results = clipping(
resource=ncs,
polygons=regions, # self.region.getValue(),
mosaic=mosaic,
spatial_wrapping='wrap',
# variable=variable,
# dir_output=os.path.abspath(os.curdir),
# dimension_map=dimension_map,
)
LOGGER.info('results %s' % results)
except Exception as e:
msg = 'clipping failed: %s' % e
LOGGER.exception(msg)
raise Exception(msg)
if not results:
raise Exception('No results produced.')
# prepare tar file
try:
tarf = archive(results)
LOGGER.info('Tar file prepared')
except Exception as e:
msg = 'Tar file preparation failed: %s' % e
LOGGER.exception(msg)
raise Exception(msg)
response.outputs['output'].file = tarf
i = next((i for i, x in enumerate(results) if x), None)
response.outputs['ncout'].file = results[i]
response.update_status("done", 100)
return response
def execute(self):
init_process_logger('log.txt')
self.output_log.setValue('log.txt')
logger.info('Start process')
from datetime import datetime as dt
from flyingpigeon import weatherregimes as wr
from tempfile import mkstemp
self.status.set('execution started at : %s ' % dt.now(), 5)
################################
# reading in the input arguments
################################
try:
logger.info('read in the arguments')
# resources = self.getInputValues(identifier='resources')
season = self.getInputValues(identifier='season')[0]
bbox_obj = self.BBox.getValue()
model_var = self.getInputValues(identifier='reanalyses')[0]
period = self.getInputValues(identifier='period')[0]
anualcycle = self.getInputValues(identifier='anualcycle')[0]
model, variable = model_var.split('_')
kappa = int(self.getInputValues(identifier='kappa')[0])
logger.info('period %s' % str(period))
logger.info('season %s' % str(season))
except Exception as e:
logger.debug('failed to read in the arguments %s ' % e)
try:
start = dt.strptime(period.split('-')[0], '%Y%m%d')
end = dt.strptime(period.split('-')[1], '%Y%m%d')
if bbox_obj is not None:
logger.info("bbox_obj={0}".format(bbox_obj.coords))
bbox = [
bbox_obj.coords[0][0], bbox_obj.coords[0][1],
bbox_obj.coords[1][0], bbox_obj.coords[1][1]
]
logger.info("bbox={0}".format(bbox))
else:
bbox = None
except Exception as e:
logger.debug('failed to transform BBOXObject %s ' % e)
###########################
# set the environment
###########################
self.status.set('fetching data from archive', 10)
try:
if model == 'NCEP':
if 'z' in variable:
level = variable.strip('z')
conform_units_to = None
else:
level = None
conform_units_to = 'hPa'
elif '20CRV2' in model:
if 'z' in variable:
level = variable.strip('z')
conform_units_to = None
else:
level = None
conform_units_to = 'hPa'
else:
logger.error('Reanalyses dataset not known')
logger.info('environment set')
except Exception as e:
msg = 'failed to set environment %s ' % e
logger.error(msg)
raise Exception(msg)
##########################################
# fetch Data from original data archive
##########################################
from flyingpigeon.datafetch import reanalyses as rl
try:
model_nc = rl(start=start.year,
end=end.year,
dataset=model,
variable=variable)
logger.info('reanalyses data fetched')
except Exception as e:
msg = 'failed to get reanalyses data %s' % e
logger.debug(msg)
raise Exception(msg)
self.status.set('fetching data done', 15)
############################################################
# get the required bbox and time region from resource data
############################################################
self.status.set('subsetting region of interest', 17)
# from flyingpigeon.weatherregimes import get_level
from flyingpigeon.ocgis_module import call
time_range = [start, end]
model_subset = call(
resource=model_nc,
variable=variable,
geom=bbox,
spatial_wrapping='wrap',
time_range=time_range,
# conform_units_to=conform_units_to
)
logger.info('Dataset subset done: %s ' % model_subset)
self.status.set('dataset subsetted', 19)
##############################################
# computing anomalies
##############################################
self.status.set('computing anomalies ', 19)
cycst = anualcycle.split('-')[0]
cycen = anualcycle.split('-')[0]
reference = [
dt.strptime(cycst, '%Y%m%d'),
dt.strptime(cycen, '%Y%m%d')
]
logger.debug('reference time: %s' % reference)
model_anomal = wr.get_anomalies(model_subset, reference=reference)
#####################
# extracting season
#####################
self.status.set('normalizing data', 21)
model_season = wr.get_season(model_anomal, season=season)
self.status.set('anomalies computed and normalized', 24)
#######################
# call the R scripts
#######################
self.status.set('Start weather regime clustering ', 25)
import shlex
import subprocess
from flyingpigeon import config
from os.path import curdir, exists, join
try:
rworkspace = curdir
Rsrc = config.Rsrc_dir()
Rfile = 'weatherregimes_model.R'
infile = model_season # model_subset #model_ponderate
modelname = model
yr1 = start.year
yr2 = end.year
ip, output_graphics = mkstemp(dir=curdir, suffix='.pdf')
ip, file_pca = mkstemp(dir=curdir, suffix='.txt')
ip, file_class = mkstemp(dir=curdir, suffix='.Rdat')
args = [
'Rscript',
join(Rsrc, Rfile),
'%s/' % curdir,
'%s/' % Rsrc,
'%s' % infile,
'%s' % variable,
'%s' % output_graphics,
'%s' % file_pca,
'%s' % file_class,
'%s' % season,
'%s' % start.year,
'%s' % end.year,
'%s' % model_var,
'%s' % kappa
]
logger.info('Rcall builded')
except Exception as e:
msg = 'failed to build the R command %s' % e
logger.debug(msg)
raise Exception(msg)
try:
output, error = subprocess.Popen(
args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
logger.info('R outlog info:\n %s ' % output)
logger.debug('R outlog errors:\n %s ' % error)
if len(output) > 0:
self.status.set('**** weatherregime in R suceeded', 90)
else:
logger.error('NO! output returned from R call')
except Exception as e:
msg = 'weatherregime in R %s ' % e
logger.error(msg)
raise Exception(msg)
self.status.set('Weather regime clustering done ', 80)
############################################
# set the outputs
############################################
self.status.set('Set the process outputs ', 95)
self.Routput_graphic.setValue(output_graphics)
self.output_pca.setValue(file_pca)
self.output_classification.setValue(file_class)
self.output_netcdf.setValue(model_season)
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
LOGGER.info('Start process')
from datetime import datetime as dt
from flyingpigeon import weatherregimes as wr
from tempfile import mkstemp
response.update_status('execution started at : {}'.format(dt.now()), 5)
################################
# reading in the input arguments
################################
LOGGER.info('read in the arguments')
# resources = self.getInputValues(identifier='resources')
season = request.inputs['season'][0].data
LOGGER.info('season %s', season)
# bbox = [-80, 20, 50, 70]
# TODO: Add checking for wrong cordinates and apply default if nesessary
bbox = []
bboxStr = request.inputs['BBox'][0].data
bboxStr = bboxStr.split(',')
bbox.append(float(bboxStr[0]))
bbox.append(float(bboxStr[2]))
bbox.append(float(bboxStr[1]))
bbox.append(float(bboxStr[3]))
LOGGER.debug('BBOX for ocgis: {}'.format(bbox))
LOGGER.debug('BBOX original: {}'.format(bboxStr))
model_var = request.inputs['reanalyses'][0].data
model, variable = model_var.split('_')
period = request.inputs['period'][0].data
LOGGER.info('period: {}'.format(period))
anualcycle = request.inputs['anualcycle'][0].data
kappa = request.inputs['kappa'][0].data
LOGGER.info('kappa: {}', kappa)
method = request.inputs['method'][0].data
LOGGER.info('Calc annual cycle with {}'.format(method))
sseas = request.inputs['sseas'][0].data
LOGGER.info('Annual cycle calc with {}'.format(sseas))
start = dt.strptime(period.split('-')[0], '%Y%m%d')
end = dt.strptime(period.split('-')[1], '%Y%m%d')
LOGGER.debug('start: {0}, end: {1}'.format(start, end))
###########################
# set the environment
###########################
response.update_status('fetching data from archive', 10)
try:
if model == 'NCEP':
getlevel = False
if 'z' in variable:
level = variable.strip('z')
conform_units_to = None
else:
level = None
conform_units_to = 'hPa'
elif '20CRV2' in model:
getlevel = False
if 'z' in variable:
level = variable.strip('z')
conform_units_to = None
else:
level = None
conform_units_to = 'hPa'
else:
LOGGER.exception('Reanalyses dataset not known')
LOGGER.info('environment set for model: {}'.format(model))
except Exception as ex:
msg = 'failed to set environment: {}'.format(ex)
LOGGER.exception(msg)
raise Exception(msg)
##########################################
# fetch Data from original data archive
##########################################
from flyingpigeon.datafetch import reanalyses as rl
from flyingpigeon.utils import get_variable
# from os.path import basename, splitext
from os import system
from netCDF4 import Dataset
from numpy import squeeze
try:
model_nc = rl(start=start.year,
end=end.year,
dataset=model,
variable=variable,
getlevel=getlevel)
LOGGER.info('reanalyses data fetched')
except Exception as ex:
msg = 'failed to get reanalyses data: {}'.format(ex)
LOGGER.exception(msg)
raise Exception(msg)
response.update_status('fetching data done', 15)
############################################################
# get the required bbox and time region from resource data
############################################################
response.update_status('subsetting region of interest', 17)
# from flyingpigeon.weatherregimes import get_level
# from flyingpigeon.ocgis_module import call
time_range = [start, end]
############################################################
# Block of level and domain selection for geop huge dataset
############################################################
LevMulti = False
# ===========================================================================================
if 'z' in variable:
tmp_total = []
origvar = get_variable(model_nc)
if LevMulti == False:
for z in model_nc:
b0 = call(resource=z,
variable=origvar,
level_range=[int(level), int(level)],
geom=bbox,
spatial_wrapping='wrap',
prefix='levdom_' + basename(z)[0:-3])
tmp_total.append(b0)
else:
# multiproc - no inprovements yet, need to check in hi perf machine...
# -----------------------
try:
import ctypes
import os
# TODO: This lib is for linux
mkl_rt = ctypes.CDLL('libmkl_rt.so')
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('Current number of threads: {}'.format(nth))
mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64)))
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('NEW number of threads: {}'.format(nth))
# TODO: Does it \/\/\/ work with default shell=False in subprocess... (?)
os.environ['MKL_NUM_THREADS'] = str(nth)
os.environ['OMP_NUM_THREADS'] = str(nth)
except Exception as ex:
msg = 'Failed to set THREADS: {}'.format(ex)
LOGGER.debug(msg)
# -----------------------
from multiprocessing import Pool
pool = Pool()
# from multiprocessing.dummy import Pool as ThreadPool
# pool = ThreadPool()
tup_var = [origvar] * len(model_nc)
tup_lev = [level] * len(model_nc)
tup_bbox = [bbox] * len(model_nc)
tup_args = zip(model_nc, tup_var, tup_lev, tup_bbox)
tmp_total = pool.map(ocgis_call_wrap, tup_args)
pool.close()
pool.join()
LOGGER.debug('Temporal subset files: {}'.format(tmp_total))
tmp_total = sorted(tmp_total,
key=lambda i: splitext(basename(i))[0])
inter_subset_tmp = call(resource=tmp_total,
variable=origvar,
time_range=time_range)
# FIXME: System calls to rm are dangerous! Use os.rmdir instead!
# Clean
for i in tmp_total:
tbr = 'rm -f {}'.format(i)
system(tbr)
# Create new variable
ds = Dataset(inter_subset_tmp, mode='a')
z_var = ds.variables.pop(origvar)
dims = z_var.dimensions
new_var = ds.createVariable('z{}'.format(level),
z_var.dtype,
dimensions=(dims[0], dims[2], dims[3]))
new_var[:, :, :] = squeeze(z_var[:, 0, :, :])
# new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()})
ds.close()
model_subset = call(inter_subset_tmp, variable='z{}'.format(level))
else:
model_subset = call(
resource=model_nc,
variable=variable,
geom=bbox,
spatial_wrapping='wrap',
time_range=time_range,
# conform_units_to=conform_units_to
)
# =============================================================================================
LOGGER.info('Dataset subset done: {}'.format(model_subset))
response.update_status('dataset subsetted', 18)
##############################################
# computing anomalies
##############################################
response.update_status('computing anomalies ', 19)
cycst = anualcycle.split('-')[0]
cycen = anualcycle.split('-')[1]
reference = [
dt.strptime(cycst, '%Y%m%d'),
dt.strptime(cycen, '%Y%m%d')
]
LOGGER.info('reference time: {}'.format(reference))
model_anomal = wr.get_anomalies(model_subset,
reference=reference,
method=method,
sseas=sseas) # , variable=variable)
#####################
# extracting season
#####################
response.update_status('normalizing data', 21)
model_season = wr.get_season(model_anomal, season=season)
response.update_status('anomalies computed and normalized', 24)
#######################
# call the R scripts
#######################
response.update_status('Start weather regime clustering ', 25)
import subprocess
from flyingpigeon import config
from os.path import curdir, join
try:
rworkspace = curdir
Rsrc = config.Rsrc_dir()
Rfile = 'weatherregimes_model.R'
infile = model_season # model_subset #model_ponderate
modelname = model
yr1 = start.year
yr2 = end.year
ip, output_graphics = mkstemp(dir=curdir, suffix='.pdf')
ip, file_pca = mkstemp(dir=curdir, suffix='.txt')
ip, file_class = mkstemp(dir=curdir, suffix='.Rdat')
# TODO: Rewrite this using os.path.join or pathlib libraries
args = [
'Rscript',
join(Rsrc, Rfile),
'%s/' % curdir,
'%s/' % Rsrc,
'%s' % infile,
'%s' % variable,
'%s' % output_graphics,
'%s' % file_pca,
'%s' % file_class,
'%s' % season,
'%s' % start.year,
'%s' % end.year,
'%s' % model_var,
'%s' % kappa
]
LOGGER.info('Rcall builded')
LOGGER.debug('ARGS: %s' % (args))
except Exception as ex:
msg = 'failed to build the R command: {}'.format(ex)
LOGGER.exception(msg)
raise Exception(msg)
try:
output, error = subprocess.Popen(
args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
LOGGER.info('R outlog info:\n {}'.format(output))
LOGGER.exception('R outlog errors:\n {}'.format(error))
if len(output) > 0:
response.update_status('**** weatherregime in R suceeded', 90)
else:
LOGGER.exception('No output returned from R call')
except Exception as ex:
msg = 'failed to run the R weatherregime: {}'.format(ex)
LOGGER.exception(msg)
raise Exception(msg)
response.update_status('Weather regime clustering done ', 93)
############################################
# set the outputs
############################################
response.update_status('Set the process outputs ', 95)
response.outputs['Routput_graphic'].file = output_graphics
response.outputs['output_pca'].file = file_pca
response.outputs['output_classification'].file = file_class
response.outputs['output_netcdf'].file = model_subset
response.update_status('done', 100)
return response
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
LOGGER.info('Start process')
response.update_status('execution started at : {}'.format(dt.now()), 5)
process_start_time = time.time() # measure process execution time ...
start_time = time.time() # measure init ...
################################
# reading in the input arguments
################################
try:
response.update_status('read input parameter : %s ' % dt.now(), 7)
refSt = request.inputs['refSt'][0].data
refEn = request.inputs['refEn'][0].data
dateSt = request.inputs['dateSt'][0].data
dateEn = request.inputs['dateEn'][0].data
seasonwin = request.inputs['seasonwin'][0].data
nanalog = request.inputs['nanalog'][0].data
timres = request.inputs['timeres'][0].data
# bbox = [-80, 20, 50, 70]
# TODO: Add checking for wrong cordinates and apply default if nesessary
bbox = []
bboxStr = request.inputs['BBox'][0].data
bboxStr = bboxStr.split(',')
bbox.append(float(bboxStr[0]))
bbox.append(float(bboxStr[2]))
bbox.append(float(bboxStr[1]))
bbox.append(float(bboxStr[3]))
LOGGER.debug('BBOX for ocgis: %s ' % (bbox))
LOGGER.debug('BBOX original: %s ' % (bboxStr))
normalize = request.inputs['normalize'][0].data
detrend = request.inputs['detrend'][0].data
distance = request.inputs['dist'][0].data
outformat = request.inputs['outformat'][0].data
timewin = request.inputs['timewin'][0].data
model_var = request.inputs['reanalyses'][0].data
model, var = model_var.split('_')
# experiment = self.getInputValues(identifier='experiment')[0]
# dataset, var = experiment.split('_')
# LOGGER.info('environment set')
LOGGER.info('input parameters set')
response.update_status('Read in and convert the arguments', 8)
except Exception as e:
msg = 'failed to read input prameter %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
######################################
# convert types and set environment
######################################
try:
response.update_status('Preparing enviroment converting arguments',
9)
LOGGER.debug('date: %s %s %s %s ' %
(type(refSt), refEn, dateSt, dateSt))
start = min(refSt, dateSt)
end = max(refEn, dateEn)
#
# refSt = dt.strftime(refSt, '%Y-%m-%d')
# refEn = dt.strftime(refEn, '%Y-%m-%d')
# dateSt = dt.strftime(dateSt, '%Y-%m-%d')
# dateEn = dt.strftime(dateEn, '%Y-%m-%d')
if normalize == 'None':
seacyc = False
else:
seacyc = True
if outformat == 'ascii':
outformat = '.txt'
elif outformat == 'netCDF':
outformat = '.nc'
else:
LOGGER.exception('output format not valid')
except Exception as e:
msg = 'failed to set environment %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
###########################
# set the environment
###########################
response.update_status('fetching data from archive', 10)
try:
if model == 'NCEP':
getlevel = False
if 'z' in var:
level = var.strip('z')
conform_units_to = None
else:
level = None
conform_units_to = 'hPa'
elif '20CRV2' in model:
getlevel = False
if 'z' in var:
level = var.strip('z')
conform_units_to = None
else:
level = None
conform_units_to = 'hPa'
else:
LOGGER.exception('Reanalyses dataset not known')
LOGGER.info('environment set for model: %s' % model)
except Exception:
msg = 'failed to set environment'
LOGGER.exception(msg)
raise Exception(msg)
##########################################
# fetch Data from original data archive
##########################################
# NOTE: If ref is say 1950 - 1990, and sim is just 1 week in 2017 - ALL the data will be downloaded, 1950 - 2017
try:
model_nc = rl(start=start.year,
end=end.year,
dataset=model,
variable=var,
timres=timres,
getlevel=getlevel)
LOGGER.info('reanalyses data fetched')
except Exception:
msg = 'failed to get reanalyses data'
LOGGER.exception(msg)
raise Exception(msg)
response.update_status('subsetting region of interest', 17)
# from flyingpigeon.weatherregimes import get_level
LOGGER.debug("start and end time: %s - %s" % (start, end))
time_range = [start, end]
# For 20CRV2 geopotential height, daily dataset for 100 years is about 50 Gb
# So it makes sense, to operate it step-by-step
# TODO: need to create dictionary for such datasets (for models as well)
# TODO: benchmark the method bellow for NCEP z500 for 60 years
# if ('20CRV2' in model) and ('z' in var):
if ('z' in var):
tmp_total = []
origvar = get_variable(model_nc)
for z in model_nc:
tmp_n = 'tmp_%s' % (uuid.uuid1())
b0 = call(resource=z,
variable=origvar,
level_range=[int(level), int(level)],
geom=bbox,
spatial_wrapping='wrap',
prefix='levdom_' + os.path.basename(z)[0:-3])
tmp_total.append(b0)
tmp_total = sorted(
tmp_total,
key=lambda i: os.path.splitext(os.path.basename(i))[0])
inter_subset_tmp = call(resource=tmp_total,
variable=origvar,
time_range=time_range)
# Clean
for i in tmp_total:
tbr = 'rm -f %s' % (i)
os.system(tbr)
# Create new variable
ds = Dataset(inter_subset_tmp, mode='a')
z_var = ds.variables.pop(origvar)
dims = z_var.dimensions
new_var = ds.createVariable('z%s' % level,
z_var.dtype,
dimensions=(dims[0], dims[2], dims[3]))
new_var[:, :, :] = squeeze(z_var[:, 0, :, :])
# new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()})
ds.close()
model_subset_tmp = call(inter_subset_tmp, variable='z%s' % level)
else:
model_subset_tmp = call(
resource=model_nc,
variable=var,
geom=bbox,
spatial_wrapping='wrap',
time_range=time_range,
# conform_units_to=conform_units_to
)
# If dataset is 20CRV2 the 6 hourly file should be converted to daily.
# Option to use previously 6h data from cache (if any) and not download daily files.
if '20CRV2' in model:
if timres == '6h':
from cdo import Cdo
cdo = Cdo()
model_subset = '%s.nc' % uuid.uuid1()
tmp_f = '%s.nc' % uuid.uuid1()
cdo_op = getattr(cdo, 'daymean')
cdo_op(input=model_subset_tmp, output=tmp_f)
sti = '00:00:00'
cdo_op = getattr(cdo, 'settime')
cdo_op(sti, input=tmp_f, output=model_subset)
LOGGER.debug('File Converted from: %s to daily' % (timres))
else:
model_subset = model_subset_tmp
else:
model_subset = model_subset_tmp
LOGGER.info('Dataset subset done: %s ', model_subset)
response.update_status('dataset subsetted', 19)
# BLOCK OF DETRENDING of model_subset !
# Original model subset kept to further visualisaion if needed
# Now is issue with SLP:
# TODO 1 Keep trend as separate file
# TODO 2 Think how to add options to plot abomalies AND original data...
# May be do archive and simulation = call.. over NOT detrended data and keep it as well
# TODO 3 Check with faster smoother add removing trend of each grid
if detrend == 'None':
orig_model_subset = model_subset
else:
orig_model_subset = remove_mean_trend(model_subset, varname=var)
# ======================================
############################################################
# get the required bbox and time region from resource data
############################################################
#
#
# try:
# if dataset == 'NCEP':
# if 'z' in var:
# variable = 'hgt'
# level = var.strip('z')
# # conform_units_to=None
# else:
# variable = 'slp'
# level = None
# # conform_units_to='hPa'
# elif '20CRV2' in var:
# if 'z' in level:
# variable = 'hgt'
# level = var.strip('z')
# # conform_units_to=None
# else:
# variable = 'prmsl'
# level = None
# # conform_units_to='hPa'
# else:
# LOGGER.exception('Reanalyses dataset not known')
# LOGGER.info('environment set')
# except Exception as e:
# msg = 'failed to set environment %s ' % e
# LOGGER.exception(msg)
# # raise Exception(msg)
#
# LOGGER.debug("init took %s seconds.", time.time() - start_time)
# response.update_status('Read in and convert the arguments done', 8)
#
# #################
# # get input data
# #################
# start_time = time.time() # measure get_input_data ...
# response.update_status('fetching input data', 7)
# try:
# input = reanalyses(start=start.year, end=end.year,
# variable=var, dataset=dataset)
# LOGGER.info('input files %s' % input)
# nc_subset = call(resource=input, variable=var,
# geom=bbox, spatial_wrapping='wrap')
# except Exception as e:
# msg = 'failed to fetch or subset input files %s' % e
# LOGGER.exception(msg)
# # raise Exception(msg)
LOGGER.debug("get_input_subset_dataset took %s seconds.",
time.time() - start_time)
response.update_status('**** Input data fetched', 20)
########################
# input data preperation
########################
response.update_status('Start preparing input data', 22)
start_time = time.time() # measure data preperation ...
try:
# Construct descriptive filenames for the three files
# listed in config file
# TODO check strftime for years <1900 (!)
refDatesString = dt.strftime(
refSt, '%Y-%m-%d') + "_" + dt.strftime(refEn, '%Y-%m-%d')
simDatesString = dt.strftime(
dateSt, '%Y-%m-%d') + "_" + dt.strftime(dateEn, '%Y-%m-%d')
archiveNameString = "base_" + var + "_" + refDatesString + '_%.1f_%.1f_%.1f_%.1f' \
% (bbox[0], bbox[2], bbox[1], bbox[3])
simNameString = "sim_" + var + "_" + simDatesString + '_%.1f_%.1f_%.1f_%.1f' \
% (bbox[0], bbox[2], bbox[1], bbox[3])
archive = call(resource=model_subset,
time_range=[refSt, refEn],
prefix=archiveNameString)
simulation = call(resource=model_subset,
time_range=[dateSt, dateEn],
prefix=simNameString)
LOGGER.info('archive and simulation files generated: %s, %s' %
(archive, simulation))
except Exception as e:
msg = 'failed to prepare archive and simulation files %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
try:
if seacyc is True:
LOGGER.info('normalization function with method: %s ' %
normalize)
seasoncyc_base, seasoncyc_sim = analogs.seacyc(
archive, simulation, method=normalize)
else:
seasoncyc_base = seasoncyc_sim = None
except Exception as e:
msg = 'failed to generate normalization files %s ' % e
LOGGER.exception(msg)
raise Exception(msg)
output_file = 'output.txt'
files = [
os.path.abspath(archive),
os.path.abspath(simulation), output_file
]
LOGGER.debug("Data preperation took %s seconds.",
time.time() - start_time)
############################
# generate the config file
############################
config_file = analogs.get_configfile(
files=files,
seasoncyc_base=seasoncyc_base,
seasoncyc_sim=seasoncyc_sim,
base_id=model,
sim_id=model,
timewin=timewin,
varname=var,
seacyc=seacyc,
cycsmooth=91,
nanalog=nanalog,
seasonwin=seasonwin,
distfun=distance,
outformat=outformat,
calccor=True,
silent=False,
period=[
dt.strftime(refSt, '%Y-%m-%d'),
dt.strftime(refEn, '%Y-%m-%d')
],
bbox="{0[0]},{0[2]},{0[1]},{0[3]}".format(bbox))
response.update_status('generated config file', 25)
#######################
# CASTf90 call
#######################
start_time = time.time() # measure call castf90
# -----------------------
try:
import ctypes
# TODO: This lib is for linux
mkl_rt = ctypes.CDLL('libmkl_rt.so')
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('Current number of threads: %s' % (nth))
mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64)))
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('NEW number of threads: %s' % (nth))
# TODO: Does it \/\/\/ work with default shell=False in subprocess... (?)
os.environ['MKL_NUM_THREADS'] = str(nth)
os.environ['OMP_NUM_THREADS'] = str(nth)
except Exception as e:
msg = 'Failed to set THREADS %s ' % e
LOGGER.debug(msg)
# -----------------------
response.update_status('Start CASTf90 call', 30)
try:
# response.update_status('execution of CASTf90', 50)
cmd = ['analogue.out', config_file]
LOGGER.debug("castf90 command: %s", cmd)
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
LOGGER.info('analogue output:\n %s', output)
response.update_status('**** CASTf90 suceeded', 70)
except CalledProcessError as e:
msg = 'CASTf90 failed:\n{0}'.format(e.output)
LOGGER.exception(msg)
raise Exception(msg)
LOGGER.debug("castf90 took %s seconds.", time.time() - start_time)
# TODO: Add try - except for pdfs
analogs_pdf = analogs.plot_analogs(configfile=config_file)
response.update_status('preparing output', 75)
# response.outputs['config'].storage = FileStorage()
response.outputs['analog_pdf'].file = analogs_pdf
response.outputs['config'].file = config_file
response.outputs['analogs'].file = output_file
response.outputs['output_netcdf'].file = simulation
response.outputs['target_netcdf'].file = archive
if seacyc is True:
response.outputs['base_netcdf'].file = seasoncyc_base
response.outputs['sim_netcdf'].file = seasoncyc_sim
else:
# TODO: Still unclear how to overpass unknown number of outputs
dummy_base = 'dummy_base.nc'
dummy_sim = 'dummy_sim.nc'
with open(dummy_base, 'a'):
os.utime(dummy_base, None)
with open(dummy_sim, 'a'):
os.utime(dummy_sim, None)
response.outputs['base_netcdf'].file = dummy_base
response.outputs['sim_netcdf'].file = dummy_sim
########################
# generate analog viewer
########################
formated_analogs_file = analogs.reformat_analogs(output_file)
# response.outputs['formated_analogs'].storage = FileStorage()
response.outputs['formated_analogs'].file = formated_analogs_file
LOGGER.info('analogs reformated')
response.update_status('reformatted analog file', 80)
viewer_html = analogs.render_viewer(
# configfile=response.outputs['config'].get_url(),
configfile=config_file,
# datafile=response.outputs['formated_analogs'].get_url())
datafile=formated_analogs_file)
response.outputs['output'].file = viewer_html
response.update_status('Successfully generated analogs viewer', 90)
LOGGER.info('rendered pages: %s ', viewer_html)
response.update_status('execution ended', 100)
LOGGER.debug("total execution took %s seconds.",
time.time() - process_start_time)
return response
def execute(self):
logger.info('Start process')
init_process_logger('log.txt')
self.output_log.setValue('log.txt')
from datetime import datetime as dt
from flyingpigeon import weatherregimes as wr
from tempfile import mkstemp
################################
# reading in the input arguments
################################
try:
resource = self.getInputValues(identifier='resource')
url_Rdat = self.getInputValues(identifier='Rdat')[0]
url_dat = self.getInputValues(identifier='dat')[0]
url_ref_file = self.getInputValues(
identifier='netCDF') # can be None
season = self.getInputValues(identifier='season')[0]
period = self.getInputValues(identifier='period')[0]
anualcycle = self.getInputValues(identifier='anualcycle')[0]
except Exception as e:
logger.debug('failed to read in the arguments %s ' % e)
try:
start = dt.strptime(period.split('-')[0], '%Y%m%d')
end = dt.strptime(period.split('-')[1], '%Y%m%d')
# kappa = int(self.getInputValues(identifier='kappa')[0])
logger.info('period %s' % str(period))
logger.info('season %s' % str(season))
logger.info('read in the arguments')
logger.info('url_ref_file: %s' % url_ref_file)
logger.info('url_Rdat: %s' % url_Rdat)
logger.info('url_dat: %s' % url_dat)
except Exception as e:
logger.debug('failed to convert arguments %s ' % e)
############################
# fetching trainging data
############################
from flyingpigeon.utils import download, get_time
from os.path import abspath
try:
dat = abspath(download(url_dat))
Rdat = abspath(download(url_Rdat))
logger.info('training data fetched')
except Exception as e:
logger.error('failed to fetch training data %s' % e)
##########################################################
# get the required bbox and time region from resource data
##########################################################
# from flyingpigeon.weatherregimes import get_level
from flyingpigeon.ocgis_module import call
from flyingpigeon.utils import get_variable
time_range = [start, end]
variable = get_variable(resource)
if len(url_ref_file) > 0:
ref_file = download(url_ref_file[0])
model_subset = call(
resource=resource,
variable=variable,
time_range=
time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap',
regrid_destination=ref_file,
regrid_options='bil')
logger.info('Dataset subset with regridding done: %s ' %
model_subset)
else:
model_subset = call(
resource=resource,
variable=variable,
time_range=
time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap',
)
logger.info('Dataset time period extracted: %s ' % model_subset)
#######################
# computing anomalies
#######################
cycst = anualcycle.split('-')[0]
cycen = anualcycle.split('-')[0]
reference = [
dt.strptime(cycst, '%Y%m%d'),
dt.strptime(cycen, '%Y%m%d')
]
model_anomal = wr.get_anomalies(model_subset, reference=reference)
#####################
# extracting season
#####################
model_season = wr.get_season(model_anomal, season=season)
#######################
# call the R scripts
#######################
import shlex
import subprocess
from flyingpigeon import config
from os.path import curdir, exists, join
try:
rworkspace = curdir
Rsrc = config.Rsrc_dir()
Rfile = 'weatherregimes_projection.R'
yr1 = start.year
yr2 = end.year
time = get_time(model_season, format='%Y%m%d')
# ip, output_graphics = mkstemp(dir=curdir ,suffix='.pdf')
ip, file_pca = mkstemp(dir=curdir, suffix='.txt')
ip, file_class = mkstemp(dir=curdir, suffix='.Rdat')
ip, output_frec = mkstemp(dir=curdir, suffix='.txt')
args = [
'Rscript',
join(Rsrc, Rfile),
'%s/' % curdir,
'%s/' % Rsrc,
'%s' % model_season,
'%s' % variable,
'%s' % str(time).strip("[]").replace("'", "").replace(" ", ""),
# '%s' % output_graphics,
'%s' % dat,
'%s' % Rdat,
'%s' % file_pca,
'%s' % file_class,
'%s' % output_frec,
'%s' % season,
'%s' % start.year,
'%s' % end.year,
'%s' % 'MODEL'
]
logger.info('Rcall builded')
except Exception as e:
msg = 'failed to build the R command %s' % e
logger.error(msg)
raise Exception(msg)
try:
output, error = subprocess.Popen(
args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
# , shell=True
logger.info('R outlog info:\n %s ' % output)
logger.debug('R outlog errors:\n %s ' % error)
if len(output) > 0:
self.status.set('**** weatherregime in R suceeded', 90)
else:
logger.error('NO! output returned from R call')
except Exception as e:
msg = 'weatherregime in R %s ' % e
logger.error(msg)
raise Exception(msg)
#################
# set the outputs
#################
# self.Routput_graphic.setValue( output_graphics )
self.output_pca.setValue(file_pca)
self.output_classification.setValue(file_class)
self.output_netcdf.setValue(model_season)
self.output_frequency.setValue(output_frec)
def execute(self):
from os.path import basename
from flyingpigeon import sdm
from flyingpigeon.utils import archive, archiveextract # , get_domain
from flyingpigeon.visualisation import map_PAmask
init_process_logger('log.txt')
self.output_log.setValue('log.txt')
self.status.set('Start process', 0)
try:
self.status.set('reading the arguments', 5)
resources = archiveextract(
self.getInputValues(identifier='input_indices'))
csv_file = self.getInputValues(identifier='gbif')[0]
period = self.getInputValues(identifier='period')
period = period[0]
archive_format = self.archive_format.getValue()
except:
logger.error('failed to read in the arguments')
try:
self.status.set('read in latlon coordinates', 10)
latlon = sdm.latlon_gbifcsv(csv_file)
except:
logger.exception('failed to extract the latlon points')
try:
self.status.set('plot map', 20)
from flyingpigeon.visualisation import map_gbifoccurrences
# latlon = sdm.latlon_gbifdic(gbifdic)
occurence_map = map_gbifoccurrences(latlon)
except:
logger.exception('failed to plot occurence map')
# try:
# self.status.set('get domain', 30)
# domains = set()
# for indice in resources:
# # get_domain works only if metadata are set in a correct way
# domains = domains.union([basename(indice).split('_')[1]])
# if len(domains) == 1:
# domain = list(domains)[0]
# logger.info('Domain %s found in indices files' % domain)
# else:
# logger.warn('NOT a single domain in indices files %s' % domains)
# except:
# logger.exception('failed to get domains')
try:
# sort indices
indices_dic = sdm.sort_indices(resources)
logger.info('indice files sorted for %s Datasets' %
len(indices_dic.keys()))
except:
msg = 'failed to sort indices'
logger.exception(msg)
raise Exception(msg)
ncs_references = []
species_files = []
stat_infos = []
PAmask_pngs = []
self.status.set('Start processing for %s Datasets' %
len(indices_dic.keys()))
for count, key in enumerate(indices_dic.keys()):
try:
staus_nr = 40 + count * 10
self.status.set('Start processing of %s' % key, staus_nr)
ncs = indices_dic[key]
logger.info('with %s files' % len(ncs))
try:
self.status.set('generating the PA mask', 20)
PAmask = sdm.get_PAmask(coordinates=latlon, nc=ncs[0])
logger.info('PA mask sucessfully generated')
except:
logger.exception('failed to generate the PA mask')
try:
self.status.set('Ploting PA mask', 25)
PAmask_pngs.extend([map_PAmask(PAmask)])
except:
logger.exception('failed to plot the PA mask')
try:
ncs_reference = sdm.get_reference(ncs_indices=ncs,
period=period)
ncs_references.extend(ncs_reference)
logger.info('reference indice calculated %s ' %
ncs_references)
self.status.set('reference indice calculated',
staus_nr + 2)
except:
msg = 'failed to calculate the reference'
logger.exception(msg)
# raise Exception(msg)
try:
gam_model, predict_gam, gam_info = sdm.get_gam(
ncs_reference, PAmask, modelname=key)
stat_infos.append(gam_info)
self.status.set('GAM sucessfully trained', staus_nr + 5)
except:
msg = 'failed to train GAM for %s' % (key)
logger.exception(msg)
try:
prediction = sdm.get_prediction(gam_model, ncs)
self.status.set('prediction done', staus_nr + 7)
except:
msg = 'failed to predict tree occurence'
logger.exception(msg)
# raise Exception(msg)
# try:
# self.status.set('land sea mask for predicted data', staus_nr + 8)
# from numpy import invert, isnan, nan, broadcast_arrays # , array, zeros, linspace, meshgrid
# mask = invert(isnan(PAmask))
# mask = broadcast_arrays(prediction, mask)[1]
# prediction[mask is False] = nan
# except:
# logger.exception('failed to mask predicted data')
try:
species_files.append(sdm.write_to_file(ncs[0], prediction))
logger.info('Favourabillity written to file')
except:
msg = 'failed to write species file'
logger.exception(msg)
# raise Exception(msg)
except:
msg = 'failed to process SDM chain for %s ' % key
logger.exception(msg)
# raise Exception(msg)
try:
archive_references = None
archive_references = archive(ncs_references, format=archive_format)
logger.info('indices 2D added to archive')
except:
msg = 'failed adding 2D indices to archive'
logger.exception(msg)
raise Exception(msg)
archive_predicion = None
try:
archive_predicion = archive(species_files, format=archive_format)
logger.info('species_files added to archive')
except:
msg = 'failed adding species_files indices to archive'
logger.exception(msg)
raise Exception(msg)
try:
from flyingpigeon.visualisation import pdfmerge, concat_images
stat_infosconcat = pdfmerge(stat_infos)
logger.debug('pngs %s' % PAmask_pngs)
PAmask_png = concat_images(PAmask_pngs, orientation='h')
logger.info('stat infos pdfs and mask pngs merged')
except:
logger.exception('failed to concat images')
_, stat_infosconcat = tempfile.mkstemp(suffix='.pdf',
prefix='foobar-',
dir='.')
_, PAmask_png = tempfile.mkstemp(suffix='.png',
prefix='foobar-',
dir='.')
self.output_gbif.setValue(occurence_map)
self.output_PA.setValue(PAmask_png)
self.output_reference.setValue(archive_references)
self.output_prediction.setValue(archive_predicion)
self.output_info.setValue(stat_infosconcat)
self.status.set('done', 100)
