def execute(self):
ncfiles = self.getInputValues(identifier='resource')
var = self.variableIn.getValue()
self.status.set('plotting variable %s' % var, 10)
plotout_spagetti_file = vs.spaghetti(ncfiles,
variable=var,
title='Fieldmean of %s ' % (var),
dir_out=None)
self.status.set(
'Spagetti plot for %s %s files done' % (len(ncfiles), var), 50)
plotout_uncertainty_file = vs.uncertainty(
ncfiles,
variable=var,
title='Ensemble uncertainty for %s ' % (var),
dir_out=None)
self.status.set(
'Uncertainty plot for %s %s files done' % (len(ncfiles), var), 90)
self.plotout_spagetti.setValue(plotout_spagetti_file)
self.plotout_uncertainty.setValue(plotout_uncertainty_file)
self.status.set('visualisation done', 100)
def execute(self):
from flyingpigeon.utils import archive, archiveextract
from tempfile import mkstemp
init_process_logger('log.txt')
self.output_log.setValue('log.txt')
ncs = archiveextract(self.getInputValues(identifier='resource'))
# mosaic = self.mosaic.getValue()
regions = self.region.getValue()
self.status.set('Arguments set for subset process', 0)
logger.debug('starting: regions=%s, num_files=%s' % (len(regions), len(ncs)))
try:
from flyingpigeon.visualisation import plot_polygons
png_country = plot_polygons(regions)
except:
logger.exception('failed to plot the polygon to world map')
o1, factsheet_plot = mkstemp(dir='.', suffix='.png')
# clip the demanded polygons
from flyingpigeon.subset import clipping
subsets = clipping(resource=ncs, variable=None,
dimension_map=None,
calc=None,
output_format='nc',
calc_grouping=None,
time_range=None,
time_region=None,
historical_concatination=True,
prefix=None,
spatial_wrapping='wrap',
polygons=regions,
mosaic=True
)
try:
from flyingpigeon.visualisation import uncertainty
png_uncertainty = uncertainty(subsets)
except:
logger.exception('failed to generate the uncertainty plot')
_, png_uncertainty = mkstemp(dir='.', suffix='.png')
try:
from flyingpigeon.visualisation import spaghetti
png_spaghetti = spaghetti(subsets)
except:
logger.exception('failed to generate the spaghetti plot')
_, png_spaghetti = mkstemp(dir='.', suffix='.png')
from flyingpigeon.visualisation import factsheetbrewer
factsheet = factsheetbrewer(png_country=png_country,
png_uncertainty=png_uncertainty,
png_spaghetti=png_spaghetti)
self.output_factsheet.setValue(factsheet)
self.status.set('done', 100)
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
ncfiles = archiveextract(
resource=rename_complexinputs(request.inputs['resource']))
if 'variable' in request.inputs:
var = request.inputs['variable'][0].data
else:
var = get_variable(ncfiles[0])
# var = ncfiles[0].split("_")[0]
response.update_status('plotting variable {}'.format(var), 10)
try:
plotout_spagetti_file = vs.spaghetti(
ncfiles,
variable=var,
title='Field mean of {}'.format(var),
)
LOGGER.info("spagetti plot done")
response.update_status(
'Spagetti plot for %s %s files done' % (len(ncfiles), var), 50)
response.outputs['plotout_spagetti'].file = plotout_spagetti_file
except Exception as e:
raise Exception("spagetti plot failed : {}".format(e))
try:
plotout_uncertainty_file = vs.uncertainty(
ncfiles,
variable=var,
title='Ensemble uncertainty for {}'.format(var),
)
response.update_status(
'Uncertainty plot for {} {} files done'.format(
len(ncfiles), var), 90)
response.outputs[
'plotout_uncertainty'].file = plotout_uncertainty_file
LOGGER.info("uncertainty plot done")
except Exception as err:
raise Exception("uncertainty plot failed {}".format(err.message))
response.update_status('visualisation done', 100)
return response
def execute(self):
ncfiles = self.getInputValues(identifier='resource')
var = self.variableIn.getValue()
self.status.set('plotting variable %s' % var, 10)
plotout_spagetti_file = vs.spaghetti(ncfiles , variable=var, title=var, dir_out=None)
self.status.set('Spagetti plot for %s %s files done' % (len(ncfiles), var), 50)
plotout_uncertainty_file = vs.uncertainty(ncfiles , variable=var, title=var, dir_out=None)
self.status.set('Uncertainty plot for %s %s files done' % (len(ncfiles), var), 90)
self.plotout_spagetti.setValue( plotout_spagetti_file )
self.plotout_uncertainty.setValue( plotout_uncertainty_file )
self.status.set('visualisation done', 100)
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
ncfiles = archiveextract(
resource=rename_complexinputs(request.inputs['resource']))
var = request.inputs['variableIn']
if var is None:
from flyingpigeon.utils import get_variable
var = get_variable(ncfiles[0])
response.update_status('plotting variable %s' % var, 10)
try:
plotout_spagetti_file = vs.spaghetti(ncfiles,
variable=var,
title='Fieldmean of %s ' %
(var),
dir_out=None)
LOGGER.info("spagetti plot done")
response.update_status(
'Spagetti plot for %s %s files done' % (len(ncfiles), var), 50)
except:
LOGGER.exception("spagetti plot failed")
try:
plotout_uncertainty_file = vs.uncertainty(
ncfiles,
variable=var,
title='Ensemble uncertainty for %s ' % (var),
dir_out=None)
response.update_status(
'Uncertainty plot for %s %s files done' % (len(ncfiles), var),
90)
LOGGER.info("uncertainty plot done")
except:
LOGGER.exception("uncertainty plot failed")
response.outputs['plotout_spagetti'].file = plotout_spagetti_file
response.outputs['plotout_uncertainty'].file = plotout_uncertainty_file
response.update_status('visualisation done', 100)
return response
def execute(self):
init_process_logger('log.txt')
self.output_log.setValue('log.txt')
from flyingpigeon.utils import archiveextract
ncfiles = archiveextract(self.getInputValues(identifier='resource'))
var = self.variableIn.getValue()
if var is None:
from flyingpigeon.utils import get_variable
var = get_variable(ncfiles[0])
self.status.set('plotting variable %s' % var, 10)
try:
plotout_spagetti_file = vs.spaghetti(
ncfiles,
variable=var,
title='Fieldmean of %s ' % (var),
dir_out=None
)
logger.info("spagetti plot done")
self.status.set('Spagetti plot for %s %s files done' % (len(ncfiles), var), 50)
except:
logger.exception("spagetti plot failed")
try:
plotout_uncertainty_file = vs.uncertainty(
ncfiles,
variable=var,
title='Ensemble uncertainty for %s ' % (var),
dir_out=None
)
self.status.set('Uncertainty plot for %s %s files done' % (len(ncfiles), var), 90)
logger.info("uncertainty plot done")
except:
logger.exception("uncertainty plot failed")
self.plotout_spagetti.setValue(plotout_spagetti_file)
self.plotout_uncertainty.setValue(plotout_uncertainty_file)
self.status.set('visualisation done', 100)
def execute(self):
from os import mkdir, path, listdir
import tarfile
from tempfile import mkstemp #, mkdtemp
from datetime import datetime
from flyingpigeon import segetalflora as sf
logging.debug('starting segetalflora process execution')
self.status.set('starting calcualtion segetalflora', 5)
## prepare environment
# create the tar files
try:
(fp_tarf_tas, tarf_tas) = mkstemp(dir=".", suffix='.tar')
(fp_tarf_segetalflora, tarf_segetalflora) = mkstemp(dir=".", suffix='.tar')
(fp_tarf_fieldmeans, tarf_fieldmeans) = mkstemp(dir=".", suffix='.tar')
(fp_tarf_plots, tarf_plots) = mkstemp(dir=".", suffix='.tar')
tar_tas = tarfile.open(tarf_tas, "w")
tar_segetalflora = tarfile.open(tarf_segetalflora, "w")
tar_fieldmeans = tarfile.open(tarf_fieldmeans, "w")
tar_plots = tarfile.open(tarf_plots, "w")
logging.debug('tar files initialized')
# create output folders
dir_tas = path.abspath(path.curdir+'/dir_tas/')
dir_segetalflora = path.abspath(path.curdir+'/dir_segetalflora/')
dir_fieldmean = path.abspath(path.curdir+'/dir_fieldmean/')
dir_plots = path.abspath(path.curdir+'/dir_plots/')
mkdir(dir_tas)
mkdir(dir_segetalflora)
mkdir(dir_fieldmean)
mkdir(dir_plots)
logging.debug('out directories created')
except Exception as e:
logging.exception('tar file or mkdir failed!')
raise
countries = ['AUT','BEL','BGR','CYP','CZE','DEU','DNK','ESP',
'EST','FIN','FRA','GBR','GRC','HUN','HRV','IRL',
'ITA','LVA','LTU','LUX','MLT','NLD','POL','PRT',
'ROU','SVK','SVN','SWE','NOR','CHE','ISL','MKD',
'MNE','SRB','MDA','UKR','BIH','ALB','BLR','KOS']
# read argments to variables
#try:
ncs = self.getInputValues(identifier='netcdf_file')
climate_type = self.climate_type.getValue()
culture_type = self.culture_type.getValue()
countries = self.getInputValues(identifier='region')
if type(climate_type) != list:
climate_type = list([climate_type])
if type(culture_type) != list:
culture_type = list([culture_type])
if type(countries) != list:
countries = list([countries])
logging.debug('urls for %s ncs found' % (len(ncs)))
logging.debug('culture type: %s ' % (culture_type))
# === main call for segetalflora processing
stepps = len(culture_type) * len(climate_type)
for a, cult in enumerate(culture_type):
for b, clim in enumerate(climate_type):
start = (a + 1) * (b + 1)
per = (start / stepps) * 95
self.status.set('%s/%s processing for %s climate type: %s' %(start, stepps, culture_type, climate_type), per)
try:
sf_files = sf.get_segetalflora(ncs, culture_type=cult,
climate_type=clim,
#countries=countries,
dir_output=dir_tas,
dir_segetalflora=dir_segetalflora)
self.status.set("processing of %s segetalflora files done " % (len(sf_files)) , 95)
except Exception as e:
logging.exception('segetalflora calculation failed %s %s' % ( climate_type, culture_type))
raise
# === fieldmeans
from flyingpigeon import timeseries as ts
self.status.set('processing fieldmeans' , 97)
try:
ncs = [path.join(dir_segetalflora,nc) for nc in listdir(dir_segetalflora)]
ncs_fld = ts.fldmean(ncs, dir_output=dir_fieldmean)
logging.debug('%s fieldmeans processed' % (len(ncs_fld)))
except Exception as e:
logging.exception('fieldmeans failed')
raise
# === visualisation
from flyingpigeon import visualisation as vs
from os import rename
self.status.set('processing visualisation' , 98)
# sort files for plotting
try:
ncs = listdir(dir_segetalflora)
set_var = set()
set_contry = set()
for nc in ncs:
set_var = set_var.union([nc.split('_')[0]])
set_contry = set_contry.union([nc.split('_')[1]])
logging.debug('%s files to plots sorted' % (len(ncs)))
except Exception as e:
logging.exception('files sorting failed')
raise
# plot sorted files
try:
plots = []
for v in set_var:
for c in set_contry:
ncs = [path.join(dir_segetalflora,nc) for nc in listdir(dir_segetalflora) if v in nc and c in nc ]
p = vs.spaghetti(ncs,
variable=v,
title='Segetalflora %s in %s' % (v, c),
dir_out=dir_plots)
newname = path.dirname(p)+'/%s_%s_birdhouse_output.html' %(v,c)
rename(p,newname)
plots.append(newname)
logging.debug('plot created and renamed for %s %s' % (v, c ))
except Exception as e:
logging.exception('ploting failed')
raise
# === tar file archiving
self.status.set('files to tar archives', 99)
tar_tas.add(dir_tas, arcname = dir_tas.replace(path.abspath(path.curdir), ""))
tar_segetalflora.add(dir_segetalflora, arcname = dir_segetalflora.replace(path.abspath(path.curdir), ""))
tar_fieldmeans.add(dir_fieldmean, arcname = dir_fieldmean.replace(path.abspath(path.curdir), ""))
tar_plots.add(dir_plots, arcname = dir_plots.replace(path.abspath(path.curdir), ""))
tar_tas.close()
tar_fieldmeans.close()
tar_segetalflora.close()
tar_plots.close()
logging.debug('tar ncfiles closed')
# === set output parameter
self.out_fieldmeans.setValue( tarf_fieldmeans )
self.out_segetalflora.setValue( tarf_segetalflora )
self.out_tas.setValue( tarf_tas )
self.out_plots.setValue( tarf_plots )
self.status.set("processing done", 100)
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
ncs = archiveextract(
resource=rename_complexinputs(request.inputs['resource']))
var = get_variable(ncs[0])
LOGGER.info('variable to be plotted: {}'.format(var))
# mosaic = self.mosaic.getValue()
if 'region' in request.inputs:
regions = [inp.data for inp in request.inputs['region']]
try:
png_region = vs.plot_polygons(regions)
except Exception as ex:
msg = 'failed to plot the polygon to world map: {}'.format(
str(ex))
LOGGER.exception(msg)
raise Exception(msg)
o1, png_region = mkstemp(dir='.', suffix='.png')
# clip the demanded polygons
subsets = clipping(
resource=ncs,
variable=var,
polygons=regions,
mosaic=True,
spatial_wrapping='wrap',
)
else:
subsets = ncs
png_region = vs.plot_extend(ncs[0])
response.update_status('Arguments set for subset process', 0)
try:
tar_subsets = archive(subsets)
except Exception as ex:
msg = 'failed to archive subsets: {}'.format(ex)
LOGGER.exception(msg)
raise Exception(msg)
_, tar_subsets = mkstemp(dir='.', suffix='.tar')
try:
png_uncertainty = vs.uncertainty(subsets, variable=var)
except Exception as ex:
msg = 'failed to generate the uncertainty plot: {}'.format(ex)
LOGGER.exception(msg)
raise Exception(msg)
_, png_uncertainty = mkstemp(dir='.', suffix='.png')
try:
png_spaghetti = vs.spaghetti(
subsets,
variable=var,
)
except Exception as ex:
msg = 'failed to generate the spaghetti plot: {}'.format(str(ex))
LOGGER.exception(msg)
raise Exception(msg)
_, png_spaghetti = mkstemp(dir='.', suffix='.png')
try:
from flyingpigeon import robustness as ro
signal, low_agreement_mask, high_agreement_mask, text_src = ro.signal_noise_ratio(
resource=subsets,
# start=None, end=None,
# timeslice=None,
# variable=var
)
# if title is None:
title = 'signal robustness of %s ' % (
var) # , end1, end2, start1, start2
png_robustness = vs.map_robustness(
signal,
high_agreement_mask,
low_agreement_mask,
# cmap=cmap,
# title=title
)
LOGGER.info('robustness graphic generated')
except Exception as ex:
msg = 'failed to generate the robustness plot: {}'.format(ex)
LOGGER.exception(msg)
raise Exception(msg)
_, png_robustness = mkstemp(dir='.', suffix='.png')
factsheet = vs.factsheetbrewer(png_region=png_region,
png_uncertainty=png_uncertainty,
png_spaghetti=png_spaghetti,
png_robustness=png_robustness)
response.outputs['output_nc'].file = tar_subsets
response.outputs['output_factsheet'].file = factsheet
response.update_status("done", 100)
return response
def execute(self):
from os import mkdir, path, listdir
import tarfile
from tempfile import mkstemp # , mkdtemp
from datetime import datetime
from flyingpigeon import segetalflora as sf
logging.debug("starting segetalflora process execution")
self.status.set("starting calcualtion segetalflora", 5)
## prepare environment
# create the tar files
try:
(fp_tarf_tas, tarf_tas) = mkstemp(dir=".", suffix=".tar")
(fp_tarf_segetalflora, tarf_segetalflora) = mkstemp(dir=".", suffix=".tar")
(fp_tarf_fieldmeans, tarf_fieldmeans) = mkstemp(dir=".", suffix=".tar")
(fp_tarf_plots, tarf_plots) = mkstemp(dir=".", suffix=".tar")
tar_tas = tarfile.open(tarf_tas, "w")
tar_segetalflora = tarfile.open(tarf_segetalflora, "w")
tar_fieldmeans = tarfile.open(tarf_fieldmeans, "w")
tar_plots = tarfile.open(tarf_plots, "w")
logging.debug("tar files initialized")
# create output folders
dir_tas = path.abspath(path.curdir + "/dir_tas/")
dir_segetalflora = path.abspath(path.curdir + "/dir_segetalflora/")
dir_fieldmean = path.abspath(path.curdir + "/dir_fieldmean/")
dir_plots = path.abspath(path.curdir + "/dir_plots/")
mkdir(dir_tas)
mkdir(dir_segetalflora)
mkdir(dir_fieldmean)
mkdir(dir_plots)
logging.debug("out directories created")
except Exception as e:
logging.exception("tar file or mkdir failed!")
raise
countries = [
"AUT",
"BEL",
"BGR",
"CYP",
"CZE",
"DEU",
"DNK",
"ESP",
"EST",
"FIN",
"FRA",
"GBR",
"GRC",
"HUN",
"HRV",
"IRL",
"ITA",
"LVA",
"LTU",
"LUX",
"MLT",
"NLD",
"POL",
"PRT",
"ROU",
"SVK",
"SVN",
"SWE",
"NOR",
"CHE",
"ISL",
"MKD",
"MNE",
"SRB",
"MDA",
"UKR",
"BIH",
"ALB",
"BLR",
"KOS",
]
# read argments to variables
# try:
ncs = self.getInputValues(identifier="netcdf_file")
climate_type = self.climate_type.getValue()
culture_type = self.culture_type.getValue()
# countries = self.getInputValues(identifier='region')
if type(climate_type) != list:
climate_type = list([climate_type])
if type(culture_type) != list:
culture_type = list([culture_type])
# if type(countries) != list:
# countries = list([countries])
logging.debug("urls for %s ncs found" % (len(ncs)))
logging.debug("culture type: %s " % (culture_type))
# === main call for segetalflora processing
stepps = len(culture_type) * len(climate_type)
for a, cult in enumerate(culture_type):
for b, clim in enumerate(climate_type):
start = (a + 1) * (b + 1)
per = (start / stepps) * 95
self.status.set(
"%s/%s processing for %s climate type: %s" % (start, stepps, culture_type, climate_type), per
)
try:
sf_files = sf.get_segetalflora(
resource=ncs,
dir_output=dir_tas,
culture_type=cult,
climate_type=clim,
region=None,
dimension_map=None,
)
self.status.set("processing of %s segetalflora files done " % (len(sf_files)), 95)
except Exception as e:
logging.exception("segetalflora calculation failed %s %s" % (climate_type, culture_type))
raise
# === fieldmeans
from flyingpigeon import timeseries as ts
self.status.set("processing fieldmeans", 97)
try:
ncs = [path.join(dir_segetalflora, nc) for nc in listdir(dir_segetalflora)]
ncs_fld = ts.fldmean(ncs, dir_output=dir_fieldmean)
logging.debug("%s fieldmeans processed" % (len(ncs_fld)))
except Exception as e:
logging.exception("fieldmeans failed")
raise
# === visualisation
from flyingpigeon import visualisation as vs
from os import rename
self.status.set("processing visualisation", 98)
# sort files for plotting
try:
ncs = listdir(dir_segetalflora)
set_var = set()
set_contry = set()
for nc in ncs:
set_var = set_var.union([nc.split("_")[0]])
set_contry = set_contry.union([nc.split("_")[1]])
logging.debug("%s files to plots sorted" % (len(ncs)))
except Exception as e:
logging.exception("files sorting failed")
raise
# plot sorted files
try:
plots = []
for v in set_var:
for c in set_contry:
ncs = [path.join(dir_segetalflora, nc) for nc in listdir(dir_segetalflora) if v in nc and c in nc]
p = vs.spaghetti(ncs, variable=v, title="Segetalflora %s in %s" % (v, c), dir_out=dir_plots)
newname = path.dirname(p) + "/%s_%s_birdhouse_output.html" % (v, c)
rename(p, newname)
plots.append(newname)
logging.debug("plot created and renamed for %s %s" % (v, c))
except Exception as e:
logging.exception("ploting failed")
raise
# === tar file archiving
self.status.set("files to tar archives", 99)
tar_tas.add(dir_tas, arcname=dir_tas.replace(path.abspath(path.curdir), ""))
tar_segetalflora.add(dir_segetalflora, arcname=dir_segetalflora.replace(path.abspath(path.curdir), ""))
tar_fieldmeans.add(dir_fieldmean, arcname=dir_fieldmean.replace(path.abspath(path.curdir), ""))
tar_plots.add(dir_plots, arcname=dir_plots.replace(path.abspath(path.curdir), ""))
tar_tas.close()
tar_fieldmeans.close()
tar_segetalflora.close()
tar_plots.close()
logging.debug("tar ncfiles closed")
# === set output parameter
self.out_fieldmeans.setValue(tarf_fieldmeans)
self.out_segetalflora.setValue(tarf_segetalflora)
self.out_tas.setValue(tarf_tas)
self.out_plots.setValue(tarf_plots)
self.status.set("processing done", 100)
def _handler(self, request, response):
from flyingpigeon.utils import archive, archiveextract
from tempfile import mkstemp
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
ncs = archiveextract(
resource=rename_complexinputs(request.inputs['resource']))
# mosaic = self.mosaic.getValue()
regions = [inp.data for inp in request.inputs['region']]
response.update_status('Arguments set for subset process', 0)
LOGGER.debug('starting: regions=%s, num_files=%s' % (len(regions), len(ncs)))
try:
from flyingpigeon.visualisation import plot_polygons
png_country = plot_polygons(regions)
except:
LOGGER.exception('failed to plot the polygon to world map')
o1, png_country = mkstemp(dir='.', suffix='.png')
# clip the demanded polygons
from flyingpigeon.subset import clipping
subsets = clipping(resource=ncs, variable=None,
dimension_map=None,
calc=None,
output_format='nc',
calc_grouping=None,
time_range=None,
time_region=None,
historical_concatination=True,
prefix=None,
spatial_wrapping='wrap',
polygons=regions,
mosaic=True
)
try:
tar_subsets = archive(subsets)
except:
LOGGER.exception('failed to archive subsets')
_, tar_subsets = mkstemp(dir='.', suffix='.tar')
try:
from flyingpigeon.visualisation import uncertainty
png_uncertainty = uncertainty(subsets)
except:
LOGGER.exception('failed to generate the uncertainty plot')
_, png_uncertainty = mkstemp(dir='.', suffix='.png')
try:
from flyingpigeon.visualisation import spaghetti
png_spaghetti = spaghetti(subsets)
except:
LOGGER.exception('failed to generate the spaghetti plot')
_, png_spaghetti = mkstemp(dir='.', suffix='.png')
try:
from flyingpigeon import robustness as erob
from flyingpigeon.utils import get_variable
variable = get_variable(ncs[0])
signal, low_agreement_mask, high_agreement_mask, text_src = erob.method_A(resource=subsets,
# start=None, end=None,
# timeslice=None,
variable=variable
)
LOGGER.info('variable to be plotted: %s' % variable)
from flyingpigeon.visualisation import map_robustness
# if title is None:
title = 'signal robustness of %s ' % (variable) # , end1, end2, start1, start2
png_robustness = map_robustness(signal,
high_agreement_mask,
low_agreement_mask,
# cmap=cmap,
title=title)
LOGGER.info('graphic generated')
except:
LOGGER.exception('failed to generate the robustness plot')
_, png_robustness = mkstemp(dir='.', suffix='.png')
from flyingpigeon.visualisation import factsheetbrewer
factsheet = factsheetbrewer(png_country=png_country,
png_uncertainty=png_uncertainty,
png_spaghetti=png_spaghetti,
png_robustness=png_robustness)
response.outputs['output_nc'].file = tar_subsets
response.outputs['output_factsheet'].file = factsheet
response.update_status("done", 100)
return response