def execute(self): ncfiles = self.getInputValues(identifier='resource') var = self.variableIn.getValue() self.status.set('plotting variable %s' % var, 10) plotout_spagetti_file = vs.spaghetti(ncfiles, variable=var, title='Fieldmean of %s ' % (var), dir_out=None) self.status.set( 'Spagetti plot for %s %s files done' % (len(ncfiles), var), 50) plotout_uncertainty_file = vs.uncertainty( ncfiles, variable=var, title='Ensemble uncertainty for %s ' % (var), dir_out=None) self.status.set( 'Uncertainty plot for %s %s files done' % (len(ncfiles), var), 90) self.plotout_spagetti.setValue(plotout_spagetti_file) self.plotout_uncertainty.setValue(plotout_uncertainty_file) self.status.set('visualisation done', 100)
def execute(self): from flyingpigeon.utils import archive, archiveextract from tempfile import mkstemp init_process_logger('log.txt') self.output_log.setValue('log.txt') ncs = archiveextract(self.getInputValues(identifier='resource')) # mosaic = self.mosaic.getValue() regions = self.region.getValue() self.status.set('Arguments set for subset process', 0) logger.debug('starting: regions=%s, num_files=%s' % (len(regions), len(ncs))) try: from flyingpigeon.visualisation import plot_polygons png_country = plot_polygons(regions) except: logger.exception('failed to plot the polygon to world map') o1, factsheet_plot = mkstemp(dir='.', suffix='.png') # clip the demanded polygons from flyingpigeon.subset import clipping subsets = clipping(resource=ncs, variable=None, dimension_map=None, calc=None, output_format='nc', calc_grouping=None, time_range=None, time_region=None, historical_concatination=True, prefix=None, spatial_wrapping='wrap', polygons=regions, mosaic=True ) try: from flyingpigeon.visualisation import uncertainty png_uncertainty = uncertainty(subsets) except: logger.exception('failed to generate the uncertainty plot') _, png_uncertainty = mkstemp(dir='.', suffix='.png') try: from flyingpigeon.visualisation import spaghetti png_spaghetti = spaghetti(subsets) except: logger.exception('failed to generate the spaghetti plot') _, png_spaghetti = mkstemp(dir='.', suffix='.png') from flyingpigeon.visualisation import factsheetbrewer factsheet = factsheetbrewer(png_country=png_country, png_uncertainty=png_uncertainty, png_spaghetti=png_spaghetti) self.output_factsheet.setValue(factsheet) self.status.set('done', 100)
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' ncfiles = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) if 'variable' in request.inputs: var = request.inputs['variable'][0].data else: var = get_variable(ncfiles[0]) # var = ncfiles[0].split("_")[0] response.update_status('plotting variable {}'.format(var), 10) try: plotout_spagetti_file = vs.spaghetti( ncfiles, variable=var, title='Field mean of {}'.format(var), ) LOGGER.info("spagetti plot done") response.update_status( 'Spagetti plot for %s %s files done' % (len(ncfiles), var), 50) response.outputs['plotout_spagetti'].file = plotout_spagetti_file except Exception as e: raise Exception("spagetti plot failed : {}".format(e)) try: plotout_uncertainty_file = vs.uncertainty( ncfiles, variable=var, title='Ensemble uncertainty for {}'.format(var), ) response.update_status( 'Uncertainty plot for {} {} files done'.format( len(ncfiles), var), 90) response.outputs[ 'plotout_uncertainty'].file = plotout_uncertainty_file LOGGER.info("uncertainty plot done") except Exception as err: raise Exception("uncertainty plot failed {}".format(err.message)) response.update_status('visualisation done', 100) return response
def execute(self): ncfiles = self.getInputValues(identifier='resource') var = self.variableIn.getValue() self.status.set('plotting variable %s' % var, 10) plotout_spagetti_file = vs.spaghetti(ncfiles , variable=var, title=var, dir_out=None) self.status.set('Spagetti plot for %s %s files done' % (len(ncfiles), var), 50) plotout_uncertainty_file = vs.uncertainty(ncfiles , variable=var, title=var, dir_out=None) self.status.set('Uncertainty plot for %s %s files done' % (len(ncfiles), var), 90) self.plotout_spagetti.setValue( plotout_spagetti_file ) self.plotout_uncertainty.setValue( plotout_uncertainty_file ) self.status.set('visualisation done', 100)
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' ncfiles = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) var = request.inputs['variableIn'] if var is None: from flyingpigeon.utils import get_variable var = get_variable(ncfiles[0]) response.update_status('plotting variable %s' % var, 10) try: plotout_spagetti_file = vs.spaghetti(ncfiles, variable=var, title='Fieldmean of %s ' % (var), dir_out=None) LOGGER.info("spagetti plot done") response.update_status( 'Spagetti plot for %s %s files done' % (len(ncfiles), var), 50) except: LOGGER.exception("spagetti plot failed") try: plotout_uncertainty_file = vs.uncertainty( ncfiles, variable=var, title='Ensemble uncertainty for %s ' % (var), dir_out=None) response.update_status( 'Uncertainty plot for %s %s files done' % (len(ncfiles), var), 90) LOGGER.info("uncertainty plot done") except: LOGGER.exception("uncertainty plot failed") response.outputs['plotout_spagetti'].file = plotout_spagetti_file response.outputs['plotout_uncertainty'].file = plotout_uncertainty_file response.update_status('visualisation done', 100) return response
def execute(self): init_process_logger('log.txt') self.output_log.setValue('log.txt') from flyingpigeon.utils import archiveextract ncfiles = archiveextract(self.getInputValues(identifier='resource')) var = self.variableIn.getValue() if var is None: from flyingpigeon.utils import get_variable var = get_variable(ncfiles[0]) self.status.set('plotting variable %s' % var, 10) try: plotout_spagetti_file = vs.spaghetti( ncfiles, variable=var, title='Fieldmean of %s ' % (var), dir_out=None ) logger.info("spagetti plot done") self.status.set('Spagetti plot for %s %s files done' % (len(ncfiles), var), 50) except: logger.exception("spagetti plot failed") try: plotout_uncertainty_file = vs.uncertainty( ncfiles, variable=var, title='Ensemble uncertainty for %s ' % (var), dir_out=None ) self.status.set('Uncertainty plot for %s %s files done' % (len(ncfiles), var), 90) logger.info("uncertainty plot done") except: logger.exception("uncertainty plot failed") self.plotout_spagetti.setValue(plotout_spagetti_file) self.plotout_uncertainty.setValue(plotout_uncertainty_file) self.status.set('visualisation done', 100)
def execute(self): from os import mkdir, path, listdir import tarfile from tempfile import mkstemp #, mkdtemp from datetime import datetime from flyingpigeon import segetalflora as sf logging.debug('starting segetalflora process execution') self.status.set('starting calcualtion segetalflora', 5) ## prepare environment # create the tar files try: (fp_tarf_tas, tarf_tas) = mkstemp(dir=".", suffix='.tar') (fp_tarf_segetalflora, tarf_segetalflora) = mkstemp(dir=".", suffix='.tar') (fp_tarf_fieldmeans, tarf_fieldmeans) = mkstemp(dir=".", suffix='.tar') (fp_tarf_plots, tarf_plots) = mkstemp(dir=".", suffix='.tar') tar_tas = tarfile.open(tarf_tas, "w") tar_segetalflora = tarfile.open(tarf_segetalflora, "w") tar_fieldmeans = tarfile.open(tarf_fieldmeans, "w") tar_plots = tarfile.open(tarf_plots, "w") logging.debug('tar files initialized') # create output folders dir_tas = path.abspath(path.curdir+'/dir_tas/') dir_segetalflora = path.abspath(path.curdir+'/dir_segetalflora/') dir_fieldmean = path.abspath(path.curdir+'/dir_fieldmean/') dir_plots = path.abspath(path.curdir+'/dir_plots/') mkdir(dir_tas) mkdir(dir_segetalflora) mkdir(dir_fieldmean) mkdir(dir_plots) logging.debug('out directories created') except Exception as e: logging.exception('tar file or mkdir failed!') raise countries = ['AUT','BEL','BGR','CYP','CZE','DEU','DNK','ESP', 'EST','FIN','FRA','GBR','GRC','HUN','HRV','IRL', 'ITA','LVA','LTU','LUX','MLT','NLD','POL','PRT', 'ROU','SVK','SVN','SWE','NOR','CHE','ISL','MKD', 'MNE','SRB','MDA','UKR','BIH','ALB','BLR','KOS'] # read argments to variables #try: ncs = self.getInputValues(identifier='netcdf_file') climate_type = self.climate_type.getValue() culture_type = self.culture_type.getValue() countries = self.getInputValues(identifier='region') if type(climate_type) != list: climate_type = list([climate_type]) if type(culture_type) != list: culture_type = list([culture_type]) if type(countries) != list: countries = list([countries]) logging.debug('urls for %s ncs found' % (len(ncs))) logging.debug('culture type: %s ' % (culture_type)) # === main call for segetalflora processing stepps = len(culture_type) * len(climate_type) for a, cult in enumerate(culture_type): for b, clim in enumerate(climate_type): start = (a + 1) * (b + 1) per = (start / stepps) * 95 self.status.set('%s/%s processing for %s climate type: %s' %(start, stepps, culture_type, climate_type), per) try: sf_files = sf.get_segetalflora(ncs, culture_type=cult, climate_type=clim, #countries=countries, dir_output=dir_tas, dir_segetalflora=dir_segetalflora) self.status.set("processing of %s segetalflora files done " % (len(sf_files)) , 95) except Exception as e: logging.exception('segetalflora calculation failed %s %s' % ( climate_type, culture_type)) raise # === fieldmeans from flyingpigeon import timeseries as ts self.status.set('processing fieldmeans' , 97) try: ncs = [path.join(dir_segetalflora,nc) for nc in listdir(dir_segetalflora)] ncs_fld = ts.fldmean(ncs, dir_output=dir_fieldmean) logging.debug('%s fieldmeans processed' % (len(ncs_fld))) except Exception as e: logging.exception('fieldmeans failed') raise # === visualisation from flyingpigeon import visualisation as vs from os import rename self.status.set('processing visualisation' , 98) # sort files for plotting try: ncs = listdir(dir_segetalflora) set_var = set() set_contry = set() for nc in ncs: set_var = set_var.union([nc.split('_')[0]]) set_contry = set_contry.union([nc.split('_')[1]]) logging.debug('%s files to plots sorted' % (len(ncs))) except Exception as e: logging.exception('files sorting failed') raise # plot sorted files try: plots = [] for v in set_var: for c in set_contry: ncs = [path.join(dir_segetalflora,nc) for nc in listdir(dir_segetalflora) if v in nc and c in nc ] p = vs.spaghetti(ncs, variable=v, title='Segetalflora %s in %s' % (v, c), dir_out=dir_plots) newname = path.dirname(p)+'/%s_%s_birdhouse_output.html' %(v,c) rename(p,newname) plots.append(newname) logging.debug('plot created and renamed for %s %s' % (v, c )) except Exception as e: logging.exception('ploting failed') raise # === tar file archiving self.status.set('files to tar archives', 99) tar_tas.add(dir_tas, arcname = dir_tas.replace(path.abspath(path.curdir), "")) tar_segetalflora.add(dir_segetalflora, arcname = dir_segetalflora.replace(path.abspath(path.curdir), "")) tar_fieldmeans.add(dir_fieldmean, arcname = dir_fieldmean.replace(path.abspath(path.curdir), "")) tar_plots.add(dir_plots, arcname = dir_plots.replace(path.abspath(path.curdir), "")) tar_tas.close() tar_fieldmeans.close() tar_segetalflora.close() tar_plots.close() logging.debug('tar ncfiles closed') # === set output parameter self.out_fieldmeans.setValue( tarf_fieldmeans ) self.out_segetalflora.setValue( tarf_segetalflora ) self.out_tas.setValue( tarf_tas ) self.out_plots.setValue( tarf_plots ) self.status.set("processing done", 100)
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' ncs = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) var = get_variable(ncs[0]) LOGGER.info('variable to be plotted: {}'.format(var)) # mosaic = self.mosaic.getValue() if 'region' in request.inputs: regions = [inp.data for inp in request.inputs['region']] try: png_region = vs.plot_polygons(regions) except Exception as ex: msg = 'failed to plot the polygon to world map: {}'.format( str(ex)) LOGGER.exception(msg) raise Exception(msg) o1, png_region = mkstemp(dir='.', suffix='.png') # clip the demanded polygons subsets = clipping( resource=ncs, variable=var, polygons=regions, mosaic=True, spatial_wrapping='wrap', ) else: subsets = ncs png_region = vs.plot_extend(ncs[0]) response.update_status('Arguments set for subset process', 0) try: tar_subsets = archive(subsets) except Exception as ex: msg = 'failed to archive subsets: {}'.format(ex) LOGGER.exception(msg) raise Exception(msg) _, tar_subsets = mkstemp(dir='.', suffix='.tar') try: png_uncertainty = vs.uncertainty(subsets, variable=var) except Exception as ex: msg = 'failed to generate the uncertainty plot: {}'.format(ex) LOGGER.exception(msg) raise Exception(msg) _, png_uncertainty = mkstemp(dir='.', suffix='.png') try: png_spaghetti = vs.spaghetti( subsets, variable=var, ) except Exception as ex: msg = 'failed to generate the spaghetti plot: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) _, png_spaghetti = mkstemp(dir='.', suffix='.png') try: from flyingpigeon import robustness as ro signal, low_agreement_mask, high_agreement_mask, text_src = ro.signal_noise_ratio( resource=subsets, # start=None, end=None, # timeslice=None, # variable=var ) # if title is None: title = 'signal robustness of %s ' % ( var) # , end1, end2, start1, start2 png_robustness = vs.map_robustness( signal, high_agreement_mask, low_agreement_mask, # cmap=cmap, # title=title ) LOGGER.info('robustness graphic generated') except Exception as ex: msg = 'failed to generate the robustness plot: {}'.format(ex) LOGGER.exception(msg) raise Exception(msg) _, png_robustness = mkstemp(dir='.', suffix='.png') factsheet = vs.factsheetbrewer(png_region=png_region, png_uncertainty=png_uncertainty, png_spaghetti=png_spaghetti, png_robustness=png_robustness) response.outputs['output_nc'].file = tar_subsets response.outputs['output_factsheet'].file = factsheet response.update_status("done", 100) return response
def execute(self): from os import mkdir, path, listdir import tarfile from tempfile import mkstemp # , mkdtemp from datetime import datetime from flyingpigeon import segetalflora as sf logging.debug("starting segetalflora process execution") self.status.set("starting calcualtion segetalflora", 5) ## prepare environment # create the tar files try: (fp_tarf_tas, tarf_tas) = mkstemp(dir=".", suffix=".tar") (fp_tarf_segetalflora, tarf_segetalflora) = mkstemp(dir=".", suffix=".tar") (fp_tarf_fieldmeans, tarf_fieldmeans) = mkstemp(dir=".", suffix=".tar") (fp_tarf_plots, tarf_plots) = mkstemp(dir=".", suffix=".tar") tar_tas = tarfile.open(tarf_tas, "w") tar_segetalflora = tarfile.open(tarf_segetalflora, "w") tar_fieldmeans = tarfile.open(tarf_fieldmeans, "w") tar_plots = tarfile.open(tarf_plots, "w") logging.debug("tar files initialized") # create output folders dir_tas = path.abspath(path.curdir + "/dir_tas/") dir_segetalflora = path.abspath(path.curdir + "/dir_segetalflora/") dir_fieldmean = path.abspath(path.curdir + "/dir_fieldmean/") dir_plots = path.abspath(path.curdir + "/dir_plots/") mkdir(dir_tas) mkdir(dir_segetalflora) mkdir(dir_fieldmean) mkdir(dir_plots) logging.debug("out directories created") except Exception as e: logging.exception("tar file or mkdir failed!") raise countries = [ "AUT", "BEL", "BGR", "CYP", "CZE", "DEU", "DNK", "ESP", "EST", "FIN", "FRA", "GBR", "GRC", "HUN", "HRV", "IRL", "ITA", "LVA", "LTU", "LUX", "MLT", "NLD", "POL", "PRT", "ROU", "SVK", "SVN", "SWE", "NOR", "CHE", "ISL", "MKD", "MNE", "SRB", "MDA", "UKR", "BIH", "ALB", "BLR", "KOS", ] # read argments to variables # try: ncs = self.getInputValues(identifier="netcdf_file") climate_type = self.climate_type.getValue() culture_type = self.culture_type.getValue() # countries = self.getInputValues(identifier='region') if type(climate_type) != list: climate_type = list([climate_type]) if type(culture_type) != list: culture_type = list([culture_type]) # if type(countries) != list: # countries = list([countries]) logging.debug("urls for %s ncs found" % (len(ncs))) logging.debug("culture type: %s " % (culture_type)) # === main call for segetalflora processing stepps = len(culture_type) * len(climate_type) for a, cult in enumerate(culture_type): for b, clim in enumerate(climate_type): start = (a + 1) * (b + 1) per = (start / stepps) * 95 self.status.set( "%s/%s processing for %s climate type: %s" % (start, stepps, culture_type, climate_type), per ) try: sf_files = sf.get_segetalflora( resource=ncs, dir_output=dir_tas, culture_type=cult, climate_type=clim, region=None, dimension_map=None, ) self.status.set("processing of %s segetalflora files done " % (len(sf_files)), 95) except Exception as e: logging.exception("segetalflora calculation failed %s %s" % (climate_type, culture_type)) raise # === fieldmeans from flyingpigeon import timeseries as ts self.status.set("processing fieldmeans", 97) try: ncs = [path.join(dir_segetalflora, nc) for nc in listdir(dir_segetalflora)] ncs_fld = ts.fldmean(ncs, dir_output=dir_fieldmean) logging.debug("%s fieldmeans processed" % (len(ncs_fld))) except Exception as e: logging.exception("fieldmeans failed") raise # === visualisation from flyingpigeon import visualisation as vs from os import rename self.status.set("processing visualisation", 98) # sort files for plotting try: ncs = listdir(dir_segetalflora) set_var = set() set_contry = set() for nc in ncs: set_var = set_var.union([nc.split("_")[0]]) set_contry = set_contry.union([nc.split("_")[1]]) logging.debug("%s files to plots sorted" % (len(ncs))) except Exception as e: logging.exception("files sorting failed") raise # plot sorted files try: plots = [] for v in set_var: for c in set_contry: ncs = [path.join(dir_segetalflora, nc) for nc in listdir(dir_segetalflora) if v in nc and c in nc] p = vs.spaghetti(ncs, variable=v, title="Segetalflora %s in %s" % (v, c), dir_out=dir_plots) newname = path.dirname(p) + "/%s_%s_birdhouse_output.html" % (v, c) rename(p, newname) plots.append(newname) logging.debug("plot created and renamed for %s %s" % (v, c)) except Exception as e: logging.exception("ploting failed") raise # === tar file archiving self.status.set("files to tar archives", 99) tar_tas.add(dir_tas, arcname=dir_tas.replace(path.abspath(path.curdir), "")) tar_segetalflora.add(dir_segetalflora, arcname=dir_segetalflora.replace(path.abspath(path.curdir), "")) tar_fieldmeans.add(dir_fieldmean, arcname=dir_fieldmean.replace(path.abspath(path.curdir), "")) tar_plots.add(dir_plots, arcname=dir_plots.replace(path.abspath(path.curdir), "")) tar_tas.close() tar_fieldmeans.close() tar_segetalflora.close() tar_plots.close() logging.debug("tar ncfiles closed") # === set output parameter self.out_fieldmeans.setValue(tarf_fieldmeans) self.out_segetalflora.setValue(tarf_segetalflora) self.out_tas.setValue(tarf_tas) self.out_plots.setValue(tarf_plots) self.status.set("processing done", 100)
def _handler(self, request, response): from flyingpigeon.utils import archive, archiveextract from tempfile import mkstemp init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' ncs = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) # mosaic = self.mosaic.getValue() regions = [inp.data for inp in request.inputs['region']] response.update_status('Arguments set for subset process', 0) LOGGER.debug('starting: regions=%s, num_files=%s' % (len(regions), len(ncs))) try: from flyingpigeon.visualisation import plot_polygons png_country = plot_polygons(regions) except: LOGGER.exception('failed to plot the polygon to world map') o1, png_country = mkstemp(dir='.', suffix='.png') # clip the demanded polygons from flyingpigeon.subset import clipping subsets = clipping(resource=ncs, variable=None, dimension_map=None, calc=None, output_format='nc', calc_grouping=None, time_range=None, time_region=None, historical_concatination=True, prefix=None, spatial_wrapping='wrap', polygons=regions, mosaic=True ) try: tar_subsets = archive(subsets) except: LOGGER.exception('failed to archive subsets') _, tar_subsets = mkstemp(dir='.', suffix='.tar') try: from flyingpigeon.visualisation import uncertainty png_uncertainty = uncertainty(subsets) except: LOGGER.exception('failed to generate the uncertainty plot') _, png_uncertainty = mkstemp(dir='.', suffix='.png') try: from flyingpigeon.visualisation import spaghetti png_spaghetti = spaghetti(subsets) except: LOGGER.exception('failed to generate the spaghetti plot') _, png_spaghetti = mkstemp(dir='.', suffix='.png') try: from flyingpigeon import robustness as erob from flyingpigeon.utils import get_variable variable = get_variable(ncs[0]) signal, low_agreement_mask, high_agreement_mask, text_src = erob.method_A(resource=subsets, # start=None, end=None, # timeslice=None, variable=variable ) LOGGER.info('variable to be plotted: %s' % variable) from flyingpigeon.visualisation import map_robustness # if title is None: title = 'signal robustness of %s ' % (variable) # , end1, end2, start1, start2 png_robustness = map_robustness(signal, high_agreement_mask, low_agreement_mask, # cmap=cmap, title=title) LOGGER.info('graphic generated') except: LOGGER.exception('failed to generate the robustness plot') _, png_robustness = mkstemp(dir='.', suffix='.png') from flyingpigeon.visualisation import factsheetbrewer factsheet = factsheetbrewer(png_country=png_country, png_uncertainty=png_uncertainty, png_spaghetti=png_spaghetti, png_robustness=png_robustness) response.outputs['output_nc'].file = tar_subsets response.outputs['output_factsheet'].file = factsheet response.update_status("done", 100) return response