def _handler(self, request, response): import uuid import time import json outputpath = configuration.get_config_value('server', 'outputpath') init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' # -------------- # # Input handling # # -------------- # resource = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) LOGGER.info("resource: %s " % resource) dest = archiveextract( resource=rename_complexinputs(request.inputs['dest'])) LOGGER.info("dest: %s " % dest) method = request.inputs['method'][0].data LOGGER.info("method: %s " % method) snippet = request.inputs['snippet'][0].data LOGGER.info("snippet: %s " % snippet) # -------------------- # # Regridding operation # # -------------------- # d = ocgis.RequestDataset(dest) m = getattr(ESMF.RegridMethod, method.upper()) LOGGER.info('Start ocgis module call function') # Prepare the environment ocgis.env.OVERWRITE = True prefix = str(uuid.uuid1()) ocgis.env.PREFIX = prefix outputs = [] for source in resource: s = ocgis.RequestDataset(source) ops = ocgis.OcgOperations(dataset=s, regrid_destination=d, regrid_options={'regrid_method': m}, snippet=snippet, dir_output=outputpath, output_format='nc', prefix=prefix) outputs.append(ops.execute()) response.outputs['output_netcdf'].file = outputs[0] time_str = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) output_json = "esmf_regrid_results_{}.json".format(time_str) with open(output_json, 'w') as f: f.write(json.dumps([actual_output_path(o) for o in outputs])) response.outputs['output'].file = output_json response.outputs['output'].output_format = json_format return response
def _handler(self, request, response): from rpy2 import robjects from rpy2.robjects.packages import importr import os import datetime as dt tic = dt.datetime.now() init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' LOGGER.info('Start process') response.update_status('Execution started at : {}'.format(tic), 1) ###################################### # Read inputs ###################################### try: obs = archiveextract( resource=rename_complexinputs(request.inputs['obs'])) ref = archiveextract( resource=rename_complexinputs(request.inputs['ref'])) fut = archiveextract( resource=rename_complexinputs(request.inputs['fut'])) # dedrizzle = request.inputs['dedrizzle'][0].data # norm = request.inputs['norm'][0].data except Exception as e: msg = 'Failed to read input parameter {}'.format(e) msg += "obs: " + request.inputs['obs'] msg += "ref: " + request.inputs['ref'] msg += "fut: " + request.inputs['fut'] LOGGER.error(msg) raise Exception(msg) response.update_status('Input parameters ingested', 2) rp, ext = os.path.splitext(ref[0]) ref_out = rp + '_kddm-bc' + ext fp, ext = os.path.splitext(fut[0]) fut_out = fp + '_kddm-bc' + ext # Assuming all files share the same variable. rd = ocgis.RequestDataset(ref) varname = rd.variable # Calling the R code Rsrc = os.path.join(flyingpigeon.config.Rsrc_dir(), 'bc.kddm.R') devtools = importr("devtools") rfunc = robjects.r(open(Rsrc).read()) rfunc(varname, obs[0], ref[0], fut[0], ref_out, fut_out, False) response.outputs['output_netcdf_ref'].file = ref_out response.outputs['output_netcdf_fut'].file = fut_out return response
def execute(self): init_process_logger('log.txt') self.output_log.setValue('log.txt') from flyingpigeon.utils import searchfile from flyingpigeon.subset import masking from flyingpigeon.utils import archive, archiveextract from flyingpigeon import config from os import path resources = archiveextract(self.getInputValues(identifier='resource')) masks = archiveextract(self.getInputValues(identifier='mask')) land_area = self.land_area.getValue() fp_cache = config.cache_path().split('/') base_dir = '/'.join(fp_cache[0:-1]) # base dir for all birds logger.debug('base dir of directory tree: %s' % base_dir) ncs = [] sftlf = [] for nc in resources: try: basename = path.basename(nc) bs = basename.split('_') pattern = 'sftlf_' + '_'.join(bs[1:-2]) + '_fx.nc' pattern = pattern.replace('historical', '*').replace('rcp85', '*').replace('rcp65', '*').replace('rcp45', '*').replace('rcp26', '*') logger.debug('searching for %s ' % pattern) sftlf.extend(searchfile(pattern, path.curdir)) sftlf.extend(searchfile(pattern, base_dir)) logger.debug('lenght of sftlf: %s' % len(sftlf)) if len(sftlf) >= 1: if len(sftlf) > 1: logger.warn( 'more than one sftlf file is found fitting to the pattern, first one will be taken %s' % sftlf[0]) prefix = 'masked%s' % basename.replace('.nc', '') nc_mask = masking(nc, sftlf[0], land_area=land_area, prefix=prefix) ncs.extend([nc_mask]) logger.info('masking processed for %s' % basename) else: logger.warn('no masked found. Please perform a "Download Resources"\ to make sure the land_area file is in cache') except: logger.exception('failed to mask file: %s' % basename) nc_archive = archive(ncs) self.output_archive.setValue(nc_archive) i = next((i for i, x in enumerate(ncs) if x), None) self.output_example.setValue(ncs[i])
def execute(self): self.status.set('starting uncertainty process', 0) init_process_logger('log.txt') self.output_log.setValue('log.txt') from flyingpigeon.utils import archiveextract ncfiles = archiveextract(self.getInputValues(identifier='resource')) start = self.start.getValue() end = self.end.getValue() timeslice = self.timeslice.getValue() variable = self.variableIn.getValue() method = self.method.getValue() self.status.set('arguments read', 5) if method == 'Method_A': signal, low_agreement_mask, high_agreement_mask, graphic, text_src = erob.method_A( resource=ncfiles, start=start, end=end, timeslice=timeslice, variable=variable) # graphic, self.status.set('process worker done', 95) self.output_signal.setValue(signal) self.output_high.setValue(high_agreement_mask) self.output_low.setValue(low_agreement_mask) self.output_graphic.setValue(graphic) self.output_text.setValue(text_src) self.status.set('uncertainty process done', 100)
def execute(self): from ast import literal_eval from flyingpigeon.utils import archive, archiveextract init_process_logger('log.txt') self.output_log.setValue('log.txt') ncs = archiveextract(self.getInputValues(identifier='resource')) mosaic = self.mosaic.getValue() regions = self.region.getValue() # variable = self.variable.getValue() # logger.info('regions: %s' % regions) # dimension_map = self.dimension_map.getValue() # if dimension_map != None: # dimension_map = literal_eval(dimension_map) logger.info('ncs = %s', ncs) logger.info('regions = %s', regions) logger.info('mosaic = %s', mosaic) # logger.info('dimension_map = %s', dimension_map) self.status.set('Arguments set for subset process', 0) logger.debug('starting: regions=%s, num_files=%s' % (len(regions), len(ncs))) try: results = clipping( resource=ncs, polygons=regions, # self.region.getValue(), mosaic=mosaic, spatial_wrapping='wrap', # variable=variable, dir_output=os.path.abspath(os.curdir), # dimension_map=dimension_map, ) logger.info('results %s' % results) except Exception as e: msg = 'clipping failed' logger.exception(msg) raise Exception(msg) if not results: raise Exception('no results produced.') # prepare tar file try: tarf = archive(results) logger.info('Tar file prepared') except Exception as e: msg = 'Tar file preparation failed' logger.exception(msg) raise Exception(msg) self.output.setValue(tarf) i = next((i for i, x in enumerate(results) if x), None) self.output_netcdf.setValue(results[i]) self.status.set('done', 100)
def _resource_input_handler(self, request): out = OrderedDict() for obj in self.resource_inputs: key = obj.identifier out[key] = archiveextract( resource=rename_complexinputs(request.inputs[key])) return out
def execute(self): from flyingpigeon.utils import archive, archiveextract from tempfile import mkstemp init_process_logger('log.txt') self.output_log.setValue('log.txt') ncs = archiveextract(self.getInputValues(identifier='resource')) # mosaic = self.mosaic.getValue() regions = self.region.getValue() self.status.set('Arguments set for subset process', 0) logger.debug('starting: regions=%s, num_files=%s' % (len(regions), len(ncs))) try: from flyingpigeon.visualisation import plot_polygons png_country = plot_polygons(regions) except: logger.exception('failed to plot the polygon to world map') o1, factsheet_plot = mkstemp(dir='.', suffix='.png') # clip the demanded polygons from flyingpigeon.subset import clipping subsets = clipping(resource=ncs, variable=None, dimension_map=None, calc=None, output_format='nc', calc_grouping=None, time_range=None, time_region=None, historical_concatination=True, prefix=None, spatial_wrapping='wrap', polygons=regions, mosaic=True ) try: from flyingpigeon.visualisation import uncertainty png_uncertainty = uncertainty(subsets) except: logger.exception('failed to generate the uncertainty plot') _, png_uncertainty = mkstemp(dir='.', suffix='.png') try: from flyingpigeon.visualisation import spaghetti png_spaghetti = spaghetti(subsets) except: logger.exception('failed to generate the spaghetti plot') _, png_spaghetti = mkstemp(dir='.', suffix='.png') from flyingpigeon.visualisation import factsheetbrewer factsheet = factsheetbrewer(png_country=png_country, png_uncertainty=png_uncertainty, png_spaghetti=png_spaghetti) self.output_factsheet.setValue(factsheet) self.status.set('done', 100)
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' response.update_status('Start process', 0) try: LOGGER.info('reading the arguments') resources = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) indices = [inpt.data for inpt in request.inputs['indices']] LOGGER.debug("indices = %s", indices) archive_format = request.inputs['archive_format'][0].data except: msg = 'failed to read the arguments.' LOGGER.exception(msg) raise Exception(msg) LOGGER.info('indices %s ' % indices) ################################# # calculate the climate indices ################################# # indices calculation ncs_indices = None datasets = sort_by_filename(resources, historical_concatination=True) LOGGER.debug("datasets=%s", datasets.keys()) for ds_name in datasets: try: response.update_status('calculation of {}'.format(ds_name), 30) # TODO: what is happening with the results for each ds? ncs_indices = sdm.get_indices(resource=datasets[ds_name], indices=indices) except: msg = 'indice calculation failed for {}'.format(ds_name) LOGGER.exception(msg) raise Exception(msg) # archive multiple output files to one archive file try: archive_indices = archive(ncs_indices, format=archive_format) LOGGER.info('indices 3D added to tarfile') except: msg = 'failed adding indices to tar' LOGGER.exception(msg) raise Exception(msg) response.outputs['output_indices'].file = archive_indices i = next((i for i, x in enumerate(ncs_indices) if x), None) response.outputs['ncout'].file = ncs_indices[i] response.update_status('done', 100) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' ncs = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) LOGGER.info('ncs: {}'.format(ncs)) coords = [] for coord in request.inputs['coords']: coords.append(coord.data) LOGGER.info('coords {}'.format(coords)) filenames = [] nc_exp = sort_by_filename(ncs, historical_concatination=True) for key in nc_exp.keys(): try: LOGGER.info('start calculation for {}'.format(key)) ncs = nc_exp[key] times = get_time(ncs) # , format='%Y-%m-%d_%H:%M:%S') concat_vals = times # ['%s-%02d-%02d_%02d:%02d:%02d' % # (t.year, t.month, t.day, t.hour, t.minute, t.second) for t in times] header = 'date_time' filename = '{}.csv'.format(key) filenames.append(filename) for p in coords: try: response.update_status('processing point: {}'.format(p), 20) # define the point: p = p.split(',') point = Point(float(p[0]), float(p[1])) # get the values timeseries = call(resource=ncs, geom=point, select_nearest=True) vals = get_values(timeseries) # concatenation of values header = header + ',{}-{}'.format(p[0], p[1]) concat_vals = column_stack([concat_vals, vals]) except Exception as e: LOGGER.debug('failed for point {} {}'.format(p, e)) response.update_status('*** all points processed for {0} ****'.format(key), 50) # TODO: Ascertain whether this 'savetxt' is a valid command without string formatting argument: '%s' savetxt(filename, concat_vals, fmt='%s', delimiter=',', header=header) except Exception as ex: LOGGER.debug('failed for {}: {}'.format(key, str(ex))) # set the outputs response.update_status('*** creating output tar archive ****', 90) tarout_file = archive(filenames) response.outputs['tarout'].file = tarout_file return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' datasets = [] # append file urls if 'dataset' in request.inputs: datasets.extend( archiveextract( resource=rename_complexinputs(request.inputs['dataset']))) # append opendap urls if 'dataset_opendap' in request.inputs: for dataset in request.inputs['dataset_opendap']: datasets.append(dataset.data) # land or sea flag land_area_flag = request.inputs['land_or_sea'][0].data == 'land' masked_datasets = [] count = 0 max_count = len(datasets) for ds in datasets: ds_name = os.path.basename(ds) LOGGER.info('masking dataset: {}'.format(ds_name)) if 'mask' in request.inputs: landsea_mask = request.inputs['mask'][0].data else: landsea_mask = search_landsea_mask_by_esgf(ds) LOGGER.info("using landsea_mask: {}".format(landsea_mask)) prefix = 'masked_{}'.format(ds_name.replace('.nc', '')) try: new_ds = masking(ds, landsea_mask, land_area=land_area_flag, prefix=prefix) masked_datasets.append(new_ds) except Exception as ex: msg = 'Could not subset dataset {}: {}'.format( ds_name, str(ex)) LOGGER.exception(msg) raise Exception(msg) count = count + 1 response.update_status( "masked: {:d}/{:d}".format(count, max_count), int(100.0 * count / max_count)) response.outputs['output_archive'].file = archive(masked_datasets) response.outputs['output_example'].file = masked_datasets[0] response.update_status("done", 100) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' response.update_status('Start process', 0) try: LOGGER.info('reading the arguments') resources = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) indices = [inpt.data for inpt in request.inputs['indices']] LOGGER.debug("indices = {}".format(indices)) archive_format = request.inputs['archive_format'][0].data except Exception as ex: msg = 'failed to read the arguments: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) LOGGER.info('indices {}'.format(indices)) ################################# # calculate the climate indices ################################# # indices calculation try: response.update_status('calculation of indices', 30) ncs_indices = sdm.get_indices(resource=resources, indices=indices) LOGGER.info('indice calculation done') except Exception as ex: # TODO: 'ds_name' does not resolve. What is this referring to? This will throw a critical error! msg = 'indice calculation failed for {}: {}'.format(ds_name, str(ex)) LOGGER.exception(msg) raise Exception(msg) # archive multiple output files to one archive file try: archive_indices = archive(ncs_indices, format=archive_format) LOGGER.info('indices 3D added to tarfile') except Exception as ex: msg = 'failed adding indices to tar: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) response.outputs['output_indices'].file = archive_indices i = next((i for i, x in enumerate(ncs_indices) if x), None) response.outputs['ncout'].file = ncs_indices[i] response.update_status('done', 100) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' ncs = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) LOGGER.info("ncs: %s " % ncs) coords = request.inputs['coords'] # self.getInputValues(identifier='coords') LOGGER.info("coords %s", coords) filenames = [] nc_exp = sort_by_filename(ncs, historical_concatination=True) for key in nc_exp.keys(): try: LOGGER.info('start calculation for %s ' % key) ncs = nc_exp[key] times = get_time(ncs, format='%Y-%m-%d_%H:%M:%S') concat_vals = times # ['%s-%02d-%02d_%02d:%02d:%02d' % # (t.year, t.month, t.day, t.hour, t.minute, t.second) for t in times] header = 'date_time' filename = '%s.csv' % key filenames.append(filename) for p in coords: try: response.update_status('processing point : {0}'.format(p), 20) # define the point: p = p.split(',') point = Point(float(p[0]), float(p[1])) # get the values timeseries = call(resource=ncs, geom=point, select_nearest=True) vals = get_values(timeseries) # concatenation of values header = header + ',%s-%s' % (p[0], p[1]) concat_vals = column_stack([concat_vals, vals]) except Exception as e: LOGGER.debug('failed for point %s %s' % (p, e)) response.update_status('*** all points processed for {0} ****'.format(key), 50) savetxt(filename, concat_vals, fmt='%s', delimiter=',', header=header) except Exception as e: LOGGER.debug('failed for %s %s' % (key, e)) # set the outputs response.update_status('*** creating output tar archive ****', 90) tarout_file = archive(filenames) response.outputs['tarout'].file = tarout_file return response
def _handler(self, request, response): from tempfile import mkstemp tic = dt.now() init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' LOGGER.info('Start process') response.update_status('Execution started at : {}'.format(tic), 1) ###################################### # Read inputs ###################################### try: resource = archiveextract( resource=rename_complexinputs(request.inputs['resource']))[0] fmts = [e.data for e in request.inputs['fmt']] title = request.inputs['title'][0].data except Exception as e: msg = 'Failed to read input parameter {}'.format(e) LOGGER.error(msg) raise Exception(msg) response.update_status('Input parameters ingested', 2) try: fig = map_spatial_analog(resource, title=title) output = [] for fmt in fmts: output.append(fig2plot(fig, fmt)) except Exception as e: msg = "Failed to create figure: {}".format(e) LOGGER.error(msg) raise Exception(msg) finally: plt.close() if len(fmts) == 1: output = output[0] else: output = archive(output) response.outputs['output_figure'].file = output response.update_status("done", 100) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' ncfiles = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) if 'variable' in request.inputs: var = request.inputs['variable'][0].data else: var = get_variable(ncfiles[0]) # var = ncfiles[0].split("_")[0] response.update_status('plotting variable {}'.format(var), 10) try: plotout_spagetti_file = vs.spaghetti( ncfiles, variable=var, title='Field mean of {}'.format(var), ) LOGGER.info("spagetti plot done") response.update_status( 'Spagetti plot for %s %s files done' % (len(ncfiles), var), 50) response.outputs['plotout_spagetti'].file = plotout_spagetti_file except Exception as e: raise Exception("spagetti plot failed : {}".format(e)) try: plotout_uncertainty_file = vs.uncertainty( ncfiles, variable=var, title='Ensemble uncertainty for {}'.format(var), ) response.update_status( 'Uncertainty plot for {} {} files done'.format( len(ncfiles), var), 90) response.outputs[ 'plotout_uncertainty'].file = plotout_uncertainty_file LOGGER.info("uncertainty plot done") except Exception as err: raise Exception("uncertainty plot failed {}".format(err.message)) response.update_status('visualisation done', 100) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' ncfiles = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) var = request.inputs['variableIn'] if var is None: from flyingpigeon.utils import get_variable var = get_variable(ncfiles[0]) response.update_status('plotting variable %s' % var, 10) try: plotout_spagetti_file = vs.spaghetti(ncfiles, variable=var, title='Fieldmean of %s ' % (var), dir_out=None) LOGGER.info("spagetti plot done") response.update_status( 'Spagetti plot for %s %s files done' % (len(ncfiles), var), 50) except: LOGGER.exception("spagetti plot failed") try: plotout_uncertainty_file = vs.uncertainty( ncfiles, variable=var, title='Ensemble uncertainty for %s ' % (var), dir_out=None) response.update_status( 'Uncertainty plot for %s %s files done' % (len(ncfiles), var), 90) LOGGER.info("uncertainty plot done") except: LOGGER.exception("uncertainty plot failed") response.outputs['plotout_spagetti'].file = plotout_spagetti_file response.outputs['plotout_uncertainty'].file = plotout_uncertainty_file response.update_status('visualisation done', 100) return response
def execute(self): init_process_logger('log.txt') self.output_log.setValue('log.txt') from flyingpigeon.utils import archiveextract ncfiles = archiveextract(self.getInputValues(identifier='resource')) var = self.variableIn.getValue() if var is None: from flyingpigeon.utils import get_variable var = get_variable(ncfiles[0]) self.status.set('plotting variable %s' % var, 10) try: plotout_spagetti_file = vs.spaghetti( ncfiles, variable=var, title='Fieldmean of %s ' % (var), dir_out=None ) logger.info("spagetti plot done") self.status.set('Spagetti plot for %s %s files done' % (len(ncfiles), var), 50) except: logger.exception("spagetti plot failed") try: plotout_uncertainty_file = vs.uncertainty( ncfiles, variable=var, title='Ensemble uncertainty for %s ' % (var), dir_out=None ) self.status.set('Uncertainty plot for %s %s files done' % (len(ncfiles), var), 90) logger.info("uncertainty plot done") except: logger.exception("uncertainty plot failed") self.plotout_spagetti.setValue(plotout_spagetti_file) self.plotout_uncertainty.setValue(plotout_uncertainty_file) self.status.set('visualisation done', 100)
def _handler(self, request, response): try: ocgis.env.DIR_OUTPUT = tempfile.mkdtemp(dir=os.getcwd()) ocgis.env.OVERWRITE = True nc_files = archiveextract(resource=rename_complexinputs( request.inputs['resource'])) rd = RequestDataset(nc_files) rd.dimension_map.set_bounds('time', None) if nc_files[0][-3:] == '.nc': out_prefix = nc_files[0][:-3] + '_merged' else: out_prefix = nc_files[0] + '_merged' ops = OcgOperations(dataset=rd, output_format='nc', prefix=out_prefix) ret = ops.execute() response.outputs['output'].file = ret response.outputs['output'].output_format = \ Format('application/x-netcdf') return response except: raise Exception(traceback.format_exc())
def _handler(self, request, response): resource = archiveextract(resource=rename_complexinputs( request.inputs['resource']))[0] init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' LOGGER.info('Start process') LOGGER.debug("Rank : {}".format(ocgis.vm.rank)) print("Rank : {}".format(ocgis.vm.rank)) rd = ocgis.RequestDataset(resource) ops = ocgis.OcgOperations(dataset=rd, calc=[{'func': 'mean', 'name': 'mean'}], calc_grouping=['month'], output_format='nc') output = ops.execute() response.outputs['output_netcdf'].file = output response.update_status('Execution completed', 100) return response
def _handler(self, request, response): ocgis.env.DIR_OUTPUT = tempfile.mkdtemp(dir=os.getcwd()) ocgis.env.OVERWRITE = True tic = dt.now() init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' LOGGER.info('Start process') response.update_status('Execution started at : {}'.format(tic), 1) ###################################### # Read inputs ###################################### try: candidate = archiveextract( resource=rename_complexinputs(request.inputs['candidate'])) target = archiveextract( resource=rename_complexinputs(request.inputs['target'])) location = request.inputs['location'][0].data indices = [el.data for el in request.inputs['indices']] dist = request.inputs['dist'][0].data dateStartCandidate = request.inputs['dateStartCandidate'][0].data dateEndCandidate = request.inputs['dateEndCandidate'][0].data dateStartTarget = request.inputs['dateStartTarget'][0].data dateEndTarget = request.inputs['dateEndTarget'][0].data except Exception as ex: msg = 'Failed to read input parameter {}'.format(ex) LOGGER.error(msg) raise Exception(msg) response.update_status('Input parameters ingested', 2) ###################################### # Process inputs ###################################### try: point = Point(*map(float, location.split(','))) dateStartCandidate = dt.strptime(dateStartCandidate, '%Y-%m-%d') dateEndCandidate = dt.strptime(dateEndCandidate, '%Y-%m-%d') dateStartTarget = dt.strptime(dateStartTarget, '%Y-%m-%d') dateEndTarget = dt.strptime(dateEndTarget, '%Y-%m-%d') except Exception as ex: msg = 'failed to process inputs {}'.format(ex) LOGGER.error(msg) raise Exception(msg) LOGGER.debug("init took {}".format(dt.now() - tic)) response.update_status('Processed input parameters', 3) ###################################### # Extract target time series ###################################### savetarget = False try: # Using `call` creates a netCDF file in the tmp directory. # # Here we keep this stuff in memory if savetarget: prefix = 'target_ts' target_ts = call(resource=target, geom=point, variable=indices, time_range=[dateStartTarget, dateEndTarget], select_nearest=True, prefix=prefix) # target_ts = [get_values(prefix+'.nc', ind) for ind in indices] else: trd = RequestDataset( target, variable=indices, time_range=[dateStartTarget, dateEndTarget]) op = OcgOperations(trd, geom=point, select_nearest=True, search_radius_mult=1.75) out = op.execute() target_ts = out.get_element() except Exception as ex: msg = 'Target extraction failed {}'.format(ex) LOGGER.debug(msg) raise Exception(msg) response.update_status('Extracted target series', 5) ###################################### # Compute dissimilarity metric ###################################### response.update_status('Computing spatial analog', 6) try: output = call( resource=candidate, calc=[{ 'func': 'dissimilarity', 'name': 'spatial_analog', 'kwds': { 'dist': dist, 'target': target_ts, 'candidate': indices } }], time_range=[dateStartCandidate, dateEndCandidate], ) except Exception as ex: msg = 'Spatial analog failed: {}'.format(ex) LOGGER.exception(msg) raise Exception(msg) add_metadata(output, dist=dist, indices=",".join(indices), target_location=location, candidate_time_range="{},{}".format( dateStartCandidate, dateEndCandidate), target_time_range="{},{}".format(dateStartTarget, dateEndTarget)) response.update_status('Computed spatial analog', 95) response.outputs['output_netcdf'].file = output response.update_status('Execution completed', 100) LOGGER.debug("Total execution took {}".format(dt.now() - tic)) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' ncs = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) var = get_variable(ncs[0]) LOGGER.info('variable to be plotted: {}'.format(var)) # mosaic = self.mosaic.getValue() if 'region' in request.inputs: regions = [inp.data for inp in request.inputs['region']] try: png_region = vs.plot_polygons(regions) except Exception as ex: msg = 'failed to plot the polygon to world map: {}'.format( str(ex)) LOGGER.exception(msg) raise Exception(msg) o1, png_region = mkstemp(dir='.', suffix='.png') # clip the demanded polygons subsets = clipping( resource=ncs, variable=var, polygons=regions, mosaic=True, spatial_wrapping='wrap', ) else: subsets = ncs png_region = vs.plot_extend(ncs[0]) response.update_status('Arguments set for subset process', 0) try: tar_subsets = archive(subsets) except Exception as ex: msg = 'failed to archive subsets: {}'.format(ex) LOGGER.exception(msg) raise Exception(msg) _, tar_subsets = mkstemp(dir='.', suffix='.tar') try: png_uncertainty = vs.uncertainty(subsets, variable=var) except Exception as ex: msg = 'failed to generate the uncertainty plot: {}'.format(ex) LOGGER.exception(msg) raise Exception(msg) _, png_uncertainty = mkstemp(dir='.', suffix='.png') try: png_spaghetti = vs.spaghetti( subsets, variable=var, ) except Exception as ex: msg = 'failed to generate the spaghetti plot: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) _, png_spaghetti = mkstemp(dir='.', suffix='.png') try: from flyingpigeon import robustness as ro signal, low_agreement_mask, high_agreement_mask, text_src = ro.signal_noise_ratio( resource=subsets, # start=None, end=None, # timeslice=None, # variable=var ) # if title is None: title = 'signal robustness of %s ' % ( var) # , end1, end2, start1, start2 png_robustness = vs.map_robustness( signal, high_agreement_mask, low_agreement_mask, # cmap=cmap, # title=title ) LOGGER.info('robustness graphic generated') except Exception as ex: msg = 'failed to generate the robustness plot: {}'.format(ex) LOGGER.exception(msg) raise Exception(msg) _, png_robustness = mkstemp(dir='.', suffix='.png') factsheet = vs.factsheetbrewer(png_region=png_region, png_uncertainty=png_uncertainty, png_spaghetti=png_spaghetti, png_robustness=png_robustness) response.outputs['output_nc'].file = tar_subsets response.outputs['output_factsheet'].file = factsheet response.update_status("done", 100) return response
def execute(self): init_process_logger('log.txt') self.output_log.setValue('log.txt') from os.path import basename from flyingpigeon import sdm from flyingpigeon.utils import archive, archiveextract self.status.set('Start process', 0) try: logger.info('reading the arguments') resources = archiveextract( self.getInputValues(identifier='resources')) taxon_name = self.getInputValues(identifier='taxon_name')[0] bbox_obj = self.BBox.getValue() bbox = [ bbox_obj.coords[0][0], bbox_obj.coords[0][1], bbox_obj.coords[1][0], bbox_obj.coords[1][1] ] period = self.getInputValues(identifier='period') period = period[0] indices = self.getInputValues(identifier='input_indices') archive_format = self.archive_format.getValue() logger.debug("indices = %s for %s ", indices, taxon_name) logger.info("bbox={0}".format(bbox)) except Exception as e: logger.error('failed to read in the arguments %s ' % e) logger.info('indices %s ' % indices) try: self.status.set('Fetching GBIF Data', 10) gbifdic = sdm.get_gbif(taxon_name, bbox=bbox) except Exception as e: msg = 'failed to search gbif.' logger.exception(msg) raise Exception(msg) try: self.status.set('write csv file', 70) gbifcsv = sdm.gbifdic2csv(gbifdic) except Exception as e: msg = 'failed to write csv file.' logger.exception(msg) raise Exception(msg) try: self.status.set('plot map', 80) from flyingpigeon.visualisation import map_gbifoccurrences latlon = sdm.latlon_gbifdic(gbifdic) occurence_map = map_gbifoccurrences(latlon) except Exception as e: msg = 'failed to plot occurence map.' logger.exception(msg) raise Exception(msg) ################################# # calculate the climate indices ################################# # get the indices ncs_indices = None try: self.status.set( 'start calculation of climate indices for %s' % indices, 30) ncs_indices = sdm.get_indices(resources=resources, indices=indices) logger.info('indice calculation done') except: msg = 'failed to calculate indices' logger.exception(msg) raise Exception(msg) try: self.status.set('get domain', 30) domains = set() for resource in ncs_indices: # get_domain works only if metadata are set in a correct way domains = domains.union([basename(resource).split('_')[1]]) if len(domains) == 1: domain = list(domains)[0] logger.debug('Domain %s found in indices files' % domain) else: logger.error('Not a single domain in indices files %s' % domains) except Exception as e: logger.exception('failed to get domains %s' % e) try: self.status.set('generating the PA mask', 20) PAmask = sdm.get_PAmask(coordinates=latlon, domain=domain) logger.info('PA mask sucessfully generated') except Exception as e: logger.exception('failed to generate the PA mask: %s' % e) try: self.status.set('Ploting PA mask', 25) from flyingpigeon.visualisation import map_PAmask PAmask_png = map_PAmask(PAmask) except Exception as e: logger.exception('failed to plot the PA mask: %s' % e) try: # sort indices indices_dic = None indices_dic = sdm.sort_indices(ncs_indices) logger.info('indice files sorted for %s Datasets' % len(indices_dic.keys())) except: msg = 'failed to sort indices' logger.exception(msg) raise Exception(msg) ncs_references = [] species_files = [] stat_infos = [] for count, key in enumerate(indices_dic.keys()): try: staus_nr = 40 + count * 10 self.status.set('Start processing of %s' % key, staus_nr) ncs = indices_dic[key] logger.info('with %s files' % len(ncs)) try: ncs_reference = sdm.get_reference(ncs_indices=ncs, period=period) ncs_references.extend(ncs_reference) logger.info('reference indice calculated %s ' % ncs_references) except: msg = 'failed to calculate the reference' logger.exception(msg) raise Exception(msg) try: gam_model, predict_gam, gam_info = sdm.get_gam( ncs_reference, PAmask) stat_infos.append(gam_info) self.status.set('GAM sucessfully trained', staus_nr + 5) except Exception as e: msg = 'failed to train GAM for %s : %s' % (key, e) logger.debug(msg) try: prediction = sdm.get_prediction(gam_model, ncs) self.status.set('prediction done', staus_nr + 7) except Exception as e: msg = 'failed to predict tree occurence %s' % e logger.exception(msg) # raise Exception(msg) try: self.status.set('land sea mask for predicted data', staus_nr + 8) from numpy import invert, isnan, nan, broadcast_arrays # , array, zeros, linspace, meshgrid mask = invert(isnan(PAmask)) mask = broadcast_arrays(prediction, mask)[1] prediction[mask is False] = nan except Exception as e: logger.debug('failed to mask predicted data: %s' % e) try: species_files.append(sdm.write_to_file(ncs[0], prediction)) logger.info('Favourabillity written to file') except Exception as e: msg = 'failed to write species file %s' % e logger.debug(msg) # raise Exception(msg) except Exception as e: msg = 'failed to calculate reference indices. %s ' % e logger.exception(msg) raise Exception(msg) try: archive_indices = None archive_indices = archive(ncs_indices, format=archive_format) logger.info('indices added to archive') except: msg = 'failed adding indices to archive' logger.exception(msg) raise Exception(msg) archive_references = None try: archive_references = archive(ncs_references, format=archive_format) logger.info('indices reference added to archive') except: msg = 'failed adding reference indices to archive' logger.exception(msg) raise Exception(msg) archive_predicion = None try: archive_predicion = archive(species_files, format=archive_format) logger.info('species_files added to archive') except: msg = 'failed adding species_files indices to archive' logger.exception(msg) raise Exception(msg) try: from flyingpigeon.visualisation import pdfmerge stat_infosconcat = pdfmerge(stat_infos) logger.info('stat infos pdfs merged') except: logger.exception('failed to concat images') _, stat_infosconcat = tempfile.mkstemp(suffix='.pdf', prefix='foobar-', dir='.') self.output_csv.setValue(gbifcsv) self.output_gbif.setValue(occurence_map) self.output_PA.setValue(PAmask_png) self.output_indices.setValue(archive_indices) self.output_reference.setValue(archive_references) self.output_prediction.setValue(archive_predicion) self.output_info.setValue(stat_infosconcat) self.status.set('done', 100)
def execute(self): from os.path import basename from flyingpigeon import sdm from flyingpigeon.utils import archive, archiveextract # , get_domain from flyingpigeon.visualisation import map_PAmask init_process_logger('log.txt') self.output_log.setValue('log.txt') self.status.set('Start process', 0) try: self.status.set('reading the arguments', 5) resources = archiveextract( self.getInputValues(identifier='input_indices')) csv_file = self.getInputValues(identifier='gbif')[0] period = self.getInputValues(identifier='period') period = period[0] archive_format = self.archive_format.getValue() except: logger.error('failed to read in the arguments') try: self.status.set('read in latlon coordinates', 10) latlon = sdm.latlon_gbifcsv(csv_file) except: logger.exception('failed to extract the latlon points') try: self.status.set('plot map', 20) from flyingpigeon.visualisation import map_gbifoccurrences # latlon = sdm.latlon_gbifdic(gbifdic) occurence_map = map_gbifoccurrences(latlon) except: logger.exception('failed to plot occurence map') # try: # self.status.set('get domain', 30) # domains = set() # for indice in resources: # # get_domain works only if metadata are set in a correct way # domains = domains.union([basename(indice).split('_')[1]]) # if len(domains) == 1: # domain = list(domains)[0] # logger.info('Domain %s found in indices files' % domain) # else: # logger.warn('NOT a single domain in indices files %s' % domains) # except: # logger.exception('failed to get domains') try: # sort indices indices_dic = sdm.sort_indices(resources) logger.info('indice files sorted for %s Datasets' % len(indices_dic.keys())) except: msg = 'failed to sort indices' logger.exception(msg) raise Exception(msg) ncs_references = [] species_files = [] stat_infos = [] PAmask_pngs = [] self.status.set('Start processing for %s Datasets' % len(indices_dic.keys())) for count, key in enumerate(indices_dic.keys()): try: staus_nr = 40 + count * 10 self.status.set('Start processing of %s' % key, staus_nr) ncs = indices_dic[key] logger.info('with %s files' % len(ncs)) try: self.status.set('generating the PA mask', 20) PAmask = sdm.get_PAmask(coordinates=latlon, nc=ncs[0]) logger.info('PA mask sucessfully generated') except: logger.exception('failed to generate the PA mask') try: self.status.set('Ploting PA mask', 25) PAmask_pngs.extend([map_PAmask(PAmask)]) except: logger.exception('failed to plot the PA mask') try: ncs_reference = sdm.get_reference(ncs_indices=ncs, period=period) ncs_references.extend(ncs_reference) logger.info('reference indice calculated %s ' % ncs_references) self.status.set('reference indice calculated', staus_nr + 2) except: msg = 'failed to calculate the reference' logger.exception(msg) # raise Exception(msg) try: gam_model, predict_gam, gam_info = sdm.get_gam( ncs_reference, PAmask, modelname=key) stat_infos.append(gam_info) self.status.set('GAM sucessfully trained', staus_nr + 5) except: msg = 'failed to train GAM for %s' % (key) logger.exception(msg) try: prediction = sdm.get_prediction(gam_model, ncs) self.status.set('prediction done', staus_nr + 7) except: msg = 'failed to predict tree occurence' logger.exception(msg) # raise Exception(msg) # try: # self.status.set('land sea mask for predicted data', staus_nr + 8) # from numpy import invert, isnan, nan, broadcast_arrays # , array, zeros, linspace, meshgrid # mask = invert(isnan(PAmask)) # mask = broadcast_arrays(prediction, mask)[1] # prediction[mask is False] = nan # except: # logger.exception('failed to mask predicted data') try: species_files.append(sdm.write_to_file(ncs[0], prediction)) logger.info('Favourabillity written to file') except: msg = 'failed to write species file' logger.exception(msg) # raise Exception(msg) except: msg = 'failed to process SDM chain for %s ' % key logger.exception(msg) # raise Exception(msg) try: archive_references = None archive_references = archive(ncs_references, format=archive_format) logger.info('indices 2D added to archive') except: msg = 'failed adding 2D indices to archive' logger.exception(msg) raise Exception(msg) archive_predicion = None try: archive_predicion = archive(species_files, format=archive_format) logger.info('species_files added to archive') except: msg = 'failed adding species_files indices to archive' logger.exception(msg) raise Exception(msg) try: from flyingpigeon.visualisation import pdfmerge, concat_images stat_infosconcat = pdfmerge(stat_infos) logger.debug('pngs %s' % PAmask_pngs) PAmask_png = concat_images(PAmask_pngs, orientation='h') logger.info('stat infos pdfs and mask pngs merged') except: logger.exception('failed to concat images') _, stat_infosconcat = tempfile.mkstemp(suffix='.pdf', prefix='foobar-', dir='.') _, PAmask_png = tempfile.mkstemp(suffix='.png', prefix='foobar-', dir='.') self.output_gbif.setValue(occurence_map) self.output_PA.setValue(PAmask_png) self.output_reference.setValue(archive_references) self.output_prediction.setValue(archive_predicion) self.output_info.setValue(stat_infosconcat) self.status.set('done', 100)
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' process_start_time = time.time() # measure process execution time ... response.update_status( 'execution started at: {}'.format(str(dt.now())), 5) LOGGER.debug('starting segetalflora process execution') response.update_status('starting calcualtion segetalflora', 5) ############################ # read argments to variables ############################ try: resource = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) climate_type = request.inputs['climate_type'][0].data culture_type = request.inputs['culture_type'][0].data LOGGER.info('urls for {} ncs found'.format(len(resource))) LOGGER.info('culture type: {}'.format(culture_type)) except Exception as ex: msg = 'Failed to read in the arguments: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: if type(climate_type) != list: climate_type = list([climate_type]) if type(culture_type) != list: culture_type = list([culture_type]) LOGGER.info('arguments are lists') except Exception as ex: msg = 'Failed to transform arguments to lists: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) ############################# # get yearly mean temperature ############################# nc_tasmean = sf.get_yrmean(resource) ####################################### # main call for segetalflora processing ####################################### nc_sf = sf.get_segetalflora(resource=nc_tasmean, culture_type=culture_type, climate_type=climate_type) #################### # tar file archiving #################### try: response.update_status('preparing output', 99) LOGGER.debug('length of sf: {}'.format(len(nc_sf))) if len(nc_sf) == 1: # TODO: fix pywps output formats OR use separate output params. response.outputs['out_segetalflora'].file = nc_sf[0] response.outputs['out_segetalflora'].format = FORMATS.NETCDF else: response.outputs['out_segetalflora'].file = archive( nc_sf, format='tar', dir_output='.', mode='w') response.outputs['out_segetalflora'].format = Format( 'application/x-tar') if len(nc_tasmean) == 1: response.outputs['out_tasmean'].file = nc_tasmean[0] response.outputs['out_segetalflora'].format = FORMATS.NETCDF else: response.outputs['out_tasmean'].file = archive(nc_tasmean, format='tar', dir_output='.', mode='w') response.outputs['out_segetalflora'].format = Format( 'application/x-tar') except Exception as ex: msg = 'Failed to prepare output files: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) response.update_status('done', 100) LOGGER.debug( "total execution took {} seconds.".format(time.time() - process_start_time)) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' # input files LOGGER.debug("url={}, mime_type={}".format( request.inputs['resource'][0].url, request.inputs['resource'][0].data_format.mime_type)) ncs = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) # mime_type=request.inputs['resource'][0].data_format.mime_type) # mosaic option # TODO: fix defaults in pywps 4.x if 'mosaic' in request.inputs: mosaic = request.inputs['mosaic'][0].data else: mosaic = False # regions used for subsetting regions = [inp.data for inp in request.inputs['region']] LOGGER.info('ncs = {}'.format(ncs)) LOGGER.info('regions = {}'.format(regions)) LOGGER.info('mosaic = {}'.format(mosaic)) response.update_status("Arguments set for subset process", 0) LOGGER.debug('starting: regions=%s, num_files=%s', len(regions), len(ncs)) try: results = clipping( resource=ncs, polygons=regions, # self.region.getValue(), mosaic=mosaic, spatial_wrapping='wrap', # variable=variable, # dir_output=os.path.abspath(os.curdir), # dimension_map=dimension_map, ) LOGGER.info('results {}'.format(results)) except Exception as ex: msg = 'clipping failed: {}'.format(ex) LOGGER.exception(msg) raise Exception(msg) if not results: raise Exception('No results produced.') # prepare tar file try: tarf = archive(results) LOGGER.info('Tar file prepared') except Exception as ex: msg = 'Tar file preparation failed: {}'.format(ex) LOGGER.exception(msg) raise Exception(msg) response.outputs['output'].file = tarf i = next((i for i, x in enumerate(results) if x), None) response.outputs['ncout'].file = results[i] response.update_status("done", 100) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' response.update_status('Start process', 0) try: LOGGER.info('reading the arguments') resources = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) period = request.inputs['period'] period = period[0].data indices = [inpt.data for inpt in request.inputs['indices']] archive_format = request.inputs['archive_format'][0].data LOGGER.info( "all arguments read in nr of files in resources: {}".foirmat( len(resources))) except Exception as ex: msg = 'failed to read in the arguments: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: gbif_url = request.inputs['gbif'][0].data csv_file = download(gbif_url) LOGGER.info('CSV file fetched sucessfully: %s' % csv_file) except Exception as ex: msg = 'failed to fetch GBIF file: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: response.update_status('read in latlon coordinates', 10) latlon = sdm.latlon_gbifcsv(csv_file) LOGGER.info('got occurence coordinates %s ' % csv_file) except Exception as ex: msg = 'failed to extract the latlon points from file {}: {}'.format( csv_file, str(ex)) LOGGER.exception(msg) raise Exception(msg) try: response.update_status('plot map', 20) occurence_map = map_gbifoccurrences(latlon) LOGGER.info('GBIF occourence ploted') except Exception as ex: msg = 'failed to plot occurence map: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) ################################# # calculate the climate indices ################################# # get the indices try: response.update_status('start calculation of indices', 30) ncs_indices = sdm.get_indices(resource=resources, indices=indices) LOGGER.info('indice calculation done') except Exception as ex: msg = 'failed to calculate indices: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: # sort indices indices_dic = sdm.sort_indices(ncs_indices) LOGGER.info('indice files sorted in dictionary') except Exception as ex: msg = 'failed to sort indices: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) indices_dic = {'dummy': []} ncs_references = [] species_files = [] stat_infos = [] PAmask_pngs = [] response.update_status('Start processing for {} datasets'.format( len(indices_dic.keys()))) for count, key in enumerate(indices_dic.keys()): try: status_nr = 40 + count * 10 response.update_status('Start processing of {}'.format(key), status_nr) ncs = indices_dic[key] LOGGER.info('with {} files'.format(len(ncs))) try: response.update_status('generating the PA mask', 20) PAmask = sdm.get_PAmask(coordinates=latlon, nc=ncs[0]) LOGGER.info('PA mask sucessfully generated') except Exception as ex: msg = 'failed to generate the PA mask: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: response.update_status('Ploting PA mask', 25) PAmask_pngs.extend([map_PAmask(PAmask)]) except Exception as ex: msg = 'failed to plot the PA mask: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: ncs_reference = sdm.get_reference(ncs_indices=ncs, period=period) ncs_references.extend(ncs_reference) LOGGER.info('reference indice calculated {}'.format( ncs_references)) except Exception as ex: msg = 'failed to calculate the reference: {}'.format( str(ex)) LOGGER.exception(msg) raise Exception(msg) try: gam_model, predict_gam, gam_info = sdm.get_gam( ncs_reference, PAmask) stat_infos.append(gam_info) response.update_status('GAM sucessfully trained', status_nr + 5) except Exception as ex: msg = 'failed to train GAM for {}: {}'.format(key, str(ex)) LOGGER.debug(msg) raise Exception(msg) try: prediction = sdm.get_prediction(gam_model, ncs) response.update_status('prediction done', status_nr + 7) except Exception as ex: msg = 'failed to predict tree occurence: {}'.format( str(ex)) LOGGER.exception(msg) raise Exception(msg) # # try: # response.update_status('land sea mask for predicted data', status_nr + 8) # from numpy import invert, isnan, nan, broadcast_arrays # , array, zeros, linspace, meshgrid # mask = invert(isnan(PAmask)) # mask = broadcast_arrays(prediction, mask)[1] # prediction[mask is False] = nan # except: # LOGGER.debug('failed to mask predicted data') try: species_files.append(sdm.write_to_file(ncs[0], prediction)) LOGGER.info('Favourability written to file') except Exception as ex: msg = 'failed to write species file: {}'.format(str(ex)) LOGGER.debug(msg) raise Exception(msg) except Exception as ex: msg = 'failed to process SDM chain for {} : {}'.format( key, str(ex)) LOGGER.exception(msg) raise Exception(msg) try: archive_indices = archive(ncs_indices, format=archive_format) LOGGER.info('indices added to archive') except Exception as ex: msg = 'failed adding indices to archive: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) archive_indices = tempfile.mkstemp(suffix='.tar', prefix='foobar-', dir='.') try: archive_references = archive(ncs_references, format=archive_format) LOGGER.info('indices reference added to archive') except Exception as ex: msg = 'failed adding reference indices to archive: {}'.format( str(ex)) LOGGER.exception(msg) raise Exception(msg) archive_references = tempfile.mkstemp(suffix='.tar', prefix='foobar-', dir='.') try: archive_prediction = archive(species_files, format=archive_format) LOGGER.info('species_files added to archive') except Exception as ex: msg = 'failed adding species_files indices to archive: {}'.format( str(ex)) LOGGER.exception(msg) raise Exception(msg) try: stat_infosconcat = pdfmerge(stat_infos) LOGGER.debug('pngs {}'.format(PAmask_pngs)) PAmask_png = concat_images(PAmask_pngs, orientation='h') LOGGER.info('stat infos pdfs and mask pngs merged') except Exception as ex: msg = 'failed to concat images: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) _, stat_infosconcat = tempfile.mkstemp(suffix='.pdf', prefix='foobar-', dir='.') _, PAmask_png = tempfile.mkstemp(suffix='.png', prefix='foobar-', dir='.') # self.output_csv.setValue(csv_file) response.outputs['output_gbif'].file = occurence_map response.outputs['output_PA'].file = PAmask_png response.outputs['output_indices'].file = archive_indices response.outputs['output_reference'].file = archive_references response.outputs['output_prediction'].file = archive_prediction response.outputs['output_info'].file = stat_infosconcat response.update_status('done', 100) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' response.update_status('Start process', 0) try: LOGGER.info('reading the arguments') resources = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) taxon_name = request.inputs['taxon_name'][0].data bbox = [-180, -90, 180, 90] # bbox_obj = self.BBox.getValue() # bbox = [bbox_obj.coords[0][0], # bbox_obj.coords[0][1], # bbox_obj.coords[1][0], # bbox_obj.coords[1][1]] period = request.inputs['period'] period = period[0].data indices = [inpt.data for inpt in request.inputs['indices']] archive_format = request.inputs['archive_format'][0].data LOGGER.exception("indices = %s for %s ", indices, taxon_name) LOGGER.info("bbox={0}".format(bbox)) except: LOGGER.exception('failed to read in the arguments') LOGGER.info('indices %s ' % indices) try: response.update_status('Fetching GBIF Data', 10) gbifdic = sdm.get_gbif(taxon_name, bbox=bbox) LOGGER.info('Fetched GBIF data') except: msg = 'failed to search gbif.' LOGGER.exception(msg) raise Exception(msg) try: response.update_status('write csv file', 70) gbifcsv = sdm.gbifdic2csv(gbifdic) LOGGER.info('GBIF data written to file') except: msg = 'failed to write csv file.' LOGGER.exception(msg) raise Exception(msg) try: response.update_status('plot map', 80) latlon = sdm.latlon_gbifdic(gbifdic) occurence_map = map_gbifoccurrences(latlon) except: msg = 'failed to plot occurence map.' LOGGER.exception(msg) raise Exception(msg) ################################# # calculate the climate indices ################################# # get the indices ncs_indices = None try: response.update_status( 'start calculation of climate indices for %s' % indices, 30) ncs_indices = sdm.get_indices(resource=resources, indices=indices) LOGGER.info('indice calculation done') except: msg = 'failed to calculate indices' LOGGER.exception(msg) raise Exception(msg) try: # sort indices indices_dic = sdm.sort_indices(ncs_indices) LOGGER.info('indice files sorted in dictionary') except: msg = 'failed to sort indices' LOGGER.exception(msg) indices_dic = {'dummy': []} ncs_references = [] species_files = [] stat_infos = [] PAmask_pngs = [] response.update_status('Start processing for %s Datasets' % len(indices_dic.keys())) for count, key in enumerate(indices_dic.keys()): try: staus_nr = 40 + count * 10 response.update_status('Start processing of %s' % key, staus_nr) ncs = indices_dic[key] LOGGER.info('with %s files' % len(ncs)) try: response.update_status('generating the PA mask', 20) PAmask = sdm.get_PAmask(coordinates=latlon, nc=ncs[0]) LOGGER.info('PA mask sucessfully generated') except: LOGGER.exception('failed to generate the PA mask') try: response.update_status('Ploting PA mask', 25) PAmask_pngs.extend([map_PAmask(PAmask)]) except: LOGGER.exception('failed to plot the PA mask') try: ncs_reference = sdm.get_reference(ncs_indices=ncs, period=period) ncs_references.extend(ncs_reference) LOGGER.info('reference indice calculated %s ' % ncs_references) except: msg = 'failed to calculate the reference' LOGGER.exception(msg) try: gam_model, predict_gam, gam_info = sdm.get_gam( ncs_reference, PAmask) stat_infos.append(gam_info) response.update_status('GAM sucessfully trained', staus_nr + 5) except: msg = 'failed to train GAM for %s' % (key) LOGGER.exception(msg) try: prediction = sdm.get_prediction(gam_model, ncs) response.update_status('prediction done', staus_nr + 7) except: msg = 'failed to predict tree occurence' LOGGER.exception(msg) # raise Exception(msg) # try: # response.update_status('land sea mask for predicted data', staus_nr + 8) # from numpy import invert, isnan, nan, broadcast_arrays # , array, zeros, linspace, meshgrid # mask = invert(isnan(PAmask)) # mask = broadcast_arrays(prediction, mask)[1] # prediction[mask is False] = nan # except: # LOGGER.exception('failed to mask predicted data') try: species_files.append(sdm.write_to_file(ncs[0], prediction)) LOGGER.info('Favourabillity written to file') except: msg = 'failed to write species file' LOGGER.exception(msg) # raise Exception(msg) except: msg = 'failed to calculate reference indices' LOGGER.exception(msg) raise Exception(msg) try: archive_indices = archive(ncs_indices, format=archive_format) LOGGER.info('indices added to archive') except: msg = 'failed adding indices to archive' LOGGER.exception(msg) archive_indices = tempfile.mkstemp(suffix='.tar', prefix='foobar-', dir='.') try: archive_references = archive(ncs_references, format=archive_format) LOGGER.info('indices reference added to archive') except: msg = 'failed adding reference indices to archive' LOGGER.exception(msg) archive_references = tempfile.mkstemp(suffix='.tar', prefix='foobar-', dir='.') try: archive_prediction = archive(species_files, format=archive_format) LOGGER.info('species_files added to archive') except: msg = 'failed adding species_files indices to archive' LOGGER.exception(msg) raise Exception(msg) try: stat_infosconcat = pdfmerge(stat_infos) LOGGER.debug('pngs %s' % PAmask_pngs) PAmask_png = concat_images(PAmask_pngs, orientation='h') LOGGER.info('stat infos pdfs and mask pngs merged') except: LOGGER.exception('failed to concat images') _, stat_infosconcat = tempfile.mkstemp(suffix='.pdf', prefix='foobar-', dir='.') _, PAmask_png = tempfile.mkstemp(suffix='.png', prefix='foobar-', dir='.') response.outputs['output_gbif'].file = occurence_map response.outputs['output_PA'].file = PAmask_png response.outputs['output_indices'].file = archive_indices response.outputs['output_reference'].file = archive_references response.outputs['output_prediction'].file = archive_prediction response.outputs['output_info'].file = stat_infosconcat response.outputs['output_csv'].file = gbifcsv response.update_status('done', 100) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' try: resources = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) indices = [inpt.data for inpt in request.inputs['indices']] grouping = [inpt.data for inpt in request.inputs['grouping']] if 'mosaic' in request.inputs: mosaic = request.inputs['mosaic'][0].data else: mosaic = False if 'region' in request.inputs: region = [inpt.data for inpt in request.inputs['region']] else: region = None LOGGER.debug('grouping: {}'.format(grouping)) LOGGER.debug('mosaic: {}'.format(mosaic)) LOGGER.debug('indices: {}'.format(indices)) LOGGER.debug('region: {}'.format(region)) LOGGER.debug('Nr of input files: {}'.format(len(resources))) except Exception as ex: LOGGER.exception('failed to read in the arguments: {}'.format(str(ex))) response.update_status( 'starting: indices={}, grouping={}, num_files={}'.format(indices, grouping, len(resources)), 2) results = [] from flyingpigeon.utils import sort_by_filename datasets = sort_by_filename(resources, historical_concatination=True) results = [] try: group = grouping[0] # for group in grouping: indice = indices[0] # for indice in indices: for key in datasets.keys(): try: response.update_status('Dataset {}: {}'.format(len(results) + 1, key), 10) LOGGER.debug('grouping: {}'.format(grouping)) LOGGER.debug('mosaic: {}'.format(mosaic)) LOGGER.debug('indice: {}'.format(indice)) LOGGER.debug('region: {}'.format(region)) LOGGER.debug('Nr of input files: {}'.format(len(datasets[key]))) result = calc_indice_simple( resource=datasets[key], mosaic=mosaic, indice=indice, polygons=region, grouping=group, # dir_output=path.curdir, ) LOGGER.debug('result: {}'.format(result)) results.extend(result) except Exception as ex: msg = 'failed for {}: {}'.format(key, str(ex)) LOGGER.exception(msg) raise Exception(msg) except Exception as ex: msg = 'Failed to calculate indices: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) # # if not results: # # raise Exception("failed to produce results") # # response.update_status('num results %s' % len(results), 90) tarf = archive(results) response.outputs['output_archive'].file = tarf i = next((i for i, x in enumerate(results) if x), None) if i is None: i = 'dummy.nc' response.outputs['ncout'].file = results[i] # response.update_status("done", 100) return response
def _handler(self, request, response): response.update_status('starting uncertainty process', 0) init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' try: ncfiles = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) start = request.inputs['start'][0].data end = request.inputs['end'][0].data timeslice = request.inputs['timeslice'][0].data # # variable = self.variableIn.getValue() method = request.inputs['method'][0].data response.update_status('arguments read', 5) LOGGER.info('Successfully read in the arguments') except: LOGGER.exception("failed to read in the arguments") raise response.outputs['output_text'].file = write_fileinfo(ncfiles) # LOGGER.debug('variable set to %s' % variable) # if method == 'Method_A': signal, low_agreement_mask, high_agreement_mask, text_src = erob.method_A( resource=ncfiles, start=start, end=end, timeslice=timeslice, # variable=variable ) # graphic, LOGGER.debug('Robustness calculated') try: # LOGGER.info('variable to be plotted: %s' % variable) from flyingpigeon.visualisation import map_robustness # if title is None: title = 'signal robustness' # , end1, end2, start1, start2 graphic = map_robustness( signal, high_agreement_mask, low_agreement_mask, # variable=variable, # cmap=cmap, title=title) LOGGER.info('graphic generated') except: msg = 'graphic generation failed' LOGGER.exception(msg) _, graphic = mkstemp(dir='.', suffix='.png') response.update_status('process worker done', 95) response.outputs['output_signal'].file = signal response.outputs['output_high'].file = high_agreement_mask response.outputs['output_low'].file = low_agreement_mask response.outputs['output_graphic'].file = graphic # = text_src response.update_status('uncertainty process done', 100) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' try: resources = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) if 'region' in request.inputs: region = request.inputs['region'][0].data else: region = None if 'mosaic' in request.inputs: mosaic = request.inputs['mosaic'][0].data else: mosaic = False percentile = request.inputs['percentile'][0].data LOGGER.debug("mosaic %s " % mosaic) LOGGER.debug('percentile: %s' % percentile) LOGGER.debug('region %s' % region) LOGGER.debug('Nr of input files %s ' % len(resources)) except: LOGGER.exception('failed to read in the arguments') from flyingpigeon.utils import sort_by_filename from flyingpigeon.ocgis_module import call datasets = sort_by_filename(resources, historical_concatination=True) results = [] kwds = {'percentile': 90, 'window_width': 5} calc = [{'func': 'daily_perc', 'name': 'dp', 'kwds': kwds}] try: for key in datasets.keys(): try: if region is None: result = call( resource=datasets[key], output_format='nc', calc=calc, # prefix=key, # time_region={'year': [1995, 2000]} # calc_grouping='year' ) results.extend([result]) LOGGER.debug('percentile based indice done for %s' % result) else: result = clipping( resource=datasets[key], # variable=None, calc=calc, # calc_grouping=None, # time_range=None, # time_region=None, polygons=region, mosaic=mosaic) results.extend(result) except: LOGGER.exception( "failed to calculate percentil based indice for %s " % key) except: LOGGER.exception("failed to calculate percentile indices") tarf = archive(results) response.outputs['output_archive'].file = tarf i = next((i for i, x in enumerate(results) if x), None) if i is None: i = "dummy.nc" response.outputs['ncout'].file = results[i] # response.update_status("done", 100) response.update_status("done", 100) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' process_start_time = time.time() # measure process execution time ... response.update_status('execution started at : %s ' % dt.now(), 5) start_time = time.time() # measure init ... resource = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) refSt = request.inputs['refSt'][0].data refEn = request.inputs['refEn'][0].data dateSt = request.inputs['dateSt'][0].data dateEn = request.inputs['dateEn'][0].data regrset = request.inputs['regrset'][0].data # fix 31 December issue # refSt = dt.combine(refSt,dt_time(12,0)) # refEn = dt.combine(refEn,dt_time(12,0)) # dateSt = dt.combine(dateSt,dt_time(12,0)) # dateEn = dt.combine(dateEn,dt_time(12,0)) seasonwin = request.inputs['seasonwin'][0].data nanalog = request.inputs['nanalog'][0].data # bbox = [-80, 20, 50, 70] # TODO: Add checking for wrong cordinates and apply default if nesessary bbox = [] bboxStr = request.inputs['BBox'][0].data bboxStr = bboxStr.split(',') bbox.append(float(bboxStr[0])) bbox.append(float(bboxStr[2])) bbox.append(float(bboxStr[1])) bbox.append(float(bboxStr[3])) direction = request.inputs['direction'][0].data normalize = request.inputs['normalize'][0].data distance = request.inputs['dist'][0].data outformat = request.inputs['outformat'][0].data timewin = request.inputs['timewin'][0].data model_var = request.inputs['reanalyses'][0].data model, var = model_var.split('_') try: if direction == 're2mo': anaSt = dt.combine(dateSt, dt_time( 0, 0)) #dt.strptime(dateSt[0], '%Y-%m-%d') anaEn = dt.combine(dateEn, dt_time( 0, 0)) #dt.strptime(dateEn[0], '%Y-%m-%d') refSt = dt.combine(refSt, dt_time( 12, 0)) #dt.strptime(refSt[0], '%Y-%m-%d') refEn = dt.combine(refEn, dt_time( 12, 0)) #dt.strptime(refEn[0], '%Y-%m-%d') r_time_range = [anaSt, anaEn] m_time_range = [refSt, refEn] elif direction == 'mo2re': anaSt = dt.combine(dateSt, dt_time( 12, 0)) #dt.strptime(refSt[0], '%Y-%m-%d') anaEn = dt.combine(dateEn, dt_time( 12, 0)) #dt.strptime(refEn[0], '%Y-%m-%d') refSt = dt.combine(refSt, dt_time( 0, 0)) #dt.strptime(dateSt[0], '%Y-%m-%d') refEn = dt.combine(refEn, dt_time( 0, 0)) #dt.strptime(dateEn[0], '%Y-%m-%d') r_time_range = [refSt, refEn] m_time_range = [anaSt, anaEn] else: LOGGER.exception( 'failed to find time periods for comparison direction') except: msg = 'failed to put simulation and reference time in order' LOGGER.exception(msg) raise Exception(msg) if normalize == 'None': seacyc = False else: seacyc = True if outformat == 'ascii': outformat = '.txt' elif outformat == 'netCDF': outformat = '.nc' else: LOGGER.exception('output format not valid') try: if model == 'NCEP': getlevel = True if 'z' in var: level = var.strip('z') variable = 'hgt' # conform_units_to='hPa' else: variable = 'slp' level = None # conform_units_to='hPa' elif '20CRV2' in model: getlevel = False if 'z' in var: variable = 'hgt' level = var.strip('z') # conform_units_to=None else: variable = 'prmsl' level = None # conform_units_to='hPa' else: LOGGER.exception('Reanalyses model not known') LOGGER.info('environment set') except: msg = 'failed to set environment' LOGGER.exception(msg) raise Exception(msg) # LOGGER.exception("init took %s seconds.", time.time() - start_time) response.update_status('Read in the arguments', 6) ################# # get input data ################# # TODO: do not forget to select years start_time = time.time() # measure get_input_data ... response.update_status('fetching input data', 7) try: if direction == 're2mo': nc_reanalyses = reanalyses(start=anaSt.year, end=anaEn.year, variable=var, dataset=model, getlevel=getlevel) else: nc_reanalyses = reanalyses(start=refSt.year, end=refEn.year, variable=var, dataset=model, getlevel=getlevel) if type(nc_reanalyses) == list: nc_reanalyses = sorted( nc_reanalyses, key=lambda i: path.splitext(path.basename(i))[0]) else: nc_reanalyses = [nc_reanalyses] # For 20CRV2 geopotential height, daily dataset for 100 years is about 50 Gb # So it makes sense, to operate it step-by-step # TODO: need to create dictionary for such datasets (for models as well) # TODO: benchmark the method bellow for NCEP z500 for 60 years, may be use the same (!) # TODO Now everything regrid to the reanalysis if ('20CRV2' in model) and ('z' in var): tmp_total = [] origvar = get_variable(nc_reanalyses) for z in nc_reanalyses: tmp_n = 'tmp_%s' % (uuid.uuid1()) b0 = call(resource=z, variable=origvar, level_range=[int(level), int(level)], geom=bbox, spatial_wrapping='wrap', prefix='levdom_' + path.basename(z)[0:-3]) tmp_total.append(b0) tmp_total = sorted( tmp_total, key=lambda i: path.splitext(path.basename(i))[0]) inter_subset_tmp = call(resource=tmp_total, variable=origvar, time_range=r_time_range) # Clean for i in tmp_total: tbr = 'rm -f %s' % (i) #system(tbr) # Create new variable ds = Dataset(inter_subset_tmp, mode='a') z_var = ds.variables.pop(origvar) dims = z_var.dimensions new_var = ds.createVariable('z%s' % level, z_var.dtype, dimensions=(dims[0], dims[2], dims[3])) new_var[:, :, :] = squeeze(z_var[:, 0, :, :]) # new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()}) ds.close() nc_subset = call(inter_subset_tmp, variable='z%s' % level) else: nc_subset = call( resource=nc_reanalyses, variable=var, geom=bbox, spatial_wrapping='wrap', time_range=r_time_range, # conform_units_to=conform_units_to ) # nc_subset = call(resource=nc_reanalyses, variable=var, geom=bbox, spatial_wrapping='wrap') # XXXXXX wrap # LOGGER.exception("get_input_subset_model took %s seconds.", time.time() - start_time) response.update_status('**** Input reanalyses data fetched', 10) except: msg = 'failed to fetch or subset input files' LOGGER.exception(msg) raise Exception(msg) ######################## # input data preperation ######################## response.update_status('Start preparing input data', 12) # Filter resource: if type(resource) == list: resource = sorted(resource, key=lambda i: path.splitext(path.basename(i))[0]) else: resource = [resource] tmp_resource = [] m_start = m_time_range[0] m_end = m_time_range[1] for re in resource: s, e = get_timerange(re) tmpSt = dt.strptime(s, '%Y%m%d') tmpEn = dt.strptime(e, '%Y%m%d') if ((tmpSt <= m_end) and (tmpEn >= m_start)): tmp_resource.append(re) LOGGER.debug('Selected file: %s ' % (re)) resource = tmp_resource start_time = time.time() # mesure data preperation ... # TODO: Check the callendars ! for model vs reanalyses. # TODO: Check the units! model vs reanalyses. try: m_total = [] modvar = get_variable(resource) # resource properties ds = Dataset(resource[0]) m_var = ds.variables[modvar] dims = list(m_var.dimensions) dimlen = len(dims) try: model_id = ds.getncattr('model_id') except AttributeError: model_id = 'Unknown model' LOGGER.debug('MODEL: %s ' % (model_id)) lev_units = 'hPa' if (dimlen > 3): lev = ds.variables[dims[1]] # actually index [1] need to be detected... assuming zg(time, plev, lat, lon) lev_units = lev.units if (lev_units == 'Pa'): m_level = str(int(level) * 100) else: m_level = level else: m_level = None if level == None: level_range = None else: level_range = [int(m_level), int(m_level)] for z in resource: tmp_n = 'tmp_%s' % (uuid.uuid1()) # select level and regrid b0 = call( resource=z, variable=modvar, level_range=level_range, spatial_wrapping='wrap', #cdover='system', regrid_destination=nc_reanalyses[0], regrid_options='bil', prefix=tmp_n) # select domain b01 = call(resource=b0, geom=bbox, spatial_wrapping='wrap', prefix='levregr_' + path.basename(z)[0:-3]) tbr = 'rm -f %s' % (b0) #system(tbr) tbr = 'rm -f %s' % (tmp_n) #system(tbr) # get full resource m_total.append(b01) ds.close() model_subset = call(m_total, time_range=m_time_range) for i in m_total: tbr = 'rm -f %s' % (i) #system(tbr) if m_level is not None: # Create new variable in model set ds = Dataset(model_subset, mode='a') mod_var = ds.variables.pop(modvar) dims = mod_var.dimensions new_modvar = ds.createVariable('z%s' % level, mod_var.dtype, dimensions=(dims[0], dims[2], dims[3])) new_modvar[:, :, :] = squeeze(mod_var[:, 0, :, :]) # new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()}) ds.close() mod_subset = call(model_subset, variable='z%s' % level) else: mod_subset = model_subset # if direction == 're2mo': # try: # response.update_status('Preparing simulation data', 15) # reanalyses_subset = call(resource=nc_subset, time_range=[anaSt, anaEn]) # except: # msg = 'failed to prepare simulation period' # LOGGER.exception(msg) # try: # response.update_status('Preparing target data', 17) # var_target = get_variable(resource) # # var_simulation = get_variable(simulation) # model_subset_tmp = call(resource=resource, variable=var_target, # time_range=[refSt, refEn], # t_calendar='standard', # spatial_wrapping='wrap', # regrid_destination=nc_reanalyses[0], # regrid_options='bil') # # model_subset = call(resource=resource, variable=var_target, # # time_range=[refSt, refEn], # # geom=bbox, # # t_calendar='standard', # # # conform_units_to=conform_units_to, # # spatial_wrapping='wrap', # # regrid_destination=reanalyses_subset, # # regrid_options='bil') # XXXXXXXXXXXX ADD WRAP rem calendar # model_subset = call(resource=model_subset_tmp,variable=var_target, geom=bbox, spatial_wrapping='wrap', t_calendar='standard') # # ISSUE: the regrided model has white border with null! Check it. # # check t_calendar! # except: # msg = 'failed subset archive model' # LOGGER.exception(msg) # raise Exception(msg) # else: # try: # response.update_status('Preparing target data', 15) # var_target = get_variable(resource) # # var_simulation = get_variable(simulation) # model_subset = call(resource=resource, variable=var_target, # time_range=[refSt, refEn], # geom=bbox, # t_calendar='standard', # # conform_units_to=conform_units_to, # # spatial_wrapping='wrap', # ) # except: # msg = 'failed subset archive model' # LOGGER.exception(msg) # raise Exception(msg) # try: # response.update_status('Preparing simulation data', 17) # reanalyses_subset = call(resource=nc_subset, # time_range=[anaSt, anaEn], # regrid_destination=model_subset, # regrid_options='bil') # except: # msg = 'failed to prepare simulation period' # LOGGER.exception(msg) except: msg = 'failed to subset simulation or reference data' LOGGER.exception(msg) raise Exception(msg) # -------------------------------------------- try: if direction == 'mo2re': simulation = mod_subset archive = nc_subset base_id = model sim_id = model_id elif direction == 're2mo': simulation = nc_subset archive = mod_subset base_id = model_id sim_id = model else: LOGGER.exception('direction not valid: %s ' % direction) except: msg = 'failed to find comparison direction' LOGGER.exception(msg) raise Exception(msg) try: if level is not None: out_var = 'z%s' % level else: var_archive = get_variable(archive) var_simulation = get_variable(simulation) if var_archive != var_simulation: rename_variable(archive, oldname=var_archive, newname=var_simulation) out_var = var_simulation LOGGER.info('varname %s in netCDF renamed to %s' % (var_archive, var_simulation)) except: msg = 'failed to rename variable in target files' LOGGER.exception(msg) raise Exception(msg) try: if seacyc is True: seasoncyc_base, seasoncyc_sim = analogs.seacyc( archive, simulation, method=normalize) else: seasoncyc_base = None seasoncyc_sim = None except: msg = 'failed to prepare seasonal cycle reference files' LOGGER.exception(msg) raise Exception(msg) ip, output = mkstemp(dir='.', suffix='.txt') output_file = path.abspath(output) files = [path.abspath(archive), path.abspath(simulation), output_file] # LOGGER.exception("data preperation took %s seconds.", time.time() - start_time) ############################ # generating the config file ############################ response.update_status('writing config file', 18) start_time = time.time() # measure write config ... try: config_file = analogs.get_configfile( files=files, seasoncyc_base=seasoncyc_base, seasoncyc_sim=seasoncyc_sim, base_id=base_id, sim_id=sim_id, timewin=timewin, varname=var, seacyc=seacyc, cycsmooth=91, nanalog=nanalog, seasonwin=seasonwin, distfun=distance, outformat=outformat, calccor=True, silent=False, period=[ dt.strftime(refSt, '%Y-%m-%d'), dt.strftime(refEn, '%Y-%m-%d') ], bbox="%s,%s,%s,%s" % (bbox[0], bbox[2], bbox[1], bbox[3])) except: msg = 'failed to generate config file' LOGGER.exception(msg) raise Exception(msg) # LOGGER.exception("write_config took %s seconds.", time.time() - start_time) ####################### # CASTf90 call ####################### import subprocess import shlex start_time = time.time() # measure call castf90 response.update_status('Start CASTf90 call', 20) try: # response.update_status('execution of CASTf90', 50) cmd = 'analogue.out %s' % path.relpath(config_file) # system(cmd) args = shlex.split(cmd) output, error = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() LOGGER.info('analogue.out info:\n %s ' % output) LOGGER.exception('analogue.out errors:\n %s ' % error) response.update_status('**** CASTf90 suceeded', 90) except: msg = 'CASTf90 failed' LOGGER.exception(msg) raise Exception(msg) LOGGER.debug("castf90 took %s seconds.", time.time() - start_time) response.update_status('preparting output', 91) # Stopper to keep twitcher results, for debug # dummy=dummy response.outputs[ 'config'].file = config_file #config_output_url # config_file ) response.outputs['analogs'].file = output_file response.outputs['output_netcdf'].file = simulation response.outputs['target_netcdf'].file = archive ######################## # generate analog viewer ######################## formated_analogs_file = analogs.reformat_analogs(output_file) # response.outputs['formated_analogs'].storage = FileStorage() response.outputs['formated_analogs'].file = formated_analogs_file LOGGER.info('analogs reformated') response.update_status('reformatted analog file', 95) viewer_html = analogs.render_viewer( # configfile=response.outputs['config'].get_url(), configfile=config_file, # datafile=response.outputs['formated_analogs'].get_url()) datafile=formated_analogs_file) response.outputs['output'].file = viewer_html response.update_status('Successfully generated analogs viewer', 99) LOGGER.info('rendered pages: %s ', viewer_html) response.update_status('execution ended', 100) LOGGER.debug("total execution took %s seconds.", time.time() - process_start_time) return response