def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' response.update_status('Start process', 0) try: LOGGER.info('reading the arguments') resources = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) taxon_name = request.inputs['taxon_name'][0].data bbox = [-180, -90, 180, 90] # bbox_obj = self.BBox.getValue() # bbox = [bbox_obj.coords[0][0], # bbox_obj.coords[0][1], # bbox_obj.coords[1][0], # bbox_obj.coords[1][1]] period = request.inputs['period'] period = period[0].data indices = [inpt.data for inpt in request.inputs['indices']] archive_format = request.inputs['archive_format'][0].data LOGGER.exception("indices = {} for {}".format(indices, taxon_name)) LOGGER.info("bbox={}".format(bbox)) except Exception as ex: msg = 'failed to read in the arguments: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) LOGGER.info('indices {}'.format(indices)) try: response.update_status('Fetching GBIF Data', 10) gbifdic = sdm.get_gbif(taxon_name, bbox=bbox) LOGGER.info('Fetched GBIF data') except Exception as ex: msg = 'failed to search gbif: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: response.update_status('write csv file', 70) gbifcsv = sdm.gbifdic2csv(gbifdic) LOGGER.info('GBIF data written to file') except Exception as ex: msg = 'failed to write csv file: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: response.update_status('plot map', 80) latlon = sdm.latlon_gbifdic(gbifdic) occurence_map = map_gbifoccurrences(latlon) except Exception as ex: msg = 'failed to plot occurence map: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) ################################# # calculate the climate indices ################################# # get the indices ncs_indices = None try: response.update_status('start calculation of climate indices for {}'.format(indices), 30) ncs_indices = sdm.get_indices(resource=resources, indices=indices) LOGGER.info('indice calculation done') except Exception as ex: msg = 'failed to calculate indices: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: # sort indices indices_dic = sdm.sort_indices(ncs_indices) LOGGER.info('indice files sorted in dictionary') except Exception as ex: msg = 'failed to sort indices: {}'.format(str(ex)) LOGGER.exception(msg) indices_dic = {'dummy': []} ncs_references = [] species_files = [] stat_infos = [] PAmask_pngs = [] response.update_status('Start processing for {} Datasets'.format(len(indices_dic.keys()))) for count, key in enumerate(indices_dic.keys()): try: status_nr = 40 + count * 10 response.update_status('Start processing of {}'.format(key), status_nr) ncs = indices_dic[key] LOGGER.info('with {} files'.format(len(ncs))) try: response.update_status('generating the PA mask', 20) PAmask = sdm.get_PAmask(coordinates=latlon, nc=ncs[0]) LOGGER.info('PA mask successfully generated') except Exception as ex: msg = 'failed to generate the PA mask: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: response.update_status('Plotting PA mask', 25) PAmask_pngs.extend([map_PAmask(PAmask)]) except Exception as ex: msg = 'failed to plot the PA mask: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: ncs_reference = sdm.get_reference(ncs_indices=ncs, period=period) ncs_references.extend(ncs_reference) LOGGER.info('reference indice calculated %s ' % ncs_references) except Exception as ex: msg = 'failed to calculate the reference: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: gam_model, predict_gam, gam_info = sdm.get_gam(ncs_reference, PAmask) stat_infos.append(gam_info) response.update_status('GAM sucessfully trained', status_nr + 5) except Exception as ex: msg = 'failed to train GAM for {}: {}'.format(key, str(ex)) LOGGER.exception(msg) raise Exception(msg) try: prediction = sdm.get_prediction(gam_model, ncs) response.update_status('prediction done', status_nr + 7) except Exception as ex: msg = 'failed to predict tree occurrence: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) # try: # response.update_status('land sea mask for predicted data', status_nr + 8) # from numpy import invert, isnan, nan, broadcast_arrays # , array, zeros, linspace, meshgrid # mask = invert(isnan(PAmask)) # mask = broadcast_arrays(prediction, mask)[1] # prediction[mask is False] = nan # except: # LOGGER.exception('failed to mask predicted data') try: species_files.append(sdm.write_to_file(ncs[0], prediction)) LOGGER.info('Favourability written to file') except Exception as ex: msg = 'failed to write species file: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) except Exception as ex: msg = 'failed to calculate reference indices: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: archive_indices = archive(ncs_indices, format=archive_format) LOGGER.info('indices added to archive') except Exception as ex: msg = 'failed adding indices to archive: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) archive_indices = tempfile.mkstemp(suffix='.tar', prefix='foobar-', dir='.') try: archive_references = archive(ncs_references, format=archive_format) LOGGER.info('indices reference added to archive') except Exception as ex: msg = 'failed adding reference indices to archive: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) archive_references = tempfile.mkstemp(suffix='.tar', prefix='foobar-', dir='.') try: archive_prediction = archive(species_files, format=archive_format) LOGGER.info('species_files added to archive') except Exception as ex: msg = 'failed adding species_files indices to archive: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: stat_infosconcat = pdfmerge(stat_infos) LOGGER.debug('pngs {}'.format(PAmask_pngs)) PAmask_png = concat_images(PAmask_pngs, orientation='h') LOGGER.info('stat info pdfs and mask pngs merged') except Exception as ex: msg = 'failed to concat images: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) _, stat_infosconcat = tempfile.mkstemp(suffix='.pdf', prefix='foobar-', dir='.') _, PAmask_png = tempfile.mkstemp(suffix='.png', prefix='foobar-', dir='.') response.outputs['output_gbif'].file = occurence_map response.outputs['output_PA'].file = PAmask_png response.outputs['output_indices'].file = archive_indices response.outputs['output_reference'].file = archive_references response.outputs['output_prediction'].file = archive_prediction response.outputs['output_info'].file = stat_infosconcat response.outputs['output_csv'].file = gbifcsv response.update_status('done', 100) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' response.update_status('Start process', 0) try: response.update_status('reading the arguments', 5) resources = archiveextract( resource=rename_complexinputs(request.inputs['resources'])) period = request.inputs['period'] period = period[0].data archive_format = request.inputs['archive_format'][0].data LOGGER.info("all arguments read in nr of files in resources: %s" % len(resources)) except: LOGGER.exception('failed to read in the arguments') try: gbif_url = request.inputs['gbif'][0].data csv_file = download(gbif_url) LOGGER.info('CSV file fetched sucessfully: %s' % csv_file) except: LOGGER.exception('failed to fetch GBIF file') try: response.update_status('read in latlon coordinates', 10) latlon = sdm.latlon_gbifcsv(csv_file) LOGGER.info('read in the latlon coordinates') except: LOGGER.exception('failed to extract the latlon points') try: response.update_status('plot map', 20) occurence_map = map_gbifoccurrences(latlon) LOGGER.info('GBIF occourence ploted') except: LOGGER.exception('failed to plot occurence map') try: # sort indices indices_dic = sdm.sort_indices(resources) LOGGER.info('indice files sorted in dictionary') except: msg = 'failed to sort indices' LOGGER.exception(msg) indices_dic = {'dummy': []} ncs_references = [] species_files = [] stat_infos = [] PAmask_pngs = [] response.update_status('Start processing for %s Datasets' % len(indices_dic.keys())) for count, key in enumerate(indices_dic.keys()): try: staus_nr = 40 + count * 10 response.update_status('Start processing of %s' % key, staus_nr) ncs = indices_dic[key] LOGGER.info('with %s files' % len(ncs)) try: response.update_status('generating the PA mask', 20) PAmask = sdm.get_PAmask(coordinates=latlon, nc=ncs[0]) LOGGER.info('PA mask sucessfully generated') except: LOGGER.exception('failed to generate the PA mask') try: response.update_status('Ploting PA mask', 25) PAmask_pngs.extend([map_PAmask(PAmask)]) except: LOGGER.exception('failed to plot the PA mask') try: ncs_reference = sdm.get_reference(ncs_indices=ncs, period=period) ncs_references.extend(ncs_reference) LOGGER.info('reference indice calculated %s ' % ncs_references) response.update_status('reference indice calculated', staus_nr + 2) except: msg = 'failed to calculate the reference' LOGGER.exception(msg) # raise Exception(msg) try: gam_model, predict_gam, gam_info = sdm.get_gam( ncs_reference, PAmask, modelname=key) stat_infos.append(gam_info) response.update_status('GAM sucessfully trained', staus_nr + 5) except: msg = 'failed to train GAM for %s' % (key) LOGGER.exception(msg) try: prediction = sdm.get_prediction(gam_model, ncs) response.update_status('prediction done', staus_nr + 7) except: msg = 'failed to predict tree occurence' LOGGER.exception(msg) # raise Exception(msg) # try: # response.update_status('land sea mask for predicted data', staus_nr + 8) # from numpy import invert, isnan, nan, broadcast_arrays # , array, zeros, linspace, meshgrid # mask = invert(isnan(PAmask)) # mask = broadcast_arrays(prediction, mask)[1] # prediction[mask is False] = nan # except: # LOGGER.exception('failed to mask predicted data') try: species_files.append(sdm.write_to_file(ncs[0], prediction)) LOGGER.info('Favourabillity written to file') except: msg = 'failed to write species file' LOGGER.exception(msg) # raise Exception(msg) except: msg = 'failed to process SDM chain for %s ' % key LOGGER.exception(msg) # raise Exception(msg) try: archive_references = archive(ncs_references, format=archive_format) LOGGER.info('indices 2D added to archive') except: msg = 'failed adding 2D indices to archive' LOGGER.exception(msg) archive_references = tempfile.mkstemp(suffix='.tar', prefix='foobar-', dir='.') try: archive_prediction = archive(species_files, format=archive_format) LOGGER.info('species_files added to archive') except: msg = 'failed adding species_files indices to archive' LOGGER.exception(msg) archive_predicion = tempfile.mkstemp(suffix='.tar', prefix='foobar-', dir='.') try: from flyingpigeon.visualisation import pdfmerge, concat_images stat_infosconcat = pdfmerge(stat_infos) LOGGER.debug('pngs %s' % PAmask_pngs) PAmask_png = concat_images(PAmask_pngs, orientation='h') LOGGER.info('stat infos pdfs and mask pngs merged') except: LOGGER.exception('failed to concat images') _, stat_infosconcat = tempfile.mkstemp(suffix='.pdf', prefix='foobar-', dir='.') _, PAmask_png = tempfile.mkstemp(suffix='.png', prefix='foobar-', dir='.') response.outputs['output_gbif'].file = occurence_map response.outputs['output_PA'].file = PAmask_png response.outputs['output_reference'].file = archive_references response.outputs['output_prediction'].file = archive_prediction response.outputs['output_info'].file = stat_infosconcat response.update_status('done', 100) return response
def _handler(self, request, response): tic = dt.now() init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' LOGGER.info('Start process') response.update_status('Execution started at : {}'.format(tic), 1) ###################################### # Read inputs ###################################### try: candidate = archiveextract( resource=rename_complexinputs(request.inputs['candidate'])) target = archiveextract( resource=rename_complexinputs(request.inputs['target'])) location = request.inputs['location'][0].data indices = [el.data for el in request.inputs['indices']] dist = request.inputs['dist'][0].data dateStartCandidate = request.inputs['dateStartCandidate'][0].data dateEndCandidate = request.inputs['dateEndCandidate'][0].data dateStartTarget = request.inputs['dateStartTarget'][0].data dateEndTarget = request.inputs['dateEndTarget'][0].data except Exception as e: msg = 'Failed to read input parameter {}'.format(e) LOGGER.error(msg) raise Exception(msg) response.update_status('Input parameters ingested', 2) ###################################### # Process inputs ###################################### try: point = Point(*map(float, location.split(','))) dateStartCandidate = dt.strptime(dateStartCandidate, '%Y-%m-%d') dateEndCandidate = dt.strptime(dateEndCandidate, '%Y-%m-%d') dateStartTarget = dt.strptime(dateStartTarget, '%Y-%m-%d') dateEndTarget = dt.strptime(dateEndTarget, '%Y-%m-%d') except Exception as e: msg = 'failed to process inputs {} '.format(e) LOGGER.error(msg) raise Exception(msg) LOGGER.debug("init took {}".format(dt.now() - tic)) response.update_status('Processed input parameters', 3) ###################################### # Extract target time series ###################################### savetarget = False try: # Using `call` creates a netCDF file in the tmp directory. # # Here we keep this stuff in memory if savetarget: prefix = 'target_ts' target_ts = call(resource=target, geom=point, variable=indices, time_range=[dateStartTarget, dateEndTarget], select_nearest=True, prefix=prefix) #target_ts = [get_values(prefix+'.nc', ind) for ind in indices] else: trd = RequestDataset( target, variable=indices, time_range=[dateStartTarget, dateEndTarget]) op = OcgOperations(trd, geom=point, select_nearest=True, search_radius_mult=1.75) out = op.execute() target_ts = out.get_element() except Exception as e: msg = 'Target extraction failed {}'.format(e) LOGGER.debug(msg) raise Exception(msg) response.update_status('Extracted target series', 5) ###################################### # Compute dissimilarity metric ###################################### response.update_status('Computing spatial analog', 6) try: output = call( resource=candidate, calc=[{ 'func': 'dissimilarity', 'name': 'spatial_analog', 'kwds': { 'dist': dist, 'target': target_ts, 'candidate': indices } }], time_range=[dateStartCandidate, dateEndCandidate], ) except Exception as e: msg = 'Spatial analog failed: {}'.format(e) LOGGER.exception(msg) raise Exception(msg) add_metadata(output, dist=dist, indices=",".join(indices), target_location=location, candidate_time_range="{},{}".format( dateStartCandidate, dateEndCandidate), target_time_range="{},{}".format(dateStartTarget, dateEndTarget)) response.update_status('Computed spatial analog', 95) response.outputs['output_netcdf'].file = output response.update_status('Execution completed', 100) LOGGER.debug("Total execution took {}".format(dt.now() - tic)) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' response.update_status('execution started at : {}'.format(dt.now()), 5) ################################ # reading in the input arguments ################################ try: LOGGER.info('read in the arguments') # resources = self.getInputValues(identifier='resources') season = request.inputs['season'][0].data LOGGER.info('season %s', season) period = request.inputs['period'][0].data LOGGER.info('period %s', period) anualcycle = request.inputs['anualcycle'][0].data start = dt.strptime(period.split('-')[0], '%Y%m%d') end = dt.strptime(period.split('-')[1], '%Y%m%d') LOGGER.debug('start: %s , end: %s ' % (start, end)) resource = archiveextract(resource=rename_complexinputs(request.inputs['resource'])) # resource = archiveextract(resource=[res.file for res in request.inputs['resource']]) url_Rdat = request.inputs['Rdat'][0].data url_dat = request.inputs['dat'][0].data url_ref_file = request.inputs['netCDF'][0].data # can be None # season = self.getInputValues(identifier='season')[0] # period = self.getInputValues(identifier='period')[0] # anualcycle = self.getInputValues(identifier='anualcycle')[0] LOGGER.info('period %s' % str(period)) LOGGER.info('season %s' % str(season)) LOGGER.info('read in the arguments') LOGGER.info('url_ref_file: %s' % url_ref_file) LOGGER.info('url_Rdat: %s' % url_Rdat) LOGGER.info('url_dat: %s' % url_dat) except Exception as e: LOGGER.debug('failed to convert arguments %s ' % e) ############################ # fetching trainging data ############################ try: dat = abspath(download(url_dat)) Rdat = abspath(download(url_Rdat)) LOGGER.info('training data fetched') except Exception as e: LOGGER.error('failed to fetch training data %s' % e) ########################################################## # get the required bbox and time region from resource data ########################################################## # from flyingpigeon.weatherregimes import get_level try: from flyingpigeon.ocgis_module import call from flyingpigeon.utils import get_variable time_range = [start, end] variable = get_variable(resource) if len(url_ref_file) > 0: ref_file = download(url_ref_file) model_subset = call( resource=resource, variable=variable, time_range=time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap', regrid_destination=ref_file, regrid_options='bil') LOGGER.info('Dataset subset with regridding done: %s ' % model_subset) else: model_subset = call( resource=resource, variable=variable, time_range=time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap', ) LOGGER.info('Dataset time period extracted: %s ' % model_subset) except: LOGGER.exception('failed to make a data subset ') ####################### # computing anomalies ####################### try: cycst = anualcycle.split('-')[0] cycen = anualcycle.split('-')[0] reference = [dt.strptime(cycst, '%Y%m%d'), dt.strptime(cycen, '%Y%m%d')] model_anomal = wr.get_anomalies(model_subset, reference=reference) ##################### # extracting season ##################### model_season = wr.get_season(model_anomal, season=season) except: LOGGER.exception('failed to compute anualcycle or seasons') ####################### # call the R scripts ####################### import shlex import subprocess from flyingpigeon import config from os.path import curdir, exists, join try: rworkspace = curdir Rsrc = config.Rsrc_dir() Rfile = 'weatherregimes_projection.R' yr1 = start.year yr2 = end.year time = get_time(model_season, format='%Y%m%d') # ip, output_graphics = mkstemp(dir=curdir ,suffix='.pdf') ip, file_pca = mkstemp(dir=curdir, suffix='.txt') ip, file_class = mkstemp(dir=curdir, suffix='.Rdat') ip, output_frec = mkstemp(dir=curdir, suffix='.txt') args = ['Rscript', join(Rsrc, Rfile), '%s/' % curdir, '%s/' % Rsrc, '%s' % model_season, '%s' % variable, '%s' % str(time).strip("[]").replace("'", "").replace(" ", ""), # '%s' % output_graphics, '%s' % dat, '%s' % Rdat, '%s' % file_pca, '%s' % file_class, '%s' % output_frec, '%s' % season, '%s' % start.year, '%s' % end.year, '%s' % 'MODEL'] LOGGER.info('Rcall builded') except Exception as e: msg = 'failed to build the R command %s' % e LOGGER.error(msg) raise Exception(msg) try: output, error = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() # , shell=True LOGGER.info('R outlog info:\n %s ' % output) LOGGER.debug('R outlog errors:\n %s ' % error) if len(output) > 0: response.update_status('**** weatherregime in R suceeded', 90) else: LOGGER.error('NO! output returned from R call') except Exception as e: msg = 'weatherregime in R %s ' % e LOGGER.error(msg) raise Exception(msg) ################# # set the outputs ################# response.update_status('Set the process outputs ', 95) response.outputs['output_pca'].file = file_pca response.outputs['output_classification'].file = file_class response.outputs['output_netcdf'].file = model_season response.outputs['output_frequency'].file = output_frec response.update_status('done', 100) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' process_start_time = time.time() # measure process execution time ... response.update_status('execution started at : %s ' % dt.now(), 5) LOGGER.debug('starting segetalflora process execution') response.update_status('starting calcualtion segetalflora', 5) ############################ # read argments to variables ############################ try: resource = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) climate_type = request.inputs['climate_type'][0].data culture_type = request.inputs['culture_type'][0].data LOGGER.info('urls for %s ncs found' % (len(resource))) LOGGER.info('culture type: %s ' % (culture_type)) except Exception: msg = 'Failed to read in the arguments.' LOGGER.exception(msg) raise Exception(msg) try: if type(climate_type) != list: climate_type = list([climate_type]) if type(culture_type) != list: culture_type = list([culture_type]) LOGGER.info('arguments are lists') except Exception: msg = 'Failed to transform arguments to lists.' LOGGER.exception(msg) raise Exception(msg) ############################# # get yearly mean temperature ############################# nc_tasmean = sf.get_yrmean(resource) ####################################### # main call for segetalflora processing ####################################### nc_sf = sf.get_segetalflora(resource=nc_tasmean, culture_type=culture_type, climate_type=climate_type) #################### # tar file archiving #################### try: response.update_status('preparting output', 99) LOGGER.debug('length of sf: %s', len(nc_sf)) if len(nc_sf) == 1: # TODO: fix pywps output formats OR use seperate output params. response.outputs['out_segetalflora'].file = nc_sf[0] response.outputs['out_segetalflora'].format = FORMATS.NETCDF else: response.outputs['out_segetalflora'].file = archive( nc_sf, format='tar', dir_output='.', mode='w') response.outputs['out_segetalflora'].format = Format( 'application/x-tar') if len(nc_tasmean) == 1: response.outputs['out_tasmean'].file = nc_tasmean[0] response.outputs['out_segetalflora'].format = FORMATS.NETCDF else: response.outputs['out_tasmean'].file = archive(nc_tasmean, format='tar', dir_output='.', mode='w') response.outputs['out_segetalflora'].format = Format( 'application/x-tar') except Exception: msg = 'Failed to prepare output files.' LOGGER.exception(msg) raise Exception(msg) response.update_status('done', 100) LOGGER.debug("total execution took %s seconds.", time.time() - process_start_time) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' # input files LOGGER.debug("url=%s, mime_type=%s", request.inputs['resource'][0].url, request.inputs['resource'][0].data_format.mime_type) ncs = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) # mime_type=request.inputs['resource'][0].data_format.mime_type) # mosaic option # TODO: fix defaults in pywps 4.x if 'mosaic' in request.inputs: mosaic = request.inputs['mosaic'][0].data else: mosaic = False # regions used for subsetting regions = [inp.data for inp in request.inputs['region']] LOGGER.info('ncs = %s', ncs) LOGGER.info('regions = %s', regions) LOGGER.info('mosaic = %s', mosaic) response.update_status("Arguments set for subset process", 0) LOGGER.debug('starting: regions=%s, num_files=%s', len(regions), len(ncs)) try: results = clipping( resource=ncs, polygons=regions, # self.region.getValue(), mosaic=mosaic, spatial_wrapping='wrap', # variable=variable, # dir_output=os.path.abspath(os.curdir), # dimension_map=dimension_map, ) LOGGER.info('results %s' % results) except: msg = 'clipping failed' LOGGER.exception(msg) raise Exception(msg) if not results: raise Exception('no results produced.') # prepare tar file try: tarf = archive(results) LOGGER.info('Tar file prepared') except: msg = 'Tar file preparation failed' LOGGER.exception(msg) raise Exception(msg) response.outputs['output'].file = tarf i = next((i for i, x in enumerate(results) if x), None) response.outputs['ncout'].file = results[i] response.update_status("done", 100) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' try: resources = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) # indices = request.inputs['indices'][0].data grouping = request.inputs['grouping'][0].data # grouping = [inpt.data for inpt in request.inputs['grouping']] if 'region' in request.inputs: region = request.inputs['region'][0].data else: region = None if 'mosaic' in request.inputs: mosaic = request.inputs['mosaic'][0].data else: mosaic = False percentile = request.inputs['percentile'][0].data # refperiod = request.inputs['refperiod'][0].data from datetime import datetime as dt # # if refperiod is not None: # start = dt.strptime(refperiod.split('-')[0], '%Y%m%d') # end = dt.strptime(refperiod.split('-')[1], '%Y%m%d') # refperiod = [start, end] # response.update_status('starting: indices=%s, grouping=%s, num_files=%s' # % (indices, grouping, len(resources)), 2) LOGGER.debug("grouping %s " % grouping) LOGGER.debug("mosaic %s " % mosaic) # LOGGER.debug("refperiod set to %s, %s " % (start, end)) # LOGGER.debug('indices= %s ' % indices) LOGGER.debug('percentile: %s' % percentile) LOGGER.debug('region %s' % region) LOGGER.debug('Nr of input files %s ' % len(resources)) except Exception as e: LOGGER.exception("failed to read in the arguments: %s" % e) from flyingpigeon.utils import sort_by_filename from flyingpigeon.ocgis_module import call datasets = sort_by_filename(resources, historical_concatination=True) results = [] kwds = {'percentile': percentile, 'window_width': 5} calc = [{'func': 'daily_perc', 'name': 'dp', 'kwds': kwds}] # # ops = OcgOperations(dataset=rd, calc=calc, # output_format='nc', # time_region={'year': [1980, 1990]} # ).execute() try: for key in datasets.keys(): try: result = calc( resource=datasets[key], calc=calc, # calc_grouping='year' ) LOGGER.debug('percentile based indice done for %s' % result) results.extend(result) except Exception as e: LOGGER.exception( "failed to calculate percentile based indice for %s: %s" % key, e) except Exception as e: LOGGER.exception("failed to calculate percentile indices: %s" % e) output_archive = archive(results) response.outputs['output_archive'].file = output_archive i = next((i for i, x in enumerate(results) if x), None) if i is None: i = "dummy.nc" response.outputs['ncout'].file = results[i] response.update_status("done", 100) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' LOGGER.info('Start process') response.update_status('execution started at : {}'.format(dt.now()), 5) process_start_time = time.time() # measure process execution time ... start_time = time.time() # measure init ... ################################ # reading in the input arguments ################################ response.update_status('execution started at : %s ' % dt.now(), 5) start_time = time.time() # measure init ... ################################ # reading in the input arguments ################################ try: response.update_status('read input parameter : %s ' % dt.now(), 5) resource = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) refSt = request.inputs['refSt'][0].data refEn = request.inputs['refEn'][0].data dateSt = request.inputs['dataSt'][0].data dateEn = request.inputs['dataEn'][0].data seasonwin = request.inputs['seasonwin'][0].data nanalog = request.inputs['nanalog'][0].data bbox = [-80, 20, 50, 70] # if bbox_obj is not None: # LOGGER.info("bbox_obj={0}".format(bbox_obj.coords)) # bbox = [bbox_obj.coords[0][0], # bbox_obj.coords[0][1], # bbox_obj.coords[1][0], # bbox_obj.coords[1][1]] # LOGGER.info("bbox={0}".format(bbox)) # else: # bbox = None # region = self.getInputValues(identifier='region')[0] # bbox = [float(b) for b in region.split(',')] # bbox_obj = self.BBox.getValue() normalize = request.inputs['normalize'][0].data distance = request.inputs['dist'][0].data outformat = request.inputs['outformat'][0].data timewin = request.inputs['timewin'][0].data model_var = request.inputs['reanalyses'][0].data model, var = model_var.split('_') # experiment = self.getInputValues(identifier='experiment')[0] # dataset, var = experiment.split('_') # LOGGER.info('environment set') LOGGER.info('input parameters set') response.update_status('Read in and convert the arguments', 5) except Exception as e: msg = 'failed to read input prameter %s ' % e LOGGER.error(msg) raise Exception(msg) ###################################### # convert types and set environment ###################################### try: refSt = dt.strptime(refSt[0], '%Y-%m-%d') refEn = dt.strptime(refEn[0], '%Y-%m-%d') dateSt = dt.strptime(dateSt[0], '%Y-%m-%d') dateEn = dt.strptime(dateEn[0], '%Y-%m-%d') if normalize == 'None': seacyc = False else: seacyc = True if outformat == 'ascii': outformat = '.txt' elif outformat == 'netCDF': outformat = '.nc' else: LOGGER.error('output format not valid') start = min(refSt, dateSt) end = max(refEn, dateEn) if bbox_obj is not None: LOGGER.info("bbox_obj={0}".format(bbox_obj.coords)) bbox = [ bbox_obj.coords[0][0], bbox_obj.coords[0][1], bbox_obj.coords[1][0], bbox_obj.coords[1][1] ] LOGGER.info("bbox={0}".format(bbox)) else: bbox = None LOGGER.info('environment set') except Exception as e: msg = 'failed to set environment %s ' % e LOGGER.error(msg) raise Exception(msg) LOGGER.debug("init took %s seconds.", time.time() - start_time) response.update_status('Read in and convert the arguments', 5) ######################## # input data preperation ######################## # TODO: Check if files containing more than one dataset response.update_status('Start preparing input data', 12) start_time = time.time() # mesure data preperation ... try: variable = get_variable(resource) archive = call(resource=resource, time_range=[refSt, refEn], geom=bbox, spatial_wrapping='wrap') simulation = call(resource=resource, time_range=[dateSt, dateEn], geom=bbox, spatial_wrapping='wrap') if seacyc is True: seasoncyc_base, seasoncyc_sim = analogs.seacyc( archive, simulation, method=normalize) else: seasoncyc_base = None seasoncyc_sim = None except Exception as e: msg = 'failed to prepare archive and simulation files %s ' % e LOGGER.debug(msg) raise Exception(msg) ip, output = mkstemp(dir='.', suffix='.txt') output_file = path.abspath(output) files = [path.abspath(archive), path.abspath(simulation), output_file] LOGGER.debug("data preperation took %s seconds.", time.time() - start_time) ############################ # generating the config file ############################ response.update_status('writing config file', 15) start_time = time.time() # measure write config ... try: config_file = analogs.get_configfile( files=files, seasoncyc_base=seasoncyc_base, seasoncyc_sim=seasoncyc_sim, timewin=timewin, varname=variable, seacyc=seacyc, cycsmooth=91, nanalog=nanalog, seasonwin=seasonwin, distfun=distance, outformat=outformat, calccor=True, silent=False, period=[ dt.strftime(refSt, '%Y-%m-%d'), dt.strftime(refEn, '%Y-%m-%d') ], bbox="%s,%s,%s,%s" % (bbox[0], bbox[2], bbox[1], bbox[3])) except Exception as e: msg = 'failed to generate config file %s ' % e LOGGER.debug(msg) raise Exception(msg) LOGGER.debug("write_config took %s seconds.", time.time() - start_time) ############## # CASTf90 call ############## import subprocess import shlex start_time = time.time() # measure call castf90 response.update_status('Start CASTf90 call', 20) try: # response.update_status('execution of CASTf90', 50) cmd = 'analogue.out %s' % path.relpath(config_file) # system(cmd) args = shlex.split(cmd) output, error = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() LOGGER.info('analogue.out info:\n %s ' % output) LOGGER.debug('analogue.out errors:\n %s ' % error) response.update_status('**** CASTf90 suceeded', 90) except Exception as e: msg = 'CASTf90 failed %s ' % e LOGGER.error(msg) raise Exception(msg) LOGGER.debug("castf90 took %s seconds.", time.time() - start_time) response.update_status('preparting output', 99) response.outputs['config'] = config_output_url # config_file ) response.outputs['analogs'] = output_file response.outputs['output_netcdf'] = simulation # response.outputs['output_html'] = output_av response.update_status('execution ended', 100) LOGGER.debug("total execution took %s seconds.", time.time() - process_start_time) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' ncs = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) var = get_variable(ncs[0]) LOGGER.info('variable to be plotted: %s' % var) # mosaic = self.mosaic.getValue() if 'region' in request.inputs: regions = [inp.data for inp in request.inputs['region']] try: png_region = vs.plot_polygons(regions) except Exception as e: LOGGER.exception( 'failed to plot the polygon to world map: %s' % e) o1, png_region = mkstemp(dir='.', suffix='.png') # clip the demanded polygons subsets = clipping( resource=ncs, variable=var, polygons=regions, mosaic=True, spatial_wrapping='wrap', ) else: subsets = ncs png_region = vs.plot_extend(ncs[0]) response.update_status('Arguments set for subset process', 0) try: tar_subsets = archive(subsets) except Exception as e: LOGGER.exception('failed to archive subsets: %s' % e) _, tar_subsets = mkstemp(dir='.', suffix='.tar') try: png_uncertainty = vs.uncertainty(subsets, variable=var) except Exception as e: LOGGER.exception('failed to generate the uncertainty plot: %s' % e) _, png_uncertainty = mkstemp(dir='.', suffix='.png') try: png_spaghetti = vs.spaghetti( subsets, variable=var, ) except Exception as e: LOGGER.exception('failed to generate the spaghetti plot: %s' % e) _, png_spaghetti = mkstemp(dir='.', suffix='.png') try: from flyingpigeon import robustness as ro signal, low_agreement_mask, high_agreement_mask, text_src = ro.signal_noise_ratio( resource=subsets, # start=None, end=None, # timeslice=None, # variable=var ) # if title is None: title = 'signal robustness of %s ' % ( var) # , end1, end2, start1, start2 png_robustness = vs.map_robustness( signal, high_agreement_mask, low_agreement_mask, # cmap=cmap, # title=title ) LOGGER.info('robustness graphic generated') except Exception as e: LOGGER.exception('failed to generate the robustness plot: %s' % e) _, png_robustness = mkstemp(dir='.', suffix='.png') factsheet = vs.factsheetbrewer(png_region=png_region, png_uncertainty=png_uncertainty, png_spaghetti=png_spaghetti, png_robustness=png_robustness) response.outputs['output_nc'].file = tar_subsets response.outputs['output_factsheet'].file = factsheet response.update_status("done", 100) return response
def _handler(self, request, response): response.update_status('starting uncertainty process', 0) init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' try: ncfiles = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) # TODO: See where 'method' should be called method = request.inputs['method'][0].data if 'start' in request.inputs: start = request.inputs['start'][0].data else: start = None if 'end' in request.inputs: end = request.inputs['end'][0].data else: end = None if 'timeslice' in request.inputs: timeslice = request.inputs['timeslice'][0].data else: timeslice = None response.update_status('arguments read', 5) LOGGER.info('Successfully read in the arguments') except Exception as ex: msg = 'failed to read in the arguments: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) response.outputs['output_text'].file = write_fileinfo(ncfiles) # LOGGER.debug('variable set to %s' % variable) # if method == 'signal_noise_ratio': signal, low_agreement_mask, high_agreement_mask, text_src = erob.signal_noise_ratio( resource=ncfiles, start=start, end=end, timeslice=timeslice, # variable=variable ) # graphic, LOGGER.debug('Robustness calculated') try: # LOGGER.info('variable to be plotted: %s' % variable) from flyingpigeon.visualisation import map_robustness # if title is None: title = 'signal robustness' # , end1, end2, start1, start2 graphic = map_robustness( signal, high_agreement_mask, low_agreement_mask, # variable=variable, # cmap=cmap, title=title) LOGGER.info('graphic generated') except Exception as ex: LOGGER.exception('graphic generation failed: {}'.format(str(ex))) _, graphic = mkstemp(dir='.', suffix='.png') response.update_status('process worker done', 95) response.outputs['output_signal'].file = signal response.outputs['output_high'].file = high_agreement_mask response.outputs['output_low'].file = low_agreement_mask response.outputs['output_graphic'].file = graphic response.update_status('uncertainty process done', 100) return response
def _handler(self, request, response): from flyingpigeon.utils import archive, archiveextract from tempfile import mkstemp init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' ncs = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) # mosaic = self.mosaic.getValue() regions = [inp.data for inp in request.inputs['region']] response.update_status('Arguments set for subset process', 0) LOGGER.debug('starting: regions=%s, num_files=%s' % (len(regions), len(ncs))) try: from flyingpigeon.visualisation import plot_polygons png_country = plot_polygons(regions) except: LOGGER.exception('failed to plot the polygon to world map') o1, png_country = mkstemp(dir='.', suffix='.png') # clip the demanded polygons from flyingpigeon.subset import clipping subsets = clipping(resource=ncs, variable=None, dimension_map=None, calc=None, output_format='nc', calc_grouping=None, time_range=None, time_region=None, historical_concatination=True, prefix=None, spatial_wrapping='wrap', polygons=regions, mosaic=True ) try: tar_subsets = archive(subsets) except: LOGGER.exception('failed to archive subsets') _, tar_subsets = mkstemp(dir='.', suffix='.tar') try: from flyingpigeon.visualisation import uncertainty png_uncertainty = uncertainty(subsets) except: LOGGER.exception('failed to generate the uncertainty plot') _, png_uncertainty = mkstemp(dir='.', suffix='.png') try: from flyingpigeon.visualisation import spaghetti png_spaghetti = spaghetti(subsets) except: LOGGER.exception('failed to generate the spaghetti plot') _, png_spaghetti = mkstemp(dir='.', suffix='.png') try: from flyingpigeon import robustness as erob from flyingpigeon.utils import get_variable variable = get_variable(ncs[0]) signal, low_agreement_mask, high_agreement_mask, text_src = erob.method_A(resource=subsets, # start=None, end=None, # timeslice=None, variable=variable ) LOGGER.info('variable to be plotted: %s' % variable) from flyingpigeon.visualisation import map_robustness # if title is None: title = 'signal robustness of %s ' % (variable) # , end1, end2, start1, start2 png_robustness = map_robustness(signal, high_agreement_mask, low_agreement_mask, # cmap=cmap, title=title) LOGGER.info('graphic generated') except: LOGGER.exception('failed to generate the robustness plot') _, png_robustness = mkstemp(dir='.', suffix='.png') from flyingpigeon.visualisation import factsheetbrewer factsheet = factsheetbrewer(png_country=png_country, png_uncertainty=png_uncertainty, png_spaghetti=png_spaghetti, png_robustness=png_robustness) response.outputs['output_nc'].file = tar_subsets response.outputs['output_factsheet'].file = factsheet response.update_status("done", 100) return response