def test_download_with_cache(): filename = utils.download(TESTDATA['cmip5_tasmax_2006_nc'], cache=paths.cache) assert os.path.basename(filename) == 'tasmax_Amon_MPI-ESM-MR_rcp45_r1i1p1_200601-200612.nc'
def reanalyses( start=1948, end=None, variable='slp', dataset='NCEP'): """ Fetches the reanalysis data (NCEP, 20CR or ERA_20C) to local file system :param start: int for start year to fetch source data :param end: int for end year to fetch source data (if None, current year will be the end) :param variable: variable name (default='slp'), geopotential height is given as e.g. z700 :param dataset: default='NCEP' :return list: list of path/files.nc """ try: from datetime import datetime as dt if end == None: end = dt.now().year obs_data = [] if start == None: if dataset == 'NCEP': start = 1948 if dataset == '20CR': start = 1851 logger.info('start / end date set') except Exception as e: msg = "get_OBS module failed to get start end dates %s " % e logger.debug(msg) raise Exception(msg) if 'z' in variable: level = variable.strip('z') else: level = None logger.info('level: %s' % level) try: for year in range(start, end + 1): try: if dataset == 'NCEP': if variable == 'slp': url = 'http://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/ncep.reanalysis.dailyavgs/surface/%s.%s.nc' % (variable, year) elif 'z' in variable: url = 'http://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/ncep.reanalysis.dailyavgs/pressure/hgt.%s.nc' % ( year) elif dataset == '20CRV2': if variable == 'prmsl': url = 'http://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/20thC_ReanV2/monolevel/prmsl.%s.nc' % year if 'z' in variable: url = 'http://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/20thC_ReanV2/pressure/hgt.%s.nc' % ( year ) elif dataset == '20CRV2c': if variable == 'prmsl': url = 'http://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/20thC_ReanV2c/monolevel/prmsl.%s.nc' % year if 'z' in variable: url = 'http://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/20thC_ReanV2c/pressure/hgt.%s.nc' % ( year ) else: logger.error('Dataset %s not known' % dataset) except Exception as e: msg = "could not set url: %s " % e logger.debug(msg) raise Exception(msg) try: obs_data.append(utils.download(url, cache=True)) except: msg = "wget failed on {0}.".format(url) logger.debug(msg) raise Exception(msg) logger.info('Reanalyses data fetched for %s files' % len(obs_data)) except Exception as e: msg = "get reanalyses module failed to fetch data %s " % e logger.debug(msg) raise Exception(msg) if level == None: data = obs_data else: logger.info('get level: %s' % level) data = get_level(obs_data, level=level) return data
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' response.update_status('Start process', 0) try: LOGGER.info('reading the arguments') resources = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) period = request.inputs['period'] period = period[0].data indices = [inpt.data for inpt in request.inputs['indices']] archive_format = request.inputs['archive_format'][0].data LOGGER.info( "all arguments read in nr of files in resources: {}".foirmat( len(resources))) except Exception as ex: msg = 'failed to read in the arguments: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: gbif_url = request.inputs['gbif'][0].data csv_file = download(gbif_url) LOGGER.info('CSV file fetched sucessfully: %s' % csv_file) except Exception as ex: msg = 'failed to fetch GBIF file: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: response.update_status('read in latlon coordinates', 10) latlon = sdm.latlon_gbifcsv(csv_file) LOGGER.info('got occurence coordinates %s ' % csv_file) except Exception as ex: msg = 'failed to extract the latlon points from file {}: {}'.format( csv_file, str(ex)) LOGGER.exception(msg) raise Exception(msg) try: response.update_status('plot map', 20) occurence_map = map_gbifoccurrences(latlon) LOGGER.info('GBIF occourence ploted') except Exception as ex: msg = 'failed to plot occurence map: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) ################################# # calculate the climate indices ################################# # get the indices try: response.update_status('start calculation of indices', 30) ncs_indices = sdm.get_indices(resource=resources, indices=indices) LOGGER.info('indice calculation done') except Exception as ex: msg = 'failed to calculate indices: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: # sort indices indices_dic = sdm.sort_indices(ncs_indices) LOGGER.info('indice files sorted in dictionary') except Exception as ex: msg = 'failed to sort indices: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) indices_dic = {'dummy': []} ncs_references = [] species_files = [] stat_infos = [] PAmask_pngs = [] response.update_status('Start processing for {} datasets'.format( len(indices_dic.keys()))) for count, key in enumerate(indices_dic.keys()): try: status_nr = 40 + count * 10 response.update_status('Start processing of {}'.format(key), status_nr) ncs = indices_dic[key] LOGGER.info('with {} files'.format(len(ncs))) try: response.update_status('generating the PA mask', 20) PAmask = sdm.get_PAmask(coordinates=latlon, nc=ncs[0]) LOGGER.info('PA mask sucessfully generated') except Exception as ex: msg = 'failed to generate the PA mask: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: response.update_status('Ploting PA mask', 25) PAmask_pngs.extend([map_PAmask(PAmask)]) except Exception as ex: msg = 'failed to plot the PA mask: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: ncs_reference = sdm.get_reference(ncs_indices=ncs, period=period) ncs_references.extend(ncs_reference) LOGGER.info('reference indice calculated {}'.format( ncs_references)) except Exception as ex: msg = 'failed to calculate the reference: {}'.format( str(ex)) LOGGER.exception(msg) raise Exception(msg) try: gam_model, predict_gam, gam_info = sdm.get_gam( ncs_reference, PAmask) stat_infos.append(gam_info) response.update_status('GAM sucessfully trained', status_nr + 5) except Exception as ex: msg = 'failed to train GAM for {}: {}'.format(key, str(ex)) LOGGER.debug(msg) raise Exception(msg) try: prediction = sdm.get_prediction(gam_model, ncs) response.update_status('prediction done', status_nr + 7) except Exception as ex: msg = 'failed to predict tree occurence: {}'.format( str(ex)) LOGGER.exception(msg) raise Exception(msg) # # try: # response.update_status('land sea mask for predicted data', status_nr + 8) # from numpy import invert, isnan, nan, broadcast_arrays # , array, zeros, linspace, meshgrid # mask = invert(isnan(PAmask)) # mask = broadcast_arrays(prediction, mask)[1] # prediction[mask is False] = nan # except: # LOGGER.debug('failed to mask predicted data') try: species_files.append(sdm.write_to_file(ncs[0], prediction)) LOGGER.info('Favourability written to file') except Exception as ex: msg = 'failed to write species file: {}'.format(str(ex)) LOGGER.debug(msg) raise Exception(msg) except Exception as ex: msg = 'failed to process SDM chain for {} : {}'.format( key, str(ex)) LOGGER.exception(msg) raise Exception(msg) try: archive_indices = archive(ncs_indices, format=archive_format) LOGGER.info('indices added to archive') except Exception as ex: msg = 'failed adding indices to archive: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) archive_indices = tempfile.mkstemp(suffix='.tar', prefix='foobar-', dir='.') try: archive_references = archive(ncs_references, format=archive_format) LOGGER.info('indices reference added to archive') except Exception as ex: msg = 'failed adding reference indices to archive: {}'.format( str(ex)) LOGGER.exception(msg) raise Exception(msg) archive_references = tempfile.mkstemp(suffix='.tar', prefix='foobar-', dir='.') try: archive_prediction = archive(species_files, format=archive_format) LOGGER.info('species_files added to archive') except Exception as ex: msg = 'failed adding species_files indices to archive: {}'.format( str(ex)) LOGGER.exception(msg) raise Exception(msg) try: stat_infosconcat = pdfmerge(stat_infos) LOGGER.debug('pngs {}'.format(PAmask_pngs)) PAmask_png = concat_images(PAmask_pngs, orientation='h') LOGGER.info('stat infos pdfs and mask pngs merged') except Exception as ex: msg = 'failed to concat images: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) _, stat_infosconcat = tempfile.mkstemp(suffix='.pdf', prefix='foobar-', dir='.') _, PAmask_png = tempfile.mkstemp(suffix='.png', prefix='foobar-', dir='.') # self.output_csv.setValue(csv_file) response.outputs['output_gbif'].file = occurence_map response.outputs['output_PA'].file = PAmask_png response.outputs['output_indices'].file = archive_indices response.outputs['output_reference'].file = archive_references response.outputs['output_prediction'].file = archive_prediction response.outputs['output_info'].file = stat_infosconcat response.update_status('done', 100) return response
def execute(self): logger.info('Start process') init_process_logger('log.txt') self.output_log.setValue('log.txt') from datetime import datetime as dt from flyingpigeon import weatherregimes as wr from tempfile import mkstemp ################################ # reading in the input arguments ################################ try: resource = self.getInputValues(identifier='resource') url_Rdat = self.getInputValues(identifier='Rdat')[0] url_dat = self.getInputValues(identifier='dat')[0] url_ref_file = self.getInputValues( identifier='netCDF') # can be None season = self.getInputValues(identifier='season')[0] period = self.getInputValues(identifier='period')[0] anualcycle = self.getInputValues(identifier='anualcycle')[0] except Exception as e: logger.debug('failed to read in the arguments %s ' % e) try: start = dt.strptime(period.split('-')[0], '%Y%m%d') end = dt.strptime(period.split('-')[1], '%Y%m%d') # kappa = int(self.getInputValues(identifier='kappa')[0]) logger.info('period %s' % str(period)) logger.info('season %s' % str(season)) logger.info('read in the arguments') logger.info('url_ref_file: %s' % url_ref_file) logger.info('url_Rdat: %s' % url_Rdat) logger.info('url_dat: %s' % url_dat) except Exception as e: logger.debug('failed to convert arguments %s ' % e) ############################ # fetching trainging data ############################ from flyingpigeon.utils import download, get_time from os.path import abspath try: dat = abspath(download(url_dat)) Rdat = abspath(download(url_Rdat)) logger.info('training data fetched') except Exception as e: logger.error('failed to fetch training data %s' % e) ########################################################## # get the required bbox and time region from resource data ########################################################## # from flyingpigeon.weatherregimes import get_level from flyingpigeon.ocgis_module import call from flyingpigeon.utils import get_variable time_range = [start, end] variable = get_variable(resource) if len(url_ref_file) > 0: ref_file = download(url_ref_file[0]) model_subset = call( resource=resource, variable=variable, time_range= time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap', regrid_destination=ref_file, regrid_options='bil') logger.info('Dataset subset with regridding done: %s ' % model_subset) else: model_subset = call( resource=resource, variable=variable, time_range= time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap', ) logger.info('Dataset time period extracted: %s ' % model_subset) ####################### # computing anomalies ####################### cycst = anualcycle.split('-')[0] cycen = anualcycle.split('-')[0] reference = [ dt.strptime(cycst, '%Y%m%d'), dt.strptime(cycen, '%Y%m%d') ] model_anomal = wr.get_anomalies(model_subset, reference=reference) ##################### # extracting season ##################### model_season = wr.get_season(model_anomal, season=season) ####################### # call the R scripts ####################### import shlex import subprocess from flyingpigeon import config from os.path import curdir, exists, join try: rworkspace = curdir Rsrc = config.Rsrc_dir() Rfile = 'weatherregimes_projection.R' yr1 = start.year yr2 = end.year time = get_time(model_season, format='%Y%m%d') # ip, output_graphics = mkstemp(dir=curdir ,suffix='.pdf') ip, file_pca = mkstemp(dir=curdir, suffix='.txt') ip, file_class = mkstemp(dir=curdir, suffix='.Rdat') ip, output_frec = mkstemp(dir=curdir, suffix='.txt') args = [ 'Rscript', join(Rsrc, Rfile), '%s/' % curdir, '%s/' % Rsrc, '%s' % model_season, '%s' % variable, '%s' % str(time).strip("[]").replace("'", "").replace(" ", ""), # '%s' % output_graphics, '%s' % dat, '%s' % Rdat, '%s' % file_pca, '%s' % file_class, '%s' % output_frec, '%s' % season, '%s' % start.year, '%s' % end.year, '%s' % 'MODEL' ] logger.info('Rcall builded') except Exception as e: msg = 'failed to build the R command %s' % e logger.error(msg) raise Exception(msg) try: output, error = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() # , shell=True logger.info('R outlog info:\n %s ' % output) logger.debug('R outlog errors:\n %s ' % error) if len(output) > 0: self.status.set('**** weatherregime in R suceeded', 90) else: logger.error('NO! output returned from R call') except Exception as e: msg = 'weatherregime in R %s ' % e logger.error(msg) raise Exception(msg) ################# # set the outputs ################# # self.Routput_graphic.setValue( output_graphics ) self.output_pca.setValue(file_pca) self.output_classification.setValue(file_class) self.output_netcdf.setValue(model_season) self.output_frequency.setValue(output_frec)
def reanalyses(start=1948, end=None, variable='slp', dataset='NCEP', timres='day', getlevel=True): """ Fetches the reanalysis data (NCEP, 20CR or ERA_20C) to local file system :param start: int for start year to fetch source data :param end: int for end year to fetch source data (if None, current year will be the end) :param variable: variable name (default='slp'), geopotential height is given as e.g. z700 :param dataset: default='NCEP' :return list: list of path/files.nc """ # used for NETCDF convertion from netCDF4 import Dataset from os import path, system from flyingpigeon.ocgis_module import call from shutil import move # used for NETCDF convertion try: from datetime import datetime as dt if end is None: end = dt.now().year obs_data = [] if start is None: if dataset == 'NCEP': start = 1948 if dataset == '20CR': start = 1851 LOGGER.info('start / end date set') except: msg = "get_OBS module failed to get start end dates" LOGGER.exception(msg) raise Exception(msg) if 'z' in variable: level = variable.strip('z') else: level = None LOGGER.info('level: %s' % level) try: for year in range(start, end + 1): LOGGER.debug('fetching single file for %s year %s ' % (dataset, year)) try: if dataset == 'NCEP': if variable == 'slp': url = 'https://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/ncep.reanalysis.dailyavgs/surface/%s.%s.nc' % ( variable, year) # noqa if 'z' in variable: url = 'https://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/ncep.reanalysis.dailyavgs/pressure/hgt.%s.nc' % ( year) # noqa elif dataset == '20CRV2': if variable == 'prmsl': if timres == '6h': url = 'https://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/20thC_ReanV2/monolevel/prmsl.%s.nc' % year # noqa else: url = 'https://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/20thC_ReanV2/Dailies/monolevel/prmsl.%s.nc' % year # noqa if 'z' in variable: if timres == '6h': url = 'https://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/20thC_ReanV2/pressure/hgt.%s.nc' % ( year) # noqa else: url = 'https://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/20thC_ReanV2/Dailies/pressure/hgt.%s.nc' % ( year) # noqa elif dataset == '20CRV2c': if variable == 'prmsl': if timres == '6h': url = 'https://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/20thC_ReanV2c/monolevel/prmsl.%s.nc' % year # noqa else: url = 'https://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/20thC_ReanV2c/Dailies/monolevel/prmsl.%s.nc' % year # noqa if 'z' in variable: if timres == '6h': url = 'https://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/20thC_ReanV2c/pressure/hgt.%s.nc' % ( year) # noqa else: url = 'https://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/20thC_ReanV2c/Dailies/pressure/hgt.%s.nc' % ( year) # noqa else: LOGGER.debug('Dataset %s not known' % dataset) LOGGER.debug('url: %s' % url) except: msg = "could not set url" LOGGER.exception(msg) try: df = utils.download(url, cache=True) LOGGER.debug('single file fetched %s ' % year) # convert to NETCDF4_CLASSIC try: ds = Dataset(df) df_time = ds.variables['time'] # Here, need to check not just calendar, but that file is ncdf_classic already... if (hasattr(df_time, 'calendar')) is False: p, f = path.split(path.abspath(df)) LOGGER.debug("path = %s , file %s " % (p, f)) # May be an issue if several users are working at the same time move(df, f) conv = call(resource=f, output_format_options={ 'data_model': 'NETCDF4_CLASSIC' }, dir_output=p, prefix=f.replace('.nc', '')) obs_data.append(conv) LOGGER.debug('file %s to NETCDF4_CLASSIC converted' % conv) # Cleaning, could be 50gb... for each (!) user # TODO Check how links work cmdrm = 'rm -f %s' % (f) system(cmdrm) else: obs_data.append(df) ds.close() except: LOGGER.exception('failed to convert into NETCDF4_CLASSIC') except: msg = "download failed on {0}.".format(url) LOGGER.exception(msg) LOGGER.info('Reanalyses data fetched for %s files' % len(obs_data)) except: msg = "get reanalyses module failed to fetch data" LOGGER.exception(msg) raise Exception(msg) if (level is None) or (getlevel == False): data = obs_data else: LOGGER.info('get level: %s' % level) data = get_level(obs_data, level=level) return data
def reanalyses(start=1948, end=None, variable='slp', dataset='NCEP'): """ Fetches the reanalysis data (NCEP, 20CR or ERA_20C) to local file system :param start: int for start year to fetch source data :param end: int for end year to fetch source data (if None, current year will be the end) :param variable: variable name (default='slp'), geopotential height is given as e.g. z700 :param dataset: default='NCEP' :return list: list of path/files.nc """ try: from datetime import datetime as dt if end == None: end = dt.now().year obs_data = [] if start == None: if dataset == 'NCEP': start = 1948 if dataset == '20CR': start = 1851 logger.info('start / end date set') except Exception as e: msg = "get_OBS module failed to get start end dates %s " % e logger.debug(msg) raise Exception(msg) if 'z' in variable: level = variable.strip('z') else: level = None logger.info('level: %s' % level) try: for year in range(start, end + 1): try: if dataset == 'NCEP': if variable == 'slp': url = 'http://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/ncep.reanalysis.dailyavgs/surface/%s.%s.nc' % ( variable, year) elif 'z' in variable: url = 'http://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/ncep.reanalysis.dailyavgs/pressure/hgt.%s.nc' % ( year) elif dataset == '20CRV2': if variable == 'prmsl': url = 'http://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/20thC_ReanV2/monolevel/prmsl.%s.nc' % year if 'z' in variable: url = 'http://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/20thC_ReanV2/pressure/hgt.%s.nc' % ( year) elif dataset == '20CRV2c': if variable == 'prmsl': url = 'http://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/20thC_ReanV2c/monolevel/prmsl.%s.nc' % year if 'z' in variable: url = 'http://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/20thC_ReanV2c/pressure/hgt.%s.nc' % ( year) else: logger.error('Dataset %s not known' % dataset) except Exception as e: msg = "could not set url: %s " % e logger.debug(msg) raise Exception(msg) try: obs_data.append(utils.download(url, cache=True)) except: msg = "wget failed on {0}.".format(url) logger.debug(msg) raise Exception(msg) logger.info('Reanalyses data fetched for %s files' % len(obs_data)) except Exception as e: msg = "get reanalyses module failed to fetch data %s " % e logger.debug(msg) raise Exception(msg) if level == None: data = obs_data else: logger.info('get level: %s' % level) data = get_level(obs_data, level=level) return data
def reanalyses(start=1948, end=None, variable='slp', dataset='NCEP'): """ Fetches the reanalysis data (NCEP, 20CR or ERA_20C) to local file system :param start: int for start year to fetch source data :param end: int for end year to fetch source data (if None, current year will be the end) :param variable: variable name (default='slp'), geopotential height is given as e.g. z700 :param dataset: default='NCEP' :return list: list of path/files.nc """ # used for NETCDF convertion from os import path from flyingpigeon.ocgis_module import call from shutil import move # used for NETCDF convertion try: from datetime import datetime as dt if end is None: end = dt.now().year obs_data = [] if start is None: if dataset == 'NCEP': start = 1948 if dataset == '20CR': start = 1851 LOGGER.info('start / end date set') except: msg = "get_OBS module failed to get start end dates" LOGGER.exception(msg) raise Exception(msg) if 'z' in variable: level = variable.strip('z') else: level = None LOGGER.info('level: %s' % level) try: for year in range(start, end + 1): LOGGER.debug('fetching single file for %s year %s ' % (dataset, year)) try: if dataset == 'NCEP': if variable == 'slp': url = 'https://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/ncep.reanalysis.dailyavgs/surface/%s.%s.nc' % ( variable, year) # noqa if 'z' in variable: url = 'https://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/ncep.reanalysis.dailyavgs/pressure/hgt.%s.nc' % ( year) # noqa elif dataset == '20CRV2': if variable == 'prmsl': url = 'https://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/20thC_ReanV2/monolevel/prmsl.%s.nc' % year # noqa if 'z' in variable: url = 'https://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/20thC_ReanV2/pressure/hgt.%s.nc' % ( year) # noqa elif dataset == '20CRV2c': if variable == 'prmsl': url = 'https://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/20thC_ReanV2c/monolevel/prmsl.%s.nc' % year # noqa if 'z' in variable: url = 'https://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/20thC_ReanV2c/pressure/hgt.%s.nc' % ( year) # noqa else: LOGGER.debug('Dataset %s not known' % dataset) LOGGER.debug('url: %s' % url) except: msg = "could not set url" LOGGER.exception(msg) try: df = utils.download(url, cache=True) LOGGER.debug('single file fetched %s ' % year) # convert to NETCDF4_CLASSIC try: p, f = path.split(path.abspath(df)) LOGGER.debug("path = %s , file %s " % (p, f)) move(df, f) conv = call(resource=f, output_format_options={ 'data_model': 'NETCDF4_CLASSIC' }, dir_output=p, prefix=f.replace('.nc', '')) obs_data.append(conv) LOGGER.debug('file %s to NETCDF4_CLASSIC converted' % conv) except: LOGGER.exception('failed to convert into NETCDF4_CLASSIC') except: msg = "download failed on {0}.".format(url) LOGGER.exception(msg) LOGGER.info('Reanalyses data fetched for %s files' % len(obs_data)) except: msg = "get reanalyses module failed to fetch data" LOGGER.exception(msg) raise Exception(msg) if level is None: data = obs_data else: LOGGER.info('get level: %s' % level) data = get_level(obs_data, level=level) return data
from flyingpigeon.utils import download from ocgis import RequestDataset, OcgOperations variable = 'slp' year = 2000 url = 'https://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/ncep.reanalysis.dailyavgs/surface/%s.%s.nc' % (variable, year) f = download(url) rd = RequestDataset(f) ds = OcgOperations(rd,output_format_options={'data_model': 'NETCDF4_CLASSIC'},prefix='converted_to_NETCDF_CLASSIC' ).execute() ds = OcgOperations(rd,output_format_options={'data_model': 'NETCDF4_CLASSIC'}, output_format = 'nc', prefix='converted_to_NETCDF_CLASSIC' ).execute() print ds
def execute(self): logger.info('Start process') from datetime import datetime as dt from flyingpigeon import weatherregimes as wr from tempfile import mkstemp ################################ # reading in the input arguments ################################ try: resource = self.getInputValues(identifier='resource') url_Rdat = self.getInputValues(identifier='Rdat')[0] url_dat = self.getInputValues(identifier='dat')[0] url_ref_file = self.getInputValues(identifier='netCDF') # can be None season = self.getInputValues(identifier='season')[0] period = self.getInputValues(identifier='period')[0] anualcycle = self.getInputValues(identifier='anualcycle')[0] except Exception as e: logger.debug('failed to read in the arguments %s ' % e) try: start = dt.strptime(period.split('-')[0] , '%Y%m%d') end = dt.strptime(period.split('-')[1] , '%Y%m%d') # kappa = int(self.getInputValues(identifier='kappa')[0]) logger.info('period %s' % str(period)) logger.info('season %s' % str(season)) logger.info('read in the arguments') logger.info('url_ref_file: %s' % url_ref_file) logger.info('url_Rdat: %s' % url_Rdat) logger.info('url_dat: %s' % url_dat) except Exception as e: logger.debug('failed to convert arguments %s ' % e) ############################ # fetching trainging data ############################ from flyingpigeon.utils import download, get_time from os.path import abspath try: dat = abspath(download(url_dat)) Rdat = abspath(download(url_Rdat)) logger.info('training data fetched') except Exception as e: logger.error('failed to fetch training data %s' % e) ############################################################ ### get the required bbox and time region from resource data ############################################################ # from flyingpigeon.weatherregimes import get_level from flyingpigeon.ocgis_module import call from flyingpigeon.utils import get_variable time_range = [start, end] variable = get_variable(resource) if len(url_ref_file) > 0: ref_file = download(url_ref_file[0]) model_subset = call(resource=resource, variable=variable, time_range=time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap', regrid_destination=ref_file, regrid_options='bil') logger.info('Dataset subset with regridding done: %s ' % model_subset) else: model_subset = call(resource=resource, variable=variable, time_range=time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap', ) logger.info('Dataset time period extracted: %s ' % model_subset) ############################################## ### computing anomalies ############################################## cycst = anualcycle.split('-')[0] cycen = anualcycle.split('-')[0] reference = [dt.strptime(cycst,'%Y%m%d'), dt.strptime(cycen,'%Y%m%d')] model_anomal = wr.get_anomalies(model_subset, reference=reference) ##################### ### extracting season ##################### model_season = wr.get_season(model_anomal, season=season) ####################### ### call the R scripts ####################### import shlex import subprocess from flyingpigeon import config from os.path import curdir, exists, join try: rworkspace = curdir Rsrc = config.Rsrc_dir() Rfile = 'weatherregimes_projection.R' yr1 = start.year yr2 = end.year time = get_time(model_season, format='%Y%m%d') #ip, output_graphics = mkstemp(dir=curdir ,suffix='.pdf') ip, file_pca = mkstemp(dir=curdir ,suffix='.txt') ip, file_class = mkstemp(dir=curdir ,suffix='.Rdat') ip, output_frec = mkstemp(dir=curdir ,suffix='.txt') args = ['Rscript', join(Rsrc,Rfile), '%s/' % curdir, '%s/' % Rsrc, '%s' % model_season, '%s' % variable, '%s' % str(time).strip("[]").replace("'","").replace(" ",""), # '%s' % output_graphics, '%s' % dat, '%s' % Rdat, '%s' % file_pca, '%s' % file_class, '%s' % output_frec, '%s' % season, '%s' % start.year, '%s' % end.year, '%s' % 'MODEL'] logger.info('Rcall builded') except Exception as e: msg = 'failed to build the R command %s' % e logger.error(msg) raise Exception(msg) try: output,error = subprocess.Popen(args, stdout = subprocess.PIPE, stderr= subprocess.PIPE).communicate() #, shell=True logger.info('R outlog info:\n %s ' % output) logger.debug('R outlog errors:\n %s ' % error) if len(output) > 0: self.status.set('**** weatherregime in R suceeded', 90) else: logger.error('NO! output returned from R call') except Exception as e: msg = 'weatherregime in R %s ' % e logger.error(msg) raise Exception(msg) ############################################ ### set the outputs ############################################ #self.Routput_graphic.setValue( output_graphics ) self.output_pca.setValue( file_pca ) self.output_classification.setValue( file_class ) self.output_netcdf.setValue( model_season ) self.output_frequency.setValue( output_frec )
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' response.update_status('execution started at : {}'.format(dt.now()), 5) ################################ # reading in the input arguments ################################ try: LOGGER.info('read in the arguments') # resources = self.getInputValues(identifier='resources') season = request.inputs['season'][0].data LOGGER.info('season %s', season) period = request.inputs['period'][0].data LOGGER.info('period %s', period) anualcycle = request.inputs['anualcycle'][0].data start = dt.strptime(period.split('-')[0], '%Y%m%d') end = dt.strptime(period.split('-')[1], '%Y%m%d') LOGGER.debug('start: %s , end: %s ' % (start, end)) resource = archiveextract(resource=rename_complexinputs(request.inputs['resource'])) # resource = archiveextract(resource=[res.file for res in request.inputs['resource']]) url_Rdat = request.inputs['Rdat'][0].data url_dat = request.inputs['dat'][0].data url_ref_file = request.inputs['netCDF'][0].data # can be None # season = self.getInputValues(identifier='season')[0] # period = self.getInputValues(identifier='period')[0] # anualcycle = self.getInputValues(identifier='anualcycle')[0] LOGGER.info('period %s' % str(period)) LOGGER.info('season %s' % str(season)) LOGGER.info('read in the arguments') LOGGER.info('url_ref_file: %s' % url_ref_file) LOGGER.info('url_Rdat: %s' % url_Rdat) LOGGER.info('url_dat: %s' % url_dat) except Exception as e: LOGGER.debug('failed to convert arguments %s ' % e) ############################ # fetching trainging data ############################ try: dat = abspath(download(url_dat)) Rdat = abspath(download(url_Rdat)) LOGGER.info('training data fetched') except Exception as e: LOGGER.error('failed to fetch training data %s' % e) ########################################################## # get the required bbox and time region from resource data ########################################################## # from flyingpigeon.weatherregimes import get_level try: from flyingpigeon.ocgis_module import call from flyingpigeon.utils import get_variable time_range = [start, end] variable = get_variable(resource) if len(url_ref_file) > 0: ref_file = download(url_ref_file) model_subset = call( resource=resource, variable=variable, time_range=time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap', regrid_destination=ref_file, regrid_options='bil') LOGGER.info('Dataset subset with regridding done: %s ' % model_subset) else: model_subset = call( resource=resource, variable=variable, time_range=time_range, # conform_units_to=conform_units_to, geom=bbox, spatial_wrapping='wrap', ) LOGGER.info('Dataset time period extracted: %s ' % model_subset) except: LOGGER.exception('failed to make a data subset ') ####################### # computing anomalies ####################### try: cycst = anualcycle.split('-')[0] cycen = anualcycle.split('-')[0] reference = [dt.strptime(cycst, '%Y%m%d'), dt.strptime(cycen, '%Y%m%d')] model_anomal = wr.get_anomalies(model_subset, reference=reference) ##################### # extracting season ##################### model_season = wr.get_season(model_anomal, season=season) except: LOGGER.exception('failed to compute anualcycle or seasons') ####################### # call the R scripts ####################### import shlex import subprocess from flyingpigeon import config from os.path import curdir, exists, join try: rworkspace = curdir Rsrc = config.Rsrc_dir() Rfile = 'weatherregimes_projection.R' yr1 = start.year yr2 = end.year time = get_time(model_season, format='%Y%m%d') # ip, output_graphics = mkstemp(dir=curdir ,suffix='.pdf') ip, file_pca = mkstemp(dir=curdir, suffix='.txt') ip, file_class = mkstemp(dir=curdir, suffix='.Rdat') ip, output_frec = mkstemp(dir=curdir, suffix='.txt') args = ['Rscript', join(Rsrc, Rfile), '%s/' % curdir, '%s/' % Rsrc, '%s' % model_season, '%s' % variable, '%s' % str(time).strip("[]").replace("'", "").replace(" ", ""), # '%s' % output_graphics, '%s' % dat, '%s' % Rdat, '%s' % file_pca, '%s' % file_class, '%s' % output_frec, '%s' % season, '%s' % start.year, '%s' % end.year, '%s' % 'MODEL'] LOGGER.info('Rcall builded') except Exception as e: msg = 'failed to build the R command %s' % e LOGGER.error(msg) raise Exception(msg) try: output, error = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() # , shell=True LOGGER.info('R outlog info:\n %s ' % output) LOGGER.debug('R outlog errors:\n %s ' % error) if len(output) > 0: response.update_status('**** weatherregime in R suceeded', 90) else: LOGGER.error('NO! output returned from R call') except Exception as e: msg = 'weatherregime in R %s ' % e LOGGER.error(msg) raise Exception(msg) ################# # set the outputs ################# response.update_status('Set the process outputs ', 95) response.outputs['output_pca'].file = file_pca response.outputs['output_classification'].file = file_class response.outputs['output_netcdf'].file = model_season response.outputs['output_frequency'].file = output_frec response.update_status('done', 100) return response
from flyingpigeon import utils from ocgis import RequestDataset, OcgOperations, envs from ocgis.util.large_array import compute from datetime import datetime as dt import uuid # years = range(1948,2018) years = range(2010, 2012) ncs = [] for year in years: url = 'https://www.esrl.noaa.gov/psd/thredds/fileServer/Datasets/ncep.reanalysis.dailyavgs/pressure/hgt.%s.nc' % (year) ncs.extend(utils.download(url, cache=True)) level_range = [700, 700] time_range = [dt.strptime('20100315', '%Y%m%d'), dt.strptime('20111210', '%Y%m%d')] bbox = [-80, 20, 20, 70] rd = RequestDataset(ncs) ops = OcgOperations(rd, time_range=time_range, level_range=level_range, geom=bbox, output_format='nc', prefix='ocgis_module_optimisation', add_auxiliary_files=False)
def test_download_with_cache(): filename = utils.download(TESTDATA["cmip5_tasmax_2006_nc"], cache=True) assert os.path.basename(filename) == "tasmax_Amon_MPI-ESM-MR_rcp45_r1i1p1_200601-200612.nc"
def execute(self): init_process_logger('log.txt') self.output_log.setValue('log.txt') from os.path import basename from flyingpigeon import sdm from flyingpigeon.utils import archive, archiveextract, download self.status.set('Start process', 0) try: logger.info('reading the arguments') resources_raw = self.getInputValues(identifier='resources') csv_url = self.getInputValues(identifier='gbif')[0] period = self.getInputValues(identifier='period') period = period[0] indices = self.getInputValues(identifier='input_indices') archive_format = self.archive_format.getValue() logger.info('indices %s ' % indices) logger.debug('csv_url %s' % csv_url) except Exception as e: logger.error('failed to read in the arguments %s ' % e) raise try: logger.info('set up the environment') csv_file = download(csv_url) resources = archiveextract(resources_raw) except Exception as e: logger.error('failed to set up the environment %s ' % e) raise try: self.status.set('read in latlon coordinates', 10) latlon = sdm.latlon_gbifcsv(csv_file) logger.info('got occurence coordinates %s ' % csv_file) except Exception as e: logger.exception( 'failed to extract the latlon points from file: %s: %s' % (csv_file, e)) try: self.status.set('plot map', 20) from flyingpigeon.visualisation import map_gbifoccurrences # latlon = sdm.latlon_gbifdic(gbifdic) occurence_map = map_gbifoccurrences(latlon) except Exception as e: logger.exception('failed to plot occurence map %s' % e) ################################# # calculate the climate indices ################################# # get the indices ncs_indices = None try: self.status.set( 'start calculation of climate indices for %s' % indices, 30) ncs_indices = sdm.get_indices(resources=resources, indices=indices) logger.info('indice calculation done') except: msg = 'failed to calculate indices' logger.exception(msg) raise Exception(msg) try: self.status.set('get domain', 30) domains = set() for resource in ncs_indices: # get_domain works only if metadata are set in a correct way domains = domains.union([basename(resource).split('_')[1]]) if len(domains) == 1: domain = list(domains)[0] logger.debug('Domain %s found in indices files' % domain) else: logger.error('Not a single domain in indices files %s' % domains) except Exception as e: logger.exception('failed to get domains %s' % e) try: self.status.set('generating the PA mask', 20) PAmask = sdm.get_PAmask(coordinates=latlon, domain=domain) logger.info('PA mask sucessfully generated') except Exception as e: logger.exception('failed to generate the PA mask: %s' % e) try: self.status.set('Ploting PA mask', 25) from flyingpigeon.visualisation import map_PAmask PAmask_png = map_PAmask(PAmask) except Exception as e: logger.exception('failed to plot the PA mask: %s' % e) try: # sort indices indices_dic = None indices_dic = sdm.sort_indices(ncs_indices) logger.info('indice files sorted for %s Datasets' % len(indices_dic.keys())) except: msg = 'failed to sort indices' logger.exception(msg) raise Exception(msg) ncs_references = [] species_files = [] stat_infos = [] for count, key in enumerate(indices_dic.keys()): try: staus_nr = 40 + count * 10 self.status.set('Start processing of %s' % key, staus_nr) ncs = indices_dic[key] logger.info('with %s files' % len(ncs)) try: ncs_reference = sdm.get_reference(ncs_indices=ncs, period=period) ncs_references.extend(ncs_reference) logger.info('reference indice calculated %s ' % ncs_references) except: msg = 'failed to calculate the reference' logger.exception(msg) raise Exception(msg) try: gam_model, predict_gam, gam_info = sdm.get_gam( ncs_reference, PAmask) stat_infos.append(gam_info) self.status.set('GAM sucessfully trained', staus_nr + 5) except Exception as e: msg = 'failed to train GAM for %s : %s' % (key, e) logger.debug(msg) try: prediction = sdm.get_prediction(gam_model, ncs) self.status.set('prediction done', staus_nr + 7) except Exception as e: msg = 'failed to predict tree occurence %s' % e logger.exception(msg) # raise Exception(msg) try: self.status.set('land sea mask for predicted data', staus_nr + 8) from numpy import invert, isnan, nan, broadcast_arrays # , array, zeros, linspace, meshgrid mask = invert(isnan(PAmask)) mask = broadcast_arrays(prediction, mask)[1] prediction[mask is False] = nan except Exception as e: logger.debug('failed to mask predicted data: %s' % e) try: species_files.append(sdm.write_to_file(ncs[0], prediction)) logger.info('Favourabillity written to file') except Exception as e: msg = 'failed to write species file %s' % e logger.debug(msg) # raise Exception(msg) except Exception as e: msg = 'failed to calculate reference indices. %s ' % e logger.exception(msg) raise Exception(msg) try: archive_indices = None archive_indices = archive(ncs_indices, format=archive_format) logger.info('indices added to archive') except: msg = 'failed adding indices to archive' logger.exception(msg) raise Exception(msg) archive_references = None try: archive_references = archive(ncs_references, format=archive_format) logger.info('indices reference added to archive') except: msg = 'failed adding reference indices to archive' logger.exception(msg) raise Exception(msg) archive_predicion = None try: archive_predicion = archive(species_files, format=archive_format) logger.info('species_files added to archive') except: msg = 'failed adding species_files indices to archive' logger.exception(msg) raise Exception(msg) try: from flyingpigeon.visualisation import pdfmerge stat_infosconcat = pdfmerge(stat_infos) logger.info('stat infos pdfs merged') except: logger.exception('failed to concat images') _, stat_infosconcat = tempfile.mkstemp(suffix='.pdf', prefix='foobar-', dir='.') # self.output_csv.setValue(csv_file) self.output_gbif.setValue(occurence_map) self.output_PA.setValue(PAmask_png) self.output_indices.setValue(archive_indices) self.output_reference.setValue(archive_references) self.output_prediction.setValue(archive_predicion) self.output_info.setValue(stat_infosconcat) self.status.set('done', 100)