def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
response.update_status('Start process', 0)
try:
LOGGER.info('reading the arguments')
resources = archiveextract(
resource=rename_complexinputs(request.inputs['resource']))
period = request.inputs['period']
period = period[0].data
indices = [inpt.data for inpt in request.inputs['indices']]
archive_format = request.inputs['archive_format'][0].data
LOGGER.info(
"all arguments read in nr of files in resources: {}".foirmat(
len(resources)))
except Exception as ex:
msg = 'failed to read in the arguments: {}'.format(str(ex))
LOGGER.exception(msg)
raise Exception(msg)
try:
gbif_url = request.inputs['gbif'][0].data
csv_file = download(gbif_url)
LOGGER.info('CSV file fetched sucessfully: %s' % csv_file)
except Exception as ex:
msg = 'failed to fetch GBIF file: {}'.format(str(ex))
LOGGER.exception(msg)
raise Exception(msg)
try:
response.update_status('read in latlon coordinates', 10)
latlon = sdm.latlon_gbifcsv(csv_file)
LOGGER.info('got occurence coordinates %s ' % csv_file)
except Exception as ex:
msg = 'failed to extract the latlon points from file {}: {}'.format(
csv_file, str(ex))
LOGGER.exception(msg)
raise Exception(msg)
try:
response.update_status('plot map', 20)
occurence_map = map_gbifoccurrences(latlon)
LOGGER.info('GBIF occourence ploted')
except Exception as ex:
msg = 'failed to plot occurence map: {}'.format(str(ex))
LOGGER.exception(msg)
raise Exception(msg)
#################################
# calculate the climate indices
#################################
# get the indices
try:
response.update_status('start calculation of indices', 30)
ncs_indices = sdm.get_indices(resource=resources, indices=indices)
LOGGER.info('indice calculation done')
except Exception as ex:
msg = 'failed to calculate indices: {}'.format(str(ex))
LOGGER.exception(msg)
raise Exception(msg)
try:
# sort indices
indices_dic = sdm.sort_indices(ncs_indices)
LOGGER.info('indice files sorted in dictionary')
except Exception as ex:
msg = 'failed to sort indices: {}'.format(str(ex))
LOGGER.exception(msg)
raise Exception(msg)
indices_dic = {'dummy': []}
ncs_references = []
species_files = []
stat_infos = []
PAmask_pngs = []
response.update_status('Start processing for {} datasets'.format(
len(indices_dic.keys())))
for count, key in enumerate(indices_dic.keys()):
try:
status_nr = 40 + count * 10
response.update_status('Start processing of {}'.format(key),
status_nr)
ncs = indices_dic[key]
LOGGER.info('with {} files'.format(len(ncs)))
try:
response.update_status('generating the PA mask', 20)
PAmask = sdm.get_PAmask(coordinates=latlon, nc=ncs[0])
LOGGER.info('PA mask sucessfully generated')
except Exception as ex:
msg = 'failed to generate the PA mask: {}'.format(str(ex))
LOGGER.exception(msg)
raise Exception(msg)
try:
response.update_status('Ploting PA mask', 25)
PAmask_pngs.extend([map_PAmask(PAmask)])
except Exception as ex:
msg = 'failed to plot the PA mask: {}'.format(str(ex))
LOGGER.exception(msg)
raise Exception(msg)
try:
ncs_reference = sdm.get_reference(ncs_indices=ncs,
period=period)
ncs_references.extend(ncs_reference)
LOGGER.info('reference indice calculated {}'.format(
ncs_references))
except Exception as ex:
msg = 'failed to calculate the reference: {}'.format(
str(ex))
LOGGER.exception(msg)
raise Exception(msg)
try:
gam_model, predict_gam, gam_info = sdm.get_gam(
ncs_reference, PAmask)
stat_infos.append(gam_info)
response.update_status('GAM sucessfully trained',
status_nr + 5)
except Exception as ex:
msg = 'failed to train GAM for {}: {}'.format(key, str(ex))
LOGGER.debug(msg)
raise Exception(msg)
try:
prediction = sdm.get_prediction(gam_model, ncs)
response.update_status('prediction done', status_nr + 7)
except Exception as ex:
msg = 'failed to predict tree occurence: {}'.format(
str(ex))
LOGGER.exception(msg)
raise Exception(msg)
#
# try:
# response.update_status('land sea mask for predicted data', status_nr + 8)
# from numpy import invert, isnan, nan, broadcast_arrays # , array, zeros, linspace, meshgrid
# mask = invert(isnan(PAmask))
# mask = broadcast_arrays(prediction, mask)[1]
# prediction[mask is False] = nan
# except:
# LOGGER.debug('failed to mask predicted data')
try:
species_files.append(sdm.write_to_file(ncs[0], prediction))
LOGGER.info('Favourability written to file')
except Exception as ex:
msg = 'failed to write species file: {}'.format(str(ex))
LOGGER.debug(msg)
raise Exception(msg)
except Exception as ex:
msg = 'failed to process SDM chain for {} : {}'.format(
key, str(ex))
LOGGER.exception(msg)
raise Exception(msg)
try:
archive_indices = archive(ncs_indices, format=archive_format)
LOGGER.info('indices added to archive')
except Exception as ex:
msg = 'failed adding indices to archive: {}'.format(str(ex))
LOGGER.exception(msg)
raise Exception(msg)
archive_indices = tempfile.mkstemp(suffix='.tar',
prefix='foobar-',
dir='.')
try:
archive_references = archive(ncs_references, format=archive_format)
LOGGER.info('indices reference added to archive')
except Exception as ex:
msg = 'failed adding reference indices to archive: {}'.format(
str(ex))
LOGGER.exception(msg)
raise Exception(msg)
archive_references = tempfile.mkstemp(suffix='.tar',
prefix='foobar-',
dir='.')
try:
archive_prediction = archive(species_files, format=archive_format)
LOGGER.info('species_files added to archive')
except Exception as ex:
msg = 'failed adding species_files indices to archive: {}'.format(
str(ex))
LOGGER.exception(msg)
raise Exception(msg)
try:
stat_infosconcat = pdfmerge(stat_infos)
LOGGER.debug('pngs {}'.format(PAmask_pngs))
PAmask_png = concat_images(PAmask_pngs, orientation='h')
LOGGER.info('stat infos pdfs and mask pngs merged')
except Exception as ex:
msg = 'failed to concat images: {}'.format(str(ex))
LOGGER.exception(msg)
raise Exception(msg)
_, stat_infosconcat = tempfile.mkstemp(suffix='.pdf',
prefix='foobar-',
dir='.')
_, PAmask_png = tempfile.mkstemp(suffix='.png',
prefix='foobar-',
dir='.')
# self.output_csv.setValue(csv_file)
response.outputs['output_gbif'].file = occurence_map
response.outputs['output_PA'].file = PAmask_png
response.outputs['output_indices'].file = archive_indices
response.outputs['output_reference'].file = archive_references
response.outputs['output_prediction'].file = archive_prediction
response.outputs['output_info'].file = stat_infosconcat
response.update_status('done', 100)
return response
def execute(self):
from os.path import basename
from flyingpigeon import sdm
from flyingpigeon.utils import archive, archiveextract # , get_domain
from flyingpigeon.visualisation import map_PAmask
init_process_logger('log.txt')
self.output_log.setValue('log.txt')
self.status.set('Start process', 0)
try:
self.status.set('reading the arguments', 5)
resources = archiveextract(
self.getInputValues(identifier='input_indices'))
csv_file = self.getInputValues(identifier='gbif')[0]
period = self.getInputValues(identifier='period')
period = period[0]
archive_format = self.archive_format.getValue()
except:
logger.error('failed to read in the arguments')
try:
self.status.set('read in latlon coordinates', 10)
latlon = sdm.latlon_gbifcsv(csv_file)
except:
logger.exception('failed to extract the latlon points')
try:
self.status.set('plot map', 20)
from flyingpigeon.visualisation import map_gbifoccurrences
# latlon = sdm.latlon_gbifdic(gbifdic)
occurence_map = map_gbifoccurrences(latlon)
except:
logger.exception('failed to plot occurence map')
# try:
# self.status.set('get domain', 30)
# domains = set()
# for indice in resources:
# # get_domain works only if metadata are set in a correct way
# domains = domains.union([basename(indice).split('_')[1]])
# if len(domains) == 1:
# domain = list(domains)[0]
# logger.info('Domain %s found in indices files' % domain)
# else:
# logger.warn('NOT a single domain in indices files %s' % domains)
# except:
# logger.exception('failed to get domains')
try:
# sort indices
indices_dic = sdm.sort_indices(resources)
logger.info('indice files sorted for %s Datasets' %
len(indices_dic.keys()))
except:
msg = 'failed to sort indices'
logger.exception(msg)
raise Exception(msg)
ncs_references = []
species_files = []
stat_infos = []
PAmask_pngs = []
self.status.set('Start processing for %s Datasets' %
len(indices_dic.keys()))
for count, key in enumerate(indices_dic.keys()):
try:
staus_nr = 40 + count * 10
self.status.set('Start processing of %s' % key, staus_nr)
ncs = indices_dic[key]
logger.info('with %s files' % len(ncs))
try:
self.status.set('generating the PA mask', 20)
PAmask = sdm.get_PAmask(coordinates=latlon, nc=ncs[0])
logger.info('PA mask sucessfully generated')
except:
logger.exception('failed to generate the PA mask')
try:
self.status.set('Ploting PA mask', 25)
PAmask_pngs.extend([map_PAmask(PAmask)])
except:
logger.exception('failed to plot the PA mask')
try:
ncs_reference = sdm.get_reference(ncs_indices=ncs,
period=period)
ncs_references.extend(ncs_reference)
logger.info('reference indice calculated %s ' %
ncs_references)
self.status.set('reference indice calculated',
staus_nr + 2)
except:
msg = 'failed to calculate the reference'
logger.exception(msg)
# raise Exception(msg)
try:
gam_model, predict_gam, gam_info = sdm.get_gam(
ncs_reference, PAmask, modelname=key)
stat_infos.append(gam_info)
self.status.set('GAM sucessfully trained', staus_nr + 5)
except:
msg = 'failed to train GAM for %s' % (key)
logger.exception(msg)
try:
prediction = sdm.get_prediction(gam_model, ncs)
self.status.set('prediction done', staus_nr + 7)
except:
msg = 'failed to predict tree occurence'
logger.exception(msg)
# raise Exception(msg)
# try:
# self.status.set('land sea mask for predicted data', staus_nr + 8)
# from numpy import invert, isnan, nan, broadcast_arrays # , array, zeros, linspace, meshgrid
# mask = invert(isnan(PAmask))
# mask = broadcast_arrays(prediction, mask)[1]
# prediction[mask is False] = nan
# except:
# logger.exception('failed to mask predicted data')
try:
species_files.append(sdm.write_to_file(ncs[0], prediction))
logger.info('Favourabillity written to file')
except:
msg = 'failed to write species file'
logger.exception(msg)
# raise Exception(msg)
except:
msg = 'failed to process SDM chain for %s ' % key
logger.exception(msg)
# raise Exception(msg)
try:
archive_references = None
archive_references = archive(ncs_references, format=archive_format)
logger.info('indices 2D added to archive')
except:
msg = 'failed adding 2D indices to archive'
logger.exception(msg)
raise Exception(msg)
archive_predicion = None
try:
archive_predicion = archive(species_files, format=archive_format)
logger.info('species_files added to archive')
except:
msg = 'failed adding species_files indices to archive'
logger.exception(msg)
raise Exception(msg)
try:
from flyingpigeon.visualisation import pdfmerge, concat_images
stat_infosconcat = pdfmerge(stat_infos)
logger.debug('pngs %s' % PAmask_pngs)
PAmask_png = concat_images(PAmask_pngs, orientation='h')
logger.info('stat infos pdfs and mask pngs merged')
except:
logger.exception('failed to concat images')
_, stat_infosconcat = tempfile.mkstemp(suffix='.pdf',
prefix='foobar-',
dir='.')
_, PAmask_png = tempfile.mkstemp(suffix='.png',
prefix='foobar-',
dir='.')
self.output_gbif.setValue(occurence_map)
self.output_PA.setValue(PAmask_png)
self.output_reference.setValue(archive_references)
self.output_prediction.setValue(archive_predicion)
self.output_info.setValue(stat_infosconcat)
self.status.set('done', 100)
def execute(self):
init_process_logger('log.txt')
self.output_log.setValue('log.txt')
from os.path import basename
from flyingpigeon import sdm
from flyingpigeon.utils import archive, archiveextract
self.status.set('Start process', 0)
try:
logger.info('reading the arguments')
resources = archiveextract(
self.getInputValues(identifier='resources'))
taxon_name = self.getInputValues(identifier='taxon_name')[0]
bbox_obj = self.BBox.getValue()
bbox = [
bbox_obj.coords[0][0], bbox_obj.coords[0][1],
bbox_obj.coords[1][0], bbox_obj.coords[1][1]
]
period = self.getInputValues(identifier='period')
period = period[0]
indices = self.getInputValues(identifier='input_indices')
archive_format = self.archive_format.getValue()
logger.debug("indices = %s for %s ", indices, taxon_name)
logger.info("bbox={0}".format(bbox))
except Exception as e:
logger.error('failed to read in the arguments %s ' % e)
logger.info('indices %s ' % indices)
try:
self.status.set('Fetching GBIF Data', 10)
gbifdic = sdm.get_gbif(taxon_name, bbox=bbox)
except Exception as e:
msg = 'failed to search gbif.'
logger.exception(msg)
raise Exception(msg)
try:
self.status.set('write csv file', 70)
gbifcsv = sdm.gbifdic2csv(gbifdic)
except Exception as e:
msg = 'failed to write csv file.'
logger.exception(msg)
raise Exception(msg)
try:
self.status.set('plot map', 80)
from flyingpigeon.visualisation import map_gbifoccurrences
latlon = sdm.latlon_gbifdic(gbifdic)
occurence_map = map_gbifoccurrences(latlon)
except Exception as e:
msg = 'failed to plot occurence map.'
logger.exception(msg)
raise Exception(msg)
#################################
# calculate the climate indices
#################################
# get the indices
ncs_indices = None
try:
self.status.set(
'start calculation of climate indices for %s' % indices, 30)
ncs_indices = sdm.get_indices(resources=resources, indices=indices)
logger.info('indice calculation done')
except:
msg = 'failed to calculate indices'
logger.exception(msg)
raise Exception(msg)
try:
self.status.set('get domain', 30)
domains = set()
for resource in ncs_indices:
# get_domain works only if metadata are set in a correct way
domains = domains.union([basename(resource).split('_')[1]])
if len(domains) == 1:
domain = list(domains)[0]
logger.debug('Domain %s found in indices files' % domain)
else:
logger.error('Not a single domain in indices files %s' %
domains)
except Exception as e:
logger.exception('failed to get domains %s' % e)
try:
self.status.set('generating the PA mask', 20)
PAmask = sdm.get_PAmask(coordinates=latlon, domain=domain)
logger.info('PA mask sucessfully generated')
except Exception as e:
logger.exception('failed to generate the PA mask: %s' % e)
try:
self.status.set('Ploting PA mask', 25)
from flyingpigeon.visualisation import map_PAmask
PAmask_png = map_PAmask(PAmask)
except Exception as e:
logger.exception('failed to plot the PA mask: %s' % e)
try:
# sort indices
indices_dic = None
indices_dic = sdm.sort_indices(ncs_indices)
logger.info('indice files sorted for %s Datasets' %
len(indices_dic.keys()))
except:
msg = 'failed to sort indices'
logger.exception(msg)
raise Exception(msg)
ncs_references = []
species_files = []
stat_infos = []
for count, key in enumerate(indices_dic.keys()):
try:
staus_nr = 40 + count * 10
self.status.set('Start processing of %s' % key, staus_nr)
ncs = indices_dic[key]
logger.info('with %s files' % len(ncs))
try:
ncs_reference = sdm.get_reference(ncs_indices=ncs,
period=period)
ncs_references.extend(ncs_reference)
logger.info('reference indice calculated %s ' %
ncs_references)
except:
msg = 'failed to calculate the reference'
logger.exception(msg)
raise Exception(msg)
try:
gam_model, predict_gam, gam_info = sdm.get_gam(
ncs_reference, PAmask)
stat_infos.append(gam_info)
self.status.set('GAM sucessfully trained', staus_nr + 5)
except Exception as e:
msg = 'failed to train GAM for %s : %s' % (key, e)
logger.debug(msg)
try:
prediction = sdm.get_prediction(gam_model, ncs)
self.status.set('prediction done', staus_nr + 7)
except Exception as e:
msg = 'failed to predict tree occurence %s' % e
logger.exception(msg)
# raise Exception(msg)
try:
self.status.set('land sea mask for predicted data',
staus_nr + 8)
from numpy import invert, isnan, nan, broadcast_arrays # , array, zeros, linspace, meshgrid
mask = invert(isnan(PAmask))
mask = broadcast_arrays(prediction, mask)[1]
prediction[mask is False] = nan
except Exception as e:
logger.debug('failed to mask predicted data: %s' % e)
try:
species_files.append(sdm.write_to_file(ncs[0], prediction))
logger.info('Favourabillity written to file')
except Exception as e:
msg = 'failed to write species file %s' % e
logger.debug(msg)
# raise Exception(msg)
except Exception as e:
msg = 'failed to calculate reference indices. %s ' % e
logger.exception(msg)
raise Exception(msg)
try:
archive_indices = None
archive_indices = archive(ncs_indices, format=archive_format)
logger.info('indices added to archive')
except:
msg = 'failed adding indices to archive'
logger.exception(msg)
raise Exception(msg)
archive_references = None
try:
archive_references = archive(ncs_references, format=archive_format)
logger.info('indices reference added to archive')
except:
msg = 'failed adding reference indices to archive'
logger.exception(msg)
raise Exception(msg)
archive_predicion = None
try:
archive_predicion = archive(species_files, format=archive_format)
logger.info('species_files added to archive')
except:
msg = 'failed adding species_files indices to archive'
logger.exception(msg)
raise Exception(msg)
try:
from flyingpigeon.visualisation import pdfmerge
stat_infosconcat = pdfmerge(stat_infos)
logger.info('stat infos pdfs merged')
except:
logger.exception('failed to concat images')
_, stat_infosconcat = tempfile.mkstemp(suffix='.pdf',
prefix='foobar-',
dir='.')
self.output_csv.setValue(gbifcsv)
self.output_gbif.setValue(occurence_map)
self.output_PA.setValue(PAmask_png)
self.output_indices.setValue(archive_indices)
self.output_reference.setValue(archive_references)
self.output_prediction.setValue(archive_predicion)
self.output_info.setValue(stat_infosconcat)
self.status.set('done', 100)
def execute(self):
from os.path import basename
from flyingpigeon import sdm
from flyingpigeon.utils import archive
self.status.set('Start process', 0)
try:
logger.info('reading the arguments')
resources = self.getInputValues(identifier='resources')
taxon_name = self.getInputValues(identifier='taxon_name')[0]
#period = self.period.getValue()
period = self.getInputValues(identifier='period')
period = period[0]
#indices = self.input_indices.getValue()
indices = self.getInputValues(identifier='input_indices')
logger.debug("indices = %s for %s ", indices, taxon_name)
archive_format = self.archive_format.getValue()
except Exception as e:
logger.error('failed to read in the arguments %s ' % e)
logger.info('indices %s ' % indices)
try:
self.status.set('Fetching GBIF Data', 10)
latlon = sdm.gbif_serach(taxon_name)
except Exception as e:
logger.exception('failed to search gbif %s' % e)
#try:
#self.status.set('extract csv file with tree observations', 5)
#csv_file = sdm.get_csv(taxon_name[0])
#except Exception as e:
#logger.exception('failed to extract csv file from url.')
#try:
#self.status.set('read in latlon coordinates of tree observations', 10)
#latlon = sdm.get_latlon(csv_file)
#except Exception as e:
#logger.exception('failed to extract the latlon points')
try:
from flyingpigeon.visualisation import map_gbifoccurrences
self.status.set('plotting Tree presents based on coordinates', 15)
tree_presents = map_gbifoccurrences(latlon)
except Exception as e:
msg = 'plotting points failed'
logger.exception(msg)
with open(tree_presents, 'w') as fp:
# TODO: needs to be a png file
fp.write(msg)
try:
self.status.set('generating the PA mask', 20)
PAmask = sdm.get_PAmask(coordinates=latlon)
logger.info('PA mask sucessfully generated')
except Exception as e:
logger.exception('failed to generate the PA mask')
png_PA_mask = 'PA_mask.png'
try:
import matplotlib.pyplot as plt
self.status.set('Ploting PA mask', 25)
fig = plt.figure(figsize=(20,10), dpi=300, facecolor='w', edgecolor='k')
cs = plt.contourf(PAmask)
fig.savefig(png_PA_mask)
plt.close()
except Exception as e:
msg = 'failed to plot the PA mask'
logger.exception(msg)
with open(png_PA_mask, 'w') as fp:
# TODO: needs to be a png file
fp.write(msg)
#################################
### calculate the climate indices
#################################
# get the indices
ncs_indices = None
try:
self.status.set('start calculation of climate indices for %s' % indices, 30 )
ncs_indices = sdm.get_indices(resources=resources, indices=indices)
logger.info('indice calculation done')
except:
msg = 'failed to calculate indices'
logger.exception(msg)
raise Exception(msg)
try:
archive_indices = archive(ncs_indices , format=archive_format)
logger.info('indices 3D added to tarfile')
except:
msg = 'failed adding indices to tar'
logger.exception(msg)
raise Exception(msg)
indices_dic = None
try:
# sort indices
indices_dic = sdm.sort_indices(ncs_indices)
logger.info('indice files sorted for %s Datasets' % len(indices_dic.keys()))
except:
msg = 'failed to sort indices'
logger.exception(msg)
raise Exception(msg)
# try:
# # open tar files
# tar_reference = tarfile.open('reference.tar', "w")
# tar_indices = tarfile.open('indices.tar', "w")
# tar_info = tarfile.open('info.tar', "w")
# tar_prediction = tarfile.open('prediction.tar', "w")
# logger.info('tar files prepared')
# except:
# msg = 'tar file preparation failed'
# logger.exception(msg)
# raise Exception(msg)
ncs_references = []
species_files = []
statistics_info = []
for count,key in enumerate(indices_dic.keys()):
try:
self.status.set('Start processing of %s ' % key, 40 + count * 10)
ncs = indices_dic[key]
logger.info('with %s files' % len(ncs))
try:
ncs_references.extend(sdm.get_reference(ncs_indices=ncs, period=period))
logger.info('reference indice calculated %s ' % ncs_references)
except:
msg = 'failed adding ref indices to tar'
logger.exception(msg)
raise Exception(msg)
# for nc_reference in ncs_references:
# tar_reference.add(nc_reference,
# arcname = basename(nc_reference))# nc_reference.replace(os.path.abspath(os.path.curdir), ""))
# logger.info('reference indices added to tarfile')
except:
msg = 'failed to calculate reference indices.'
logger.exception(msg)
raise Exception(msg)
try:
gam_model, predict_gam, gam_info = sdm.get_gam(ncs_references,PAmask)
statistics_info.append(gam_info)
self.status.set('GAM sucessfully trained', 70)
except:
msg = 'failed to train GAM'
logger.exception(msg)
raise Exception(msg)
try:
prediction = sdm.get_prediction(gam_model, ncs_indices)
self.status.set('prediction done', 80)
except:
msg = 'failed to predict'
logger.exception(msg)
raise Exception(msg)
try:
from numpy import invert, isnan, nan, broadcast_arrays, array, zeros, linspace, meshgrid
mask = invert(isnan(PAmask))
mask = broadcast_arrays(prediction, mask)[1]
prediction[mask==False] = nan
self.status.set('land sea mask for predicted data', 90)
except:
logger.exception('failed to mask predicted data')
try:
species_files.append(sdm.write_to_file(ncs_indices[0], prediction))
logger.info('Favourabillity written to file')
#tar_prediction.add(species_file,
# arcname = basename(species_file))#.replace(os.path.abspath(os.path.curdir), ""))
except:
msg = 'failed to write species file'
logger.exception(msg)
raise Exception(msg)
from flyingpigeon.visualisation import concat_images
statistics_infos = None
try:
statistics_infos = concat_images(statistics_info, orientation='v')
except:
msg = 'failed to concat images'
logger.exception(msg)
raise Exception(msg)
archive_references = None
try:
archive_references = archive(ncs_references , format=archive_format)
logger.info('indices 2D added to archive')
except:
msg = 'failed adding 2D indices to archive'
logger.exception(msg)
raise Exception(msg)
archive_predicion = None
try:
archive_predicion = archive(species_files , format=archive_format)
logger.info('species_files added to archive')
except:
msg = 'failed adding species_files indices to archive'
logger.exception(msg)
raise Exception(msg)
# try:
# #tar_indices.close()
# #tar_reference.close()
# tar_prediction.close()
# #tar_info.close()
# logger.info('tar files closed')
# except:
# logger.exception('tar file closing failed')
# raise Exception
#self.output_csv.setValue( csv_file )
self.output_gbif.setValue( tree_presents )
self.output_PA.setValue( png_PA_mask )
self.output_indices.setValue( archive_indices )
self.output_reference.setValue (archive_references)
self.output_prediction.setValue (archive_predicion)
self.output_info.setValue(statistics_infos)
self.status.set('done', 100)
ncs_references = ['/home/nils/data/sdm/RX1day_EUR-11_MPI-M-MPI-ESM-LR_historical_r1i1p1_CLMcom-CCLM4-8-17_v1_JJA_ref-1998-2000.nc','/home/nils/data/sdm/TG_EUR-11_MPI-M-MPI-ESM-LR_historical_r1i1p1_CLMcom-CCLM4-8-17_v1_JJA_ref-1998-2000.nc','/home/nils/data/sdm/TNn_EUR-11_MPI-M-MPI-ESM-LR_historical_r1i1p1_CLMcom-CCLM4-8-17_v1_Jan_ref-1998-2000.nc']
ncs_references = ['/home/nils/.conda/envs/birdhouse/etc/pywps/TNn_EUR-11_CNRM-CERFACS-CNRM-CM5_historical_r1i1p1_CLMcom-CCLM4-8-17_v1_Jan_ref-all.nc', '/home/nils/.conda/envs/birdhouse/etc/pywps/TG_EUR-11_CNRM-CERFACS-CNRM-CM5_historical_r1i1p1_CLMcom-CCLM4-8-17_v1_JJA_ref-all.nc', '/home/nils/.conda/envs/birdhouse/etc/pywps/PRCPTOT_EUR-11_CNRM-CERFACS-CNRM-CM5_historical_r1i1p1_CLMcom-CCLM4-8-17_v1_JJA_ref-all.nc']
ncs_indices = ['/home/nils/data/sdm/TG_EUR-11_CNRM-CERFACS-CNRM-CM5_historical_r1i1p1_CLMcom-CCLM4-8-17_v1_JJA_20010101-20050101.nc',
'/home/nils/data/sdm/TG_EUR-11_ICHEC-EC-EARTH_historical_r12i1p1_CLMcom-CCLM4-8-17_v1_JJA_20010101-20050101.nc',
'/home/nils/data/sdm/TG_EUR-11_ICHEC-EC-EARTH_historical_r1i1p1_KNMI-RACMO22E_v1_JJA_20010101-20050101.nc',
'/home/nils/data/sdm/TG_EUR-11_MOHC-HadGEM2-ES_historical_r1i1p1_CLMcom-CCLM4-8-17_v1_JJA_20010101-20050101.nc',
'/home/nils/data/sdm/TG_EUR-11_MOHC-HadGEM2-ES_historical_r1i1p1_KNMI-RACMO22E_v1_JJA_20010101-20050101.nc']
csv = '/home/nils/Downloads/0013848-160118175350007.csv'
latlon = sdm.get_latlon(csv)
PApoints = sdm.get_PAmask(coordinates=latlon)
gam_model, predict_ref, gam_info = sdm.get_gam(ncs_references,PApoints)
print gam_model.names
import matplotlib.pyplot as plt
for i in range(len(gam_model)):
try:
plt.plot(gam_model[i])
plt.show()
except:
print 'failed for' , i
prediction = sdm.get_prediction(gam_model, ncs_indices)
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
response.update_status('Start process', 0)
try:
LOGGER.info('reading the arguments')
resources = archiveextract(
resource=rename_complexinputs(request.inputs['resource']))
taxon_name = request.inputs['taxon_name'][0].data
bbox = [-180, -90, 180, 90]
# bbox_obj = self.BBox.getValue()
# bbox = [bbox_obj.coords[0][0],
# bbox_obj.coords[0][1],
# bbox_obj.coords[1][0],
# bbox_obj.coords[1][1]]
period = request.inputs['period']
period = period[0].data
indices = [inpt.data for inpt in request.inputs['indices']]
archive_format = request.inputs['archive_format'][0].data
LOGGER.exception("indices = %s for %s ", indices, taxon_name)
LOGGER.info("bbox={0}".format(bbox))
except:
LOGGER.exception('failed to read in the arguments')
LOGGER.info('indices %s ' % indices)
try:
response.update_status('Fetching GBIF Data', 10)
gbifdic = sdm.get_gbif(taxon_name, bbox=bbox)
LOGGER.info('Fetched GBIF data')
except:
msg = 'failed to search gbif.'
LOGGER.exception(msg)
raise Exception(msg)
try:
response.update_status('write csv file', 70)
gbifcsv = sdm.gbifdic2csv(gbifdic)
LOGGER.info('GBIF data written to file')
except:
msg = 'failed to write csv file.'
LOGGER.exception(msg)
raise Exception(msg)
try:
response.update_status('plot map', 80)
latlon = sdm.latlon_gbifdic(gbifdic)
occurence_map = map_gbifoccurrences(latlon)
except:
msg = 'failed to plot occurence map.'
LOGGER.exception(msg)
raise Exception(msg)
#################################
# calculate the climate indices
#################################
# get the indices
ncs_indices = None
try:
response.update_status(
'start calculation of climate indices for %s' % indices, 30)
ncs_indices = sdm.get_indices(resource=resources, indices=indices)
LOGGER.info('indice calculation done')
except:
msg = 'failed to calculate indices'
LOGGER.exception(msg)
raise Exception(msg)
try:
# sort indices
indices_dic = sdm.sort_indices(ncs_indices)
LOGGER.info('indice files sorted in dictionary')
except:
msg = 'failed to sort indices'
LOGGER.exception(msg)
indices_dic = {'dummy': []}
ncs_references = []
species_files = []
stat_infos = []
PAmask_pngs = []
response.update_status('Start processing for %s Datasets' %
len(indices_dic.keys()))
for count, key in enumerate(indices_dic.keys()):
try:
staus_nr = 40 + count * 10
response.update_status('Start processing of %s' % key,
staus_nr)
ncs = indices_dic[key]
LOGGER.info('with %s files' % len(ncs))
try:
response.update_status('generating the PA mask', 20)
PAmask = sdm.get_PAmask(coordinates=latlon, nc=ncs[0])
LOGGER.info('PA mask sucessfully generated')
except:
LOGGER.exception('failed to generate the PA mask')
try:
response.update_status('Ploting PA mask', 25)
PAmask_pngs.extend([map_PAmask(PAmask)])
except:
LOGGER.exception('failed to plot the PA mask')
try:
ncs_reference = sdm.get_reference(ncs_indices=ncs,
period=period)
ncs_references.extend(ncs_reference)
LOGGER.info('reference indice calculated %s ' %
ncs_references)
except:
msg = 'failed to calculate the reference'
LOGGER.exception(msg)
try:
gam_model, predict_gam, gam_info = sdm.get_gam(
ncs_reference, PAmask)
stat_infos.append(gam_info)
response.update_status('GAM sucessfully trained',
staus_nr + 5)
except:
msg = 'failed to train GAM for %s' % (key)
LOGGER.exception(msg)
try:
prediction = sdm.get_prediction(gam_model, ncs)
response.update_status('prediction done', staus_nr + 7)
except:
msg = 'failed to predict tree occurence'
LOGGER.exception(msg)
# raise Exception(msg)
# try:
# response.update_status('land sea mask for predicted data', staus_nr + 8)
# from numpy import invert, isnan, nan, broadcast_arrays # , array, zeros, linspace, meshgrid
# mask = invert(isnan(PAmask))
# mask = broadcast_arrays(prediction, mask)[1]
# prediction[mask is False] = nan
# except:
# LOGGER.exception('failed to mask predicted data')
try:
species_files.append(sdm.write_to_file(ncs[0], prediction))
LOGGER.info('Favourabillity written to file')
except:
msg = 'failed to write species file'
LOGGER.exception(msg)
# raise Exception(msg)
except:
msg = 'failed to calculate reference indices'
LOGGER.exception(msg)
raise Exception(msg)
try:
archive_indices = archive(ncs_indices, format=archive_format)
LOGGER.info('indices added to archive')
except:
msg = 'failed adding indices to archive'
LOGGER.exception(msg)
archive_indices = tempfile.mkstemp(suffix='.tar',
prefix='foobar-',
dir='.')
try:
archive_references = archive(ncs_references, format=archive_format)
LOGGER.info('indices reference added to archive')
except:
msg = 'failed adding reference indices to archive'
LOGGER.exception(msg)
archive_references = tempfile.mkstemp(suffix='.tar',
prefix='foobar-',
dir='.')
try:
archive_prediction = archive(species_files, format=archive_format)
LOGGER.info('species_files added to archive')
except:
msg = 'failed adding species_files indices to archive'
LOGGER.exception(msg)
raise Exception(msg)
try:
stat_infosconcat = pdfmerge(stat_infos)
LOGGER.debug('pngs %s' % PAmask_pngs)
PAmask_png = concat_images(PAmask_pngs, orientation='h')
LOGGER.info('stat infos pdfs and mask pngs merged')
except:
LOGGER.exception('failed to concat images')
_, stat_infosconcat = tempfile.mkstemp(suffix='.pdf',
prefix='foobar-',
dir='.')
_, PAmask_png = tempfile.mkstemp(suffix='.png',
prefix='foobar-',
dir='.')
response.outputs['output_gbif'].file = occurence_map
response.outputs['output_PA'].file = PAmask_png
response.outputs['output_indices'].file = archive_indices
response.outputs['output_reference'].file = archive_references
response.outputs['output_prediction'].file = archive_prediction
response.outputs['output_info'].file = stat_infosconcat
response.outputs['output_csv'].file = gbifcsv
response.update_status('done', 100)
return response
def execute(self):
from os.path import basename
from flyingpigeon import sdm
from flyingpigeon.utils import archive
self.status.set('Start process', 0)
try:
logger.info('reading the arguments')
resources = self.getInputValues(identifier='resources')
gbif = self.getInputValues(identifier='gbif')
#period = self.period.getValue()
period = self.getInputValues(identifier='period')
period = period[0]
#indices = self.input_indices.getValue()
indices = self.getInputValues(identifier='input_indices')
logger.debug("indices = %s", indices)
archive_format = self.archive_format.getValue()
except Exception as e:
logger.error('failed to read in the arguments %s ' % e)
logger.info('indices %s ' % indices)
try:
self.status.set('extract csv file with tree observations', 5)
csv_file = sdm.get_csv(gbif[0])
except Exception as e:
logger.exception('failed to extract csv file from url.')
try:
self.status.set('read in latlon coordinates of tree observations',
10)
latlon = sdm.get_latlon(csv_file)
except Exception as e:
logger.exception('failed to extract the latlon points')
tree_presents = 'tree_presents.png'
try:
self.status.set('plotting Tree presents based on coordinates', 15)
import matplotlib.pyplot as plt
from cartopy import config
from cartopy.util import add_cyclic_point
import cartopy.crs as ccrs
fig = plt.figure(figsize=(20, 10),
dpi=600,
facecolor='w',
edgecolor='k')
ax = plt.axes(projection=ccrs.Robinson(central_longitude=0))
ax.coastlines()
ax.set_global()
cs = plt.scatter(latlon[:, 1],
latlon[:, 0],
transform=ccrs.PlateCarree())
fig.savefig(tree_presents)
plt.close()
except Exception as e:
msg = 'plotting points failed'
logger.exception(msg)
with open(tree_presents, 'w') as fp:
# TODO: needs to be a png file
fp.write(msg)
try:
self.status.set('generating the PA mask', 20)
PAmask = sdm.get_PAmask(coordinates=latlon)
logger.info('PA mask sucessfully generated')
except Exception as e:
logger.exception('failed to generate the PA mask')
png_PA_mask = 'PA_mask.png'
try:
self.status.set('Ploting PA mask', 25)
fig = plt.figure(figsize=(20, 10),
dpi=300,
facecolor='w',
edgecolor='k')
cs = plt.contourf(PAmask)
fig.savefig(png_PA_mask)
plt.close()
except Exception as e:
msg = 'failed to plot the PA mask'
logger.exception(msg)
with open(png_PA_mask, 'w') as fp:
# TODO: needs to be a png file
fp.write(msg)
#################################
### calculate the climate indices
#################################
# get the indices
ncs_indices = None
try:
self.status.set(
'start calculation of climate indices for %s' % indices, 30)
ncs_indices = sdm.get_indices(resources=resources, indices=indices)
logger.info('indice calculation done')
except:
msg = 'failed to calculate indices'
logger.exception(msg)
raise Exception(msg)
try:
archive_indices = archive(ncs_indices, format=archive_format)
logger.info('indices 3D added to tarfile')
except:
msg = 'failed adding indices to tar'
logger.exception(msg)
raise Exception(msg)
indices_dic = None
try:
# sort indices
indices_dic = sdm.sort_indices(ncs_indices)
logger.info('indice files sorted for %s Datasets' %
len(indices_dic.keys()))
except:
msg = 'failed to sort indices'
logger.exception(msg)
raise Exception(msg)
# try:
# # open tar files
# tar_reference = tarfile.open('reference.tar', "w")
# tar_indices = tarfile.open('indices.tar', "w")
# tar_info = tarfile.open('info.tar', "w")
# tar_prediction = tarfile.open('prediction.tar', "w")
# logger.info('tar files prepared')
# except:
# msg = 'tar file preparation failed'
# logger.exception(msg)
# raise Exception(msg)
ncs_references = []
species_files = []
statistics_info = []
for count, key in enumerate(indices_dic.keys()):
try:
self.status.set('Start processing of %s ' % key,
40 + count * 10)
ncs = indices_dic[key]
logger.info('with %s files' % len(ncs))
try:
ncs_references.extend(
sdm.get_reference(ncs_indices=ncs, period=period))
logger.info('reference indice calculated %s ' %
ncs_references)
except:
msg = 'failed adding ref indices to tar'
logger.exception(msg)
raise Exception(msg)
# for nc_reference in ncs_references:
# tar_reference.add(nc_reference,
# arcname = basename(nc_reference))# nc_reference.replace(os.path.abspath(os.path.curdir), ""))
# logger.info('reference indices added to tarfile')
except:
msg = 'failed to calculate reference indices.'
logger.exception(msg)
raise Exception(msg)
try:
gam_model, predict_gam, gam_info = sdm.get_gam(
ncs_references, PAmask)
statistics_info.append(gam_info)
self.status.set('GAM sucessfully trained', 70)
except:
msg = 'failed to train GAM'
logger.exception(msg)
raise Exception(msg)
try:
prediction = sdm.get_prediction(gam_model, ncs_indices)
self.status.set('prediction done', 80)
except:
msg = 'failed to predict'
logger.exception(msg)
raise Exception(msg)
try:
from numpy import invert, isnan, nan, broadcast_arrays, array, zeros, linspace, meshgrid
mask = invert(isnan(PAmask))
mask = broadcast_arrays(prediction, mask)[1]
prediction[mask == False] = nan
self.status.set('land sea mask for predicted data', 90)
except:
logger.exception('failed to mask predicted data')
try:
species_files.append(
sdm.write_to_file(ncs_indices[0], prediction))
logger.info('Favourabillity written to file')
#tar_prediction.add(species_file,
# arcname = basename(species_file))#.replace(os.path.abspath(os.path.curdir), ""))
except:
msg = 'failed to write species file'
logger.exception(msg)
raise Exception(msg)
from flyingpigeon.visualisation import concat_images
statistics_infos = None
try:
statistics_infos = concat_images(statistics_info, orientation='v')
except:
msg = 'failed to concat images'
logger.exception(msg)
raise Exception(msg)
archive_references = None
try:
archive_references = archive(ncs_references, format=archive_format)
logger.info('indices 2D added to archive')
except:
msg = 'failed adding 2D indices to archive'
logger.exception(msg)
raise Exception(msg)
archive_predicion = None
try:
archive_predicion = archive(species_files, format=archive_format)
logger.info('species_files added to archive')
except:
msg = 'failed adding species_files indices to archive'
logger.exception(msg)
raise Exception(msg)
# try:
# #tar_indices.close()
# #tar_reference.close()
# tar_prediction.close()
# #tar_info.close()
# logger.info('tar files closed')
# except:
# logger.exception('tar file closing failed')
# raise Exception
self.output_csv.setValue(csv_file)
self.output_gbif.setValue(tree_presents)
self.output_PA.setValue(png_PA_mask)
self.output_indices.setValue(archive_indices)
self.output_reference.setValue(archive_references)
self.output_prediction.setValue(archive_predicion)
self.output_info.setValue(statistics_infos)
self.status.set('done', 100)
return prediction tic = dt.now() indices = '/home/nils/data/sdm/tmpTzDdv7.tar' gbif = '/home/nils/data/sdm/acacia_albida.csv' gbif_url = 'https://bovec.dkrz.de/download/wpsoutputs/flyingpigeon/392f1c34-b4d1-11e7-a589-109836a7cf3a/tmp95yvix.csv' latlon = sdm.latlon_gbifcsv(gbif) latlon occurence_map = map_gbifoccurrences(latlon) occurence_map ncs = archiveextract(indices) ncs indices_dic = sdm.sort_indices(ncs) indices_dic PAmask = sdm.get_PAmask(coordinates=latlon, nc=ncs[0]) PAmask # PAmask_pngs.extend([map_PAmask(PAmask)]) map_PAmask(PAmask) ncs_reference = sdm.get_reference(ncs_indices=ncs) ncs_reference gam_model, predict_gam, gam_info = sdm.get_gam(ncs_reference, PAmask, modelname='TG_AFR-44_CNRM-CERFACS-CNRM-CM5_historical_r1i1p1_CLMcom-CCLM4-8-17_v1') prediction = get_prediction(gam_model, ncs)
def execute(self):
from flyingpigeon import sdm
self.status.set('Start process', 0)
try:
logger.info('read in the arguments')
resources = self.getInputValues(identifier='resources')
gbif = self.getInputValues(identifier='gbif')
period = self.getInputValues(identifier='period')
period = period[0]
indices = self.getInputValues(identifier='indices')
if 'all' in indices:
indices = ['TG_JJA', 'TNn_Jan'] # 'PRCPTOT_JJA'
except Exception as e:
logger.error('failed to read in the arguments %s ' % e)
logger.info('indices %s ' % indices)
try:
self.status.set('extract csv file with tree observations', 5)
csv_file = sdm.get_csv(gbif[0])
except Exception as e:
logger.exception('failed to extract csv file from url.')
try:
self.status.set('read in latlon coordinates of tree observations', 10)
latlon = sdm.get_latlon(csv_file)
except Exception as e:
logger.exception('failed to extract the latlon points')
tree_presents = 'tree_presents.png'
try:
self.status.set('plotting Tree presents based on coordinates', 15)
import matplotlib.pyplot as plt
from cartopy import config
from cartopy.util import add_cyclic_point
import cartopy.crs as ccrs
fig = plt.figure(figsize=(20,10), dpi=600, facecolor='w', edgecolor='k')
ax = plt.axes(projection=ccrs.Robinson(central_longitude=0))
ax.coastlines()
ax.set_global()
cs = plt.scatter(latlon[:,1], latlon[:,0], transform=ccrs.PlateCarree())
fig.savefig(tree_presents)
plt.close()
except Exception as e:
msg = 'plotting points failed'
logger.exception(msg)
with open(tree_presents, 'w') as fp:
# TODO: needs to be a png file
fp.write(msg)
try:
self.status.set('generating the PA mask', 20)
PAmask = sdm.get_PAmask(coordinates=latlon)
logger.info('PA mask sucessfully generated')
except Exception as e:
logger.exception('failed to generate the PA mask')
png_PA_mask = 'PA_mask.png'
try:
self.status.set('Ploting PA mask', 25)
fig = plt.figure(figsize=(20,10), dpi=300, facecolor='w', edgecolor='k')
cs = plt.contourf(PAmask)
fig.savefig(png_PA_mask)
plt.close()
except Exception as e:
msg = 'failed to plot the PA mask'
logger.exception(msg)
with open(png_PA_mask, 'w') as fp:
# TODO: needs to be a png file
fp.write(msg)
#################################
### calculate the climate inidces
#################################
# get the indices
ncs_indices = None
try:
self.status.set('start calculation of climate indices for %s' % indices, 30 )
ncs_indices = sdm.get_indices(resources=resources, indices=indices)
logger.info('indice calculation done')
except:
msg = 'failed to calculate indices'
logger.exception(msg)
raise Exception(msg)
indices_dic = None
try:
# sort indices
indices_dic = sdm.sort_indices(ncs_indices)
logger.info('indice files sorted for %s Datasets' % len(indices_dic.keys()))
except:
msg = 'failed to sort indices'
logger.exception(msg)
raise Exception(msg)
try:
# open tar files
tar_reference = tarfile.open('reference.tar', "w")
tar_indices = tarfile.open('indices.tar', "w")
tar_info = tarfile.open('info.tar', "w")
tar_prediction = tarfile.open('prediction.tar', "w")
logger.info('tar files prepared')
except:
msg = 'tar file preparation failed'
logger.exception(msg)
raise Exception(msg)
for count,key in enumerate(indices_dic.keys()):
try:
self.status.set('Start processing of %s ' % key, 40 + count * 10)
ncs = indices_dic[key]
logger.info('with %s files' % len(ncs))
try:
for nc in ncs:
tar_indices.add(nc,
arcname = basename(nc) )# .replace(os.path.abspath(os.path.curdir), ""))
logger.info('indices added to tarfile for %s' % key)
except:
msg = 'failed adding indices to tar'
logger.exception(msg)
raise Exception(msg)
try:
ncs_references = sdm.get_reference(ncs_indices=ncs, period=period)
logger.info('reference indice calculated %s ' % ncs_references)
except:
msg = 'failed adding ref indices to tar'
logger.exception(msg)
raise Exception(msg)
for nc_reference in ncs_references:
tar_reference.add(nc_reference,
arcname = basename(nc_reference))# nc_reference.replace(os.path.abspath(os.path.curdir), ""))
logger.info('reference indices added to tarfile')
except:
msg = 'failed to calculate reference indices.'
logger.exception(msg)
raise Exception(msg)
try:
gam_model, predict_gam, gam_info = sdm.get_gam(ncs_references,PAmask)
tar_info.add(gam_info, arcname = "%s.pdf" % key)
self.status.set('GAM sucessfully trained', 70)
except:
msg = 'failed to train GAM'
logger.exception(msg)
raise Exception(msg)
try:
prediction = sdm.get_prediction(gam_model, ncs_indices)
self.status.set('prediction done', 80)
except:
msg = 'failed to predict'
logger.exception(msg)
raise Exception(msg)
try:
from numpy import invert, isnan, nan, broadcast_arrays, array, zeros, linspace, meshgrid
mask = invert(isnan(PApoints))
mask = broadcast_arrays(prediction, mask)[1]
prediction[mask==False] = nan
self.status.set('land sea mask for predicted data', 90)
except:
logger.exception('failed to mask predicted data')
try:
species_file = sdm.write_to_file(ncs_indices[0], prediction)
logger.info('Favourabillity written to file')
tar_prediction.add(species_file,
arcname = basename(species_file))#.replace(os.path.abspath(os.path.curdir), ""))
except:
msg = 'failed to write species file'
logger.exception(msg)
raise Exception(msg)
try:
tar_indices.close()
tar_reference.close()
tar_prediction.close()
tar_info.close()
logger.info('tar files closed')
except:
logger.exception('tar file closing failed')
raise Exception
self.output_csv.setValue( csv_file )
self.output_gbif.setValue( tree_presents )
self.output_PA.setValue( png_PA_mask )
self.output_indices.setValue( 'indices.tar' )
self.output_reference.setValue ('reference.tar')
self.output_prediction.setValue ('prediction.tar')
self.output_info.setValue('info.tar')
self.status.set('done', 100)
'/home/nils/.conda/envs/birdhouse/etc/pywps/TG_EUR-11_CNRM-CERFACS-CNRM-CM5_historical_r1i1p1_CLMcom-CCLM4-8-17_v1_JJA_ref-all.nc',
'/home/nils/.conda/envs/birdhouse/etc/pywps/PRCPTOT_EUR-11_CNRM-CERFACS-CNRM-CM5_historical_r1i1p1_CLMcom-CCLM4-8-17_v1_JJA_ref-all.nc'
]
ncs_indices = [
'/home/nils/data/sdm/TG_EUR-11_CNRM-CERFACS-CNRM-CM5_historical_r1i1p1_CLMcom-CCLM4-8-17_v1_JJA_20010101-20050101.nc',
'/home/nils/data/sdm/TG_EUR-11_ICHEC-EC-EARTH_historical_r12i1p1_CLMcom-CCLM4-8-17_v1_JJA_20010101-20050101.nc',
'/home/nils/data/sdm/TG_EUR-11_ICHEC-EC-EARTH_historical_r1i1p1_KNMI-RACMO22E_v1_JJA_20010101-20050101.nc',
'/home/nils/data/sdm/TG_EUR-11_MOHC-HadGEM2-ES_historical_r1i1p1_CLMcom-CCLM4-8-17_v1_JJA_20010101-20050101.nc',
'/home/nils/data/sdm/TG_EUR-11_MOHC-HadGEM2-ES_historical_r1i1p1_KNMI-RACMO22E_v1_JJA_20010101-20050101.nc'
]
csv = '/home/nils/Downloads/0013848-160118175350007.csv'
latlon = sdm.get_latlon(csv)
PApoints = sdm.get_PAmask(coordinates=latlon)
gam_model, predict_ref, gam_info = sdm.get_gam(ncs_references, PApoints)
print gam_model.names
import matplotlib.pyplot as plt
for i in range(len(gam_model)):
try:
plt.plot(gam_model[i])
plt.show()
except:
print 'failed for', i
prediction = sdm.get_prediction(gam_model, ncs_indices)
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
from os.path import basename
from flyingpigeon import sdm
from flyingpigeon.utils import archive, archiveextract, download
response.update_status('Start process', 0)
try:
LOGGER.info('reading the arguments')
resources_raw = archiveextract(
resource=rename_complexinputs(request.inputs['resources']))
csv_url = request.inputs['gbif'][0].data
period = request.inputs['period']
period = period[0].data
indices = request.inputs['input_indices']
archive_format = request.inputs['archive_format']
LOGGER.info('indices %s ' % indices)
LOGGER.debug('csv_url %s' % csv_url)
except Exception:
LOGGER.exception('failed to read in the arguments')
raise
try:
LOGGER.info('set up the environment')
csv_file = download(csv_url)
resources = archiveextract(resources_raw)
except:
LOGGER.exception('failed to set up the environment')
raise
try:
response.update_status('read in latlon coordinates', 10)
latlon = sdm.latlon_gbifcsv(csv_file)
LOGGER.info('got occurence coordinates %s ' % csv_file)
except:
LOGGER.exception(
'failed to extract the latlon points from file: %s' %
(csv_file))
try:
response.update_status('plot map', 20)
from flyingpigeon.visualisation import map_gbifoccurrences
# latlon = sdm.latlon_gbifdic(gbifdic)
occurence_map = map_gbifoccurrences(latlon)
except:
LOGGER.exception('failed to plot occurence map')
#################################
# calculate the climate indices
#################################
# get the indices
ncs_indices = None
try:
response.update_status(
'start calculation of climate indices for %s' % indices, 30)
ncs_indices = sdm.get_indices(resources=resources, indices=indices)
LOGGER.info('indice calculation done')
except:
msg = 'failed to calculate indices'
LOGGER.exception(msg)
raise Exception(msg)
try:
response.update_status('get domain', 30)
domains = set()
for resource in ncs_indices:
# get_domain works only if metadata are set in a correct way
domains = domains.union([basename(resource).split('_')[1]])
if len(domains) == 1:
domain = list(domains)[0]
LOGGER.debug('Domain %s found in indices files' % domain)
else:
LOGGER.error('Not a single domain in indices files %s' %
domains)
except:
LOGGER.exception('failed to get domains')
try:
response.update_status('generating the PA mask', 20)
PAmask = sdm.get_PAmask(coordinates=latlon, domain=domain)
LOGGER.info('PA mask sucessfully generated')
except:
LOGGER.exception('failed to generate the PA mask')
try:
response.update_status('Ploting PA mask', 25)
from flyingpigeon.visualisation import map_PAmask
PAmask_png = map_PAmask(PAmask)
except:
LOGGER.exception('failed to plot the PA mask')
try:
# sort indices
indices_dic = None
indices_dic = sdm.sort_indices(ncs_indices)
LOGGER.info('indice files sorted for %s Datasets' %
len(indices_dic.keys()))
except:
msg = 'failed to sort indices'
LOGGER.exception(msg)
raise Exception(msg)
ncs_references = []
species_files = []
stat_infos = []
for count, key in enumerate(indices_dic.keys()):
try:
staus_nr = 40 + count * 10
response.update_status('Start processing of %s' % key,
staus_nr)
ncs = indices_dic[key]
LOGGER.info('with %s files' % len(ncs))
try:
ncs_reference = sdm.get_reference(ncs_indices=ncs,
period=period)
ncs_references.extend(ncs_reference)
LOGGER.info('reference indice calculated %s ' %
ncs_references)
except:
msg = 'failed to calculate the reference'
LOGGER.exception(msg)
raise Exception(msg)
try:
gam_model, predict_gam, gam_info = sdm.get_gam(
ncs_reference, PAmask)
stat_infos.append(gam_info)
response.update_status('GAM sucessfully trained',
staus_nr + 5)
except:
msg = 'failed to train GAM for %s' % (key)
LOGGER.debug(msg)
try:
prediction = sdm.get_prediction(gam_model, ncs)
response.update_status('prediction done', staus_nr + 7)
except:
msg = 'failed to predict tree occurence'
LOGGER.exception(msg)
# raise Exception(msg)
try:
response.update_status('land sea mask for predicted data',
staus_nr + 8)
from numpy import invert, isnan, nan, broadcast_arrays # , array, zeros, linspace, meshgrid
mask = invert(isnan(PAmask))
mask = broadcast_arrays(prediction, mask)[1]
prediction[mask is False] = nan
except:
LOGGER.debug('failed to mask predicted data')
try:
species_files.append(sdm.write_to_file(ncs[0], prediction))
LOGGER.info('Favourabillity written to file')
except:
msg = 'failed to write species file'
LOGGER.debug(msg)
# raise Exception(msg)
except:
msg = 'failed to calculate reference indices.'
LOGGER.exception(msg)
raise Exception(msg)
try:
archive_indices = None
archive_indices = archive(ncs_indices, format=archive_format)
LOGGER.info('indices added to archive')
except:
msg = 'failed adding indices to archive'
LOGGER.exception(msg)
raise Exception(msg)
archive_references = None
try:
archive_references = archive(ncs_references, format=archive_format)
LOGGER.info('indices reference added to archive')
except:
msg = 'failed adding reference indices to archive'
LOGGER.exception(msg)
raise Exception(msg)
archive_predicion = None
try:
archive_predicion = archive(species_files, format=archive_format)
LOGGER.info('species_files added to archive')
except:
msg = 'failed adding species_files indices to archive'
LOGGER.exception(msg)
raise Exception(msg)
try:
from flyingpigeon.visualisation import pdfmerge
stat_infosconcat = pdfmerge(stat_infos)
LOGGER.info('stat infos pdfs merged')
except:
LOGGER.exception('failed to concat images')
_, stat_infosconcat = tempfile.mkstemp(suffix='.pdf',
prefix='foobar-',
dir='.')
# self.output_csv.setValue(csv_file)
response.outputs['output_gbif'].file = occurence_map
response.outputs['output_PA'].file = PAmask_png
response.outputs['output_indices'].file = archive_indices
response.outputs['output_reference'].file = archive_references
response.outputs['archive_predicion'].file = archive_predicion
response.outputs['output_info'].file = stat_infosconcat
response.update_status('done', 100)
return response