def plot_analogs(configfile='config.txt', simday='all', **kwargs): """ """ from datetime import datetime as dt from netCDF4 import Dataset, MFDataset import uuid from flyingpigeon.visualisation import pdfmerge simoutpdf = 'Analogs.pdf' if (os.path.isfile(configfile) == True): curdir, confile = os.path.split(os.path.abspath(configfile)) lines = [line.rstrip('\n') for line in open(configfile)] for i in lines: if 'archivefile' in i: arcfile = i.split('"')[1] if 'simulationfile' in i: simfile = i.split('"')[1] if 'outputfile' in i: analogfile = i.split('"')[1] if 'nanalog' in i: nanalog = int(i.split(' =')[1]) if 'varname' in i: varname = i.split('"')[1] if 'predictordom' in i: domain = i.split('"')[1] analogfile = curdir + '/' + analogfile arcfile = curdir + '/' + arcfile simfile = curdir + '/' + simfile try: arc_times = get_time(arcfile) sim_times = get_time(simfile) except: arc_times = get_time_nc(arcfile) sim_times = get_time_nc(simfile) sim_dataset = Dataset(simfile) simvar = sim_dataset.variables[varname][:] # TODO: check other names for lat/lon lon = sim_dataset.variables['lon'][:] lat = sim_dataset.variables['lat'][:] sim_dataset.close() domain = domain.split(",") try: domain = [float(i) for i in domain] except: domain = [lon[0], lon[-1], lat[-1], lat[0]] outlist = [] total_simmin = np.min(simvar) total_simmax = np.max(simvar) cont = [line.rstrip('\n') for line in open(analogfile)] Nlin = 30 for idx, item in enumerate(cont[1:]): ana = item.split() sim_date = dt.strptime(ana[0], '%Y%m%d') an_dates = [] for dat in ana[1:1 + nanalog]: an_dates.append(dt.strptime(dat, '%Y%m%d')) c_dists = ana[1 + nanalog:1 + 2 * nanalog] c_cors = ana[1 + 2 * nanalog:] dists = np.zeros((nanalog), dtype=float) cors = np.zeros((nanalog), dtype=float) for i in range(0, nanalog): dists[i] = float(c_dists[i]) cors[i] = float(c_cors[i]) min_dist = np.min(dists) max_corr = np.max(cors) w_dist = min_dist / dists w_corr = cors / max_corr sim_index = idx # day by day tmp_i = [] for i in arc_times: tmp_z = '%s-%s-%s' % (i.year, i.month, i.day) tmp_i.append(tmp_z) arc_index = [] for arc in an_dates: arc_date_temp = '%s-%s-%s' % (arc.year, arc.month, arc.day) arc_index.append(tmp_i.index(arc_date_temp)) simmin = np.min(simvar[sim_index, :, :]) simmax = np.max(simvar[sim_index, :, :]) # PLOT SIM ==================================== sim_title = 'Simulation Day: ' + ana[0] output_file_name = 'sim_' + ana[0] + '.pdf' output_file = pdf_from_analog(lon=lon, lat=lat, data=simvar[sim_index, :, :], vmin=simmin, vmax=simmax, Nlin=Nlin, domain=domain, output=output_file_name, title=sim_title) outlist.append(str(output_file)) # PLOT Mean analogs ==================================== arc_dataset = Dataset(arcfile) arcvar = arc_dataset.variables[varname][:] arc_dataset.close() mean_ana = np.zeros((len(arcvar[0, :, 0]), len(arcvar[0, 0, :])), dtype=float) for ida, art in enumerate(arc_index): mean_ana = mean_ana + arcvar[art, :, :] mean_ana = mean_ana / nanalog output_an_file_name = 'ana_' + ana[0] + '.pdf' an_title = 'Mean analogs for sim Day: ' + ana[0] an_output_file = pdf_from_analog(lon=lon, lat=lat, data=mean_ana, vmin=simmin, vmax=simmax, Nlin=Nlin, domain=domain, output=output_an_file_name, title=an_title) outlist.append(str(an_output_file)) # PLOT BEST (first) analog output_ban_file_name = 'bana_' + ana[0] + '.pdf' #PDF!! ban_title = 'BEST analog for sim Day ' + ana[0] + ' is: ' + ana[1] ban_output_file = pdf_from_analog(lon=lon, lat=lat, data=arcvar[arc_index[0]], vmin=simmin, vmax=simmax, Nlin=Nlin, domain=domain, output=output_ban_file_name, title=ban_title) outlist.append(str(ban_output_file)) # PLOT WORST (last) analog output_wan_file_name = 'wana_' + ana[0] + '.pdf' #PDF!! wan_title = 'LAST analog for sim Day ' + ana[0] + ' is: ' + ana[ nanalog] wan_output_file = pdf_from_analog(lon=lon, lat=lat, data=arcvar[arc_index[-1]], vmin=simmin, vmax=simmax, Nlin=Nlin, domain=domain, output=output_wan_file_name, title=wan_title) outlist.append(str(wan_output_file)) # PLOT Max and Min correl analog min_c_index = np.argmin(cors) max_c_index = np.argmax(cors) output_bcan_file_name = 'bcana_' + ana[0] + '.pdf' #PDF!! bcan_title = 'Analog with max corr for sim Day ' + ana[ 0] + ' is: ' + ana[1 + max_c_index] bcan_output_file = pdf_from_analog( lon=lon, lat=lat, data=arcvar[arc_index[max_c_index]], vmin=simmin, vmax=simmax, Nlin=Nlin, domain=domain, output=output_bcan_file_name, title=bcan_title) outlist.append(str(bcan_output_file)) output_wcan_file_name = 'wcana_' + ana[0] + '.pdf' #PDF!! wcan_title = 'Analog with min corr for sim Day ' + ana[ 0] + ' is: ' + ana[1 + min_c_index] wcan_output_file = pdf_from_analog( lon=lon, lat=lat, data=arcvar[arc_index[min_c_index]], vmin=simmin, vmax=simmax, Nlin=Nlin, domain=domain, output=output_wcan_file_name, title=wcan_title) outlist.append(str(wcan_output_file)) """ # PLOT analogs dist weighted ==================================== mean_ana = np.zeros((len(arcvar[0,:,0]),len(arcvar[0,0,:])),dtype=float) for ida, art in enumerate(arc_index): mean_ana=mean_ana+arcvar[art,:,:]*w_dist[ida] mean_ana = mean_ana/sum(w_dist) # PLOT analogs corr weighted ==================================== mean_ana = np.zeros((len(arcvar[0,:,0]),len(arcvar[0,0,:])),dtype=float) for ida, art in enumerate(arc_index): mean_ana=mean_ana+arcvar[art,:,:]*w_corr[ida] mean_ana = mean_ana/sum(w_corr) """ simoutpdf = pdfmerge(outlist) # get the information from config file: # netcdf files, period, Nanalogs, ouput analogs # get the information from config file: # netcdf files, period, Nanalogs, ouput analogs else: simoutpdf = 'Analogs.pdf' # TODO: call this func with analogfile = '..', # arguments came from kwargs # need to prescribe all input info - to use with external analogs results. # check kwargs: ncfiles, N analogs (?), periods, etc return simoutpdf
def execute(self): from os.path import basename from flyingpigeon import sdm from flyingpigeon.utils import archive, archiveextract # , get_domain from flyingpigeon.visualisation import map_PAmask init_process_logger('log.txt') self.output_log.setValue('log.txt') self.status.set('Start process', 0) try: self.status.set('reading the arguments', 5) resources = archiveextract( self.getInputValues(identifier='input_indices')) csv_file = self.getInputValues(identifier='gbif')[0] period = self.getInputValues(identifier='period') period = period[0] archive_format = self.archive_format.getValue() except: logger.error('failed to read in the arguments') try: self.status.set('read in latlon coordinates', 10) latlon = sdm.latlon_gbifcsv(csv_file) except: logger.exception('failed to extract the latlon points') try: self.status.set('plot map', 20) from flyingpigeon.visualisation import map_gbifoccurrences # latlon = sdm.latlon_gbifdic(gbifdic) occurence_map = map_gbifoccurrences(latlon) except: logger.exception('failed to plot occurence map') # try: # self.status.set('get domain', 30) # domains = set() # for indice in resources: # # get_domain works only if metadata are set in a correct way # domains = domains.union([basename(indice).split('_')[1]]) # if len(domains) == 1: # domain = list(domains)[0] # logger.info('Domain %s found in indices files' % domain) # else: # logger.warn('NOT a single domain in indices files %s' % domains) # except: # logger.exception('failed to get domains') try: # sort indices indices_dic = sdm.sort_indices(resources) logger.info('indice files sorted for %s Datasets' % len(indices_dic.keys())) except: msg = 'failed to sort indices' logger.exception(msg) raise Exception(msg) ncs_references = [] species_files = [] stat_infos = [] PAmask_pngs = [] self.status.set('Start processing for %s Datasets' % len(indices_dic.keys())) for count, key in enumerate(indices_dic.keys()): try: staus_nr = 40 + count * 10 self.status.set('Start processing of %s' % key, staus_nr) ncs = indices_dic[key] logger.info('with %s files' % len(ncs)) try: self.status.set('generating the PA mask', 20) PAmask = sdm.get_PAmask(coordinates=latlon, nc=ncs[0]) logger.info('PA mask sucessfully generated') except: logger.exception('failed to generate the PA mask') try: self.status.set('Ploting PA mask', 25) PAmask_pngs.extend([map_PAmask(PAmask)]) except: logger.exception('failed to plot the PA mask') try: ncs_reference = sdm.get_reference(ncs_indices=ncs, period=period) ncs_references.extend(ncs_reference) logger.info('reference indice calculated %s ' % ncs_references) self.status.set('reference indice calculated', staus_nr + 2) except: msg = 'failed to calculate the reference' logger.exception(msg) # raise Exception(msg) try: gam_model, predict_gam, gam_info = sdm.get_gam( ncs_reference, PAmask, modelname=key) stat_infos.append(gam_info) self.status.set('GAM sucessfully trained', staus_nr + 5) except: msg = 'failed to train GAM for %s' % (key) logger.exception(msg) try: prediction = sdm.get_prediction(gam_model, ncs) self.status.set('prediction done', staus_nr + 7) except: msg = 'failed to predict tree occurence' logger.exception(msg) # raise Exception(msg) # try: # self.status.set('land sea mask for predicted data', staus_nr + 8) # from numpy import invert, isnan, nan, broadcast_arrays # , array, zeros, linspace, meshgrid # mask = invert(isnan(PAmask)) # mask = broadcast_arrays(prediction, mask)[1] # prediction[mask is False] = nan # except: # logger.exception('failed to mask predicted data') try: species_files.append(sdm.write_to_file(ncs[0], prediction)) logger.info('Favourabillity written to file') except: msg = 'failed to write species file' logger.exception(msg) # raise Exception(msg) except: msg = 'failed to process SDM chain for %s ' % key logger.exception(msg) # raise Exception(msg) try: archive_references = None archive_references = archive(ncs_references, format=archive_format) logger.info('indices 2D added to archive') except: msg = 'failed adding 2D indices to archive' logger.exception(msg) raise Exception(msg) archive_predicion = None try: archive_predicion = archive(species_files, format=archive_format) logger.info('species_files added to archive') except: msg = 'failed adding species_files indices to archive' logger.exception(msg) raise Exception(msg) try: from flyingpigeon.visualisation import pdfmerge, concat_images stat_infosconcat = pdfmerge(stat_infos) logger.debug('pngs %s' % PAmask_pngs) PAmask_png = concat_images(PAmask_pngs, orientation='h') logger.info('stat infos pdfs and mask pngs merged') except: logger.exception('failed to concat images') _, stat_infosconcat = tempfile.mkstemp(suffix='.pdf', prefix='foobar-', dir='.') _, PAmask_png = tempfile.mkstemp(suffix='.png', prefix='foobar-', dir='.') self.output_gbif.setValue(occurence_map) self.output_PA.setValue(PAmask_png) self.output_reference.setValue(archive_references) self.output_prediction.setValue(archive_predicion) self.output_info.setValue(stat_infosconcat) self.status.set('done', 100)
def execute(self): init_process_logger('log.txt') self.output_log.setValue('log.txt') from os.path import basename from flyingpigeon import sdm from flyingpigeon.utils import archive, archiveextract self.status.set('Start process', 0) try: logger.info('reading the arguments') resources = archiveextract( self.getInputValues(identifier='resources')) taxon_name = self.getInputValues(identifier='taxon_name')[0] bbox_obj = self.BBox.getValue() bbox = [ bbox_obj.coords[0][0], bbox_obj.coords[0][1], bbox_obj.coords[1][0], bbox_obj.coords[1][1] ] period = self.getInputValues(identifier='period') period = period[0] indices = self.getInputValues(identifier='input_indices') archive_format = self.archive_format.getValue() logger.debug("indices = %s for %s ", indices, taxon_name) logger.info("bbox={0}".format(bbox)) except Exception as e: logger.error('failed to read in the arguments %s ' % e) logger.info('indices %s ' % indices) try: self.status.set('Fetching GBIF Data', 10) gbifdic = sdm.get_gbif(taxon_name, bbox=bbox) except Exception as e: msg = 'failed to search gbif.' logger.exception(msg) raise Exception(msg) try: self.status.set('write csv file', 70) gbifcsv = sdm.gbifdic2csv(gbifdic) except Exception as e: msg = 'failed to write csv file.' logger.exception(msg) raise Exception(msg) try: self.status.set('plot map', 80) from flyingpigeon.visualisation import map_gbifoccurrences latlon = sdm.latlon_gbifdic(gbifdic) occurence_map = map_gbifoccurrences(latlon) except Exception as e: msg = 'failed to plot occurence map.' logger.exception(msg) raise Exception(msg) ################################# # calculate the climate indices ################################# # get the indices ncs_indices = None try: self.status.set( 'start calculation of climate indices for %s' % indices, 30) ncs_indices = sdm.get_indices(resources=resources, indices=indices) logger.info('indice calculation done') except: msg = 'failed to calculate indices' logger.exception(msg) raise Exception(msg) try: self.status.set('get domain', 30) domains = set() for resource in ncs_indices: # get_domain works only if metadata are set in a correct way domains = domains.union([basename(resource).split('_')[1]]) if len(domains) == 1: domain = list(domains)[0] logger.debug('Domain %s found in indices files' % domain) else: logger.error('Not a single domain in indices files %s' % domains) except Exception as e: logger.exception('failed to get domains %s' % e) try: self.status.set('generating the PA mask', 20) PAmask = sdm.get_PAmask(coordinates=latlon, domain=domain) logger.info('PA mask sucessfully generated') except Exception as e: logger.exception('failed to generate the PA mask: %s' % e) try: self.status.set('Ploting PA mask', 25) from flyingpigeon.visualisation import map_PAmask PAmask_png = map_PAmask(PAmask) except Exception as e: logger.exception('failed to plot the PA mask: %s' % e) try: # sort indices indices_dic = None indices_dic = sdm.sort_indices(ncs_indices) logger.info('indice files sorted for %s Datasets' % len(indices_dic.keys())) except: msg = 'failed to sort indices' logger.exception(msg) raise Exception(msg) ncs_references = [] species_files = [] stat_infos = [] for count, key in enumerate(indices_dic.keys()): try: staus_nr = 40 + count * 10 self.status.set('Start processing of %s' % key, staus_nr) ncs = indices_dic[key] logger.info('with %s files' % len(ncs)) try: ncs_reference = sdm.get_reference(ncs_indices=ncs, period=period) ncs_references.extend(ncs_reference) logger.info('reference indice calculated %s ' % ncs_references) except: msg = 'failed to calculate the reference' logger.exception(msg) raise Exception(msg) try: gam_model, predict_gam, gam_info = sdm.get_gam( ncs_reference, PAmask) stat_infos.append(gam_info) self.status.set('GAM sucessfully trained', staus_nr + 5) except Exception as e: msg = 'failed to train GAM for %s : %s' % (key, e) logger.debug(msg) try: prediction = sdm.get_prediction(gam_model, ncs) self.status.set('prediction done', staus_nr + 7) except Exception as e: msg = 'failed to predict tree occurence %s' % e logger.exception(msg) # raise Exception(msg) try: self.status.set('land sea mask for predicted data', staus_nr + 8) from numpy import invert, isnan, nan, broadcast_arrays # , array, zeros, linspace, meshgrid mask = invert(isnan(PAmask)) mask = broadcast_arrays(prediction, mask)[1] prediction[mask is False] = nan except Exception as e: logger.debug('failed to mask predicted data: %s' % e) try: species_files.append(sdm.write_to_file(ncs[0], prediction)) logger.info('Favourabillity written to file') except Exception as e: msg = 'failed to write species file %s' % e logger.debug(msg) # raise Exception(msg) except Exception as e: msg = 'failed to calculate reference indices. %s ' % e logger.exception(msg) raise Exception(msg) try: archive_indices = None archive_indices = archive(ncs_indices, format=archive_format) logger.info('indices added to archive') except: msg = 'failed adding indices to archive' logger.exception(msg) raise Exception(msg) archive_references = None try: archive_references = archive(ncs_references, format=archive_format) logger.info('indices reference added to archive') except: msg = 'failed adding reference indices to archive' logger.exception(msg) raise Exception(msg) archive_predicion = None try: archive_predicion = archive(species_files, format=archive_format) logger.info('species_files added to archive') except: msg = 'failed adding species_files indices to archive' logger.exception(msg) raise Exception(msg) try: from flyingpigeon.visualisation import pdfmerge stat_infosconcat = pdfmerge(stat_infos) logger.info('stat infos pdfs merged') except: logger.exception('failed to concat images') _, stat_infosconcat = tempfile.mkstemp(suffix='.pdf', prefix='foobar-', dir='.') self.output_csv.setValue(gbifcsv) self.output_gbif.setValue(occurence_map) self.output_PA.setValue(PAmask_png) self.output_indices.setValue(archive_indices) self.output_reference.setValue(archive_references) self.output_prediction.setValue(archive_predicion) self.output_info.setValue(stat_infosconcat) self.status.set('done', 100)
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' response.update_status('Start process', 0) try: LOGGER.info('reading the arguments') resources = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) taxon_name = request.inputs['taxon_name'][0].data bbox = [-180, -90, 180, 90] # bbox_obj = self.BBox.getValue() # bbox = [bbox_obj.coords[0][0], # bbox_obj.coords[0][1], # bbox_obj.coords[1][0], # bbox_obj.coords[1][1]] period = request.inputs['period'] period = period[0].data indices = [inpt.data for inpt in request.inputs['indices']] archive_format = request.inputs['archive_format'][0].data LOGGER.exception("indices = %s for %s ", indices, taxon_name) LOGGER.info("bbox={0}".format(bbox)) except: LOGGER.exception('failed to read in the arguments') LOGGER.info('indices %s ' % indices) try: response.update_status('Fetching GBIF Data', 10) gbifdic = sdm.get_gbif(taxon_name, bbox=bbox) LOGGER.info('Fetched GBIF data') except: msg = 'failed to search gbif.' LOGGER.exception(msg) raise Exception(msg) try: response.update_status('write csv file', 70) gbifcsv = sdm.gbifdic2csv(gbifdic) LOGGER.info('GBIF data written to file') except: msg = 'failed to write csv file.' LOGGER.exception(msg) raise Exception(msg) try: response.update_status('plot map', 80) latlon = sdm.latlon_gbifdic(gbifdic) occurence_map = map_gbifoccurrences(latlon) except: msg = 'failed to plot occurence map.' LOGGER.exception(msg) raise Exception(msg) ################################# # calculate the climate indices ################################# # get the indices ncs_indices = None try: response.update_status( 'start calculation of climate indices for %s' % indices, 30) ncs_indices = sdm.get_indices(resource=resources, indices=indices) LOGGER.info('indice calculation done') except: msg = 'failed to calculate indices' LOGGER.exception(msg) raise Exception(msg) try: # sort indices indices_dic = sdm.sort_indices(ncs_indices) LOGGER.info('indice files sorted in dictionary') except: msg = 'failed to sort indices' LOGGER.exception(msg) indices_dic = {'dummy': []} ncs_references = [] species_files = [] stat_infos = [] PAmask_pngs = [] response.update_status('Start processing for %s Datasets' % len(indices_dic.keys())) for count, key in enumerate(indices_dic.keys()): try: staus_nr = 40 + count * 10 response.update_status('Start processing of %s' % key, staus_nr) ncs = indices_dic[key] LOGGER.info('with %s files' % len(ncs)) try: response.update_status('generating the PA mask', 20) PAmask = sdm.get_PAmask(coordinates=latlon, nc=ncs[0]) LOGGER.info('PA mask sucessfully generated') except: LOGGER.exception('failed to generate the PA mask') try: response.update_status('Ploting PA mask', 25) PAmask_pngs.extend([map_PAmask(PAmask)]) except: LOGGER.exception('failed to plot the PA mask') try: ncs_reference = sdm.get_reference(ncs_indices=ncs, period=period) ncs_references.extend(ncs_reference) LOGGER.info('reference indice calculated %s ' % ncs_references) except: msg = 'failed to calculate the reference' LOGGER.exception(msg) try: gam_model, predict_gam, gam_info = sdm.get_gam( ncs_reference, PAmask) stat_infos.append(gam_info) response.update_status('GAM sucessfully trained', staus_nr + 5) except: msg = 'failed to train GAM for %s' % (key) LOGGER.exception(msg) try: prediction = sdm.get_prediction(gam_model, ncs) response.update_status('prediction done', staus_nr + 7) except: msg = 'failed to predict tree occurence' LOGGER.exception(msg) # raise Exception(msg) # try: # response.update_status('land sea mask for predicted data', staus_nr + 8) # from numpy import invert, isnan, nan, broadcast_arrays # , array, zeros, linspace, meshgrid # mask = invert(isnan(PAmask)) # mask = broadcast_arrays(prediction, mask)[1] # prediction[mask is False] = nan # except: # LOGGER.exception('failed to mask predicted data') try: species_files.append(sdm.write_to_file(ncs[0], prediction)) LOGGER.info('Favourabillity written to file') except: msg = 'failed to write species file' LOGGER.exception(msg) # raise Exception(msg) except: msg = 'failed to calculate reference indices' LOGGER.exception(msg) raise Exception(msg) try: archive_indices = archive(ncs_indices, format=archive_format) LOGGER.info('indices added to archive') except: msg = 'failed adding indices to archive' LOGGER.exception(msg) archive_indices = tempfile.mkstemp(suffix='.tar', prefix='foobar-', dir='.') try: archive_references = archive(ncs_references, format=archive_format) LOGGER.info('indices reference added to archive') except: msg = 'failed adding reference indices to archive' LOGGER.exception(msg) archive_references = tempfile.mkstemp(suffix='.tar', prefix='foobar-', dir='.') try: archive_prediction = archive(species_files, format=archive_format) LOGGER.info('species_files added to archive') except: msg = 'failed adding species_files indices to archive' LOGGER.exception(msg) raise Exception(msg) try: stat_infosconcat = pdfmerge(stat_infos) LOGGER.debug('pngs %s' % PAmask_pngs) PAmask_png = concat_images(PAmask_pngs, orientation='h') LOGGER.info('stat infos pdfs and mask pngs merged') except: LOGGER.exception('failed to concat images') _, stat_infosconcat = tempfile.mkstemp(suffix='.pdf', prefix='foobar-', dir='.') _, PAmask_png = tempfile.mkstemp(suffix='.png', prefix='foobar-', dir='.') response.outputs['output_gbif'].file = occurence_map response.outputs['output_PA'].file = PAmask_png response.outputs['output_indices'].file = archive_indices response.outputs['output_reference'].file = archive_references response.outputs['output_prediction'].file = archive_prediction response.outputs['output_info'].file = stat_infosconcat response.outputs['output_csv'].file = gbifcsv response.update_status('done', 100) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' response.update_status('Start process', 0) try: response.update_status('reading the arguments', 5) resources = archiveextract( resource=rename_complexinputs(request.inputs['resources'])) period = request.inputs['period'] period = period[0].data archive_format = request.inputs['archive_format'][0].data LOGGER.info("all arguments read in nr of files in resources: {}".format(len(resources))) except Exception as ex: LOGGER.exception('failed to read in the arguments: {}'.format(str(ex))) try: gbif_url = request.inputs['gbif'][0].data csv_file = download(gbif_url) LOGGER.info('CSV file fetched sucessfully: %s' % csv_file) except Exception as ex: LOGGER.exception('failed to fetch GBIF file: {}'.format(str(ex))) try: response.update_status('read in latlon coordinates', 10) latlon = sdm.latlon_gbifcsv(csv_file) LOGGER.info('read in the latlon coordinates') except Exception as ex: LOGGER.exception('failed to extract the latlon points: {}'.format(str(ex))) try: response.update_status('plot map', 20) occurence_map = map_gbifoccurrences(latlon) LOGGER.info('GBIF occourence ploted') except Exception as ex: LOGGER.exception('failed to plot occurrence map: {}'.format(str(ex))) try: # sort indices indices_dic = sdm.sort_indices(resources) LOGGER.info('indice files sorted in dictionary') except Exception as ex: msg = 'failed to sort indices: {}'.format(str(ex)) LOGGER.exception(msg) indices_dic = {'dummy': []} ncs_references = [] species_files = [] stat_infos = [] PAmask_pngs = [] response.update_status('Start processing for {} datasets'.format(len(indices_dic.keys()))) for count, key in enumerate(indices_dic.keys()): try: status_nr = 40 + count * 10 response.update_status('Start processing of {}'.format(key), status_nr) ncs = indices_dic[key] LOGGER.info('with {} files'.format(len(ncs))) try: response.update_status('generating the PA mask', 20) PAmask = sdm.get_PAmask(coordinates=latlon, nc=ncs[0]) LOGGER.info('PA mask sucessfully generated') except Exception as ex: msg = 'failed to generate the PA mask: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: response.update_status('Ploting PA mask', 25) PAmask_pngs.extend([map_PAmask(PAmask)]) except Exception as ex: msg = 'failed to plot the PA mask: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: ncs_reference = sdm.get_reference(ncs_indices=ncs, period=period) ncs_references.extend(ncs_reference) LOGGER.info('reference indice calculated: {}'.format(ncs_references)) response.update_status('reference indice calculated', status_nr + 2) except Exception as ex: msg = 'failed to calculate the reference: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: gam_model, predict_gam, gam_info = sdm.get_gam(ncs_reference, PAmask, modelname=key) stat_infos.append(gam_info) response.update_status('GAM sucessfully trained', status_nr + 5) except Exception as ex: msg = 'failed to train GAM for {}: {}'.format(key, str(ex)) LOGGER.exception(msg) raise Exception(msg) try: prediction = sdm.get_prediction(gam_model, ncs) response.update_status('prediction done', status_nr + 7) except Exception as ex: msg = 'failed to predict tree occurence: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) # try: # response.update_status('land sea mask for predicted data', status_nr + 8) # from numpy import invert, isnan, nan, broadcast_arrays # , array, zeros, linspace, meshgrid # mask = invert(isnan(PAmask)) # mask = broadcast_arrays(prediction, mask)[1] # prediction[mask is False] = nan # except: # LOGGER.exception('failed to mask predicted data') try: species_files.append(sdm.write_to_file(ncs[0], prediction)) LOGGER.info('Favourabillity written to file') except Exception as ex: msg = 'failed to write species file: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) except Exception as ex: msg = 'failed to process SDM chain for {} : {}'.format(key, str(ex)) LOGGER.exception(msg) raise Exception(msg) try: archive_references = archive(ncs_references, format=archive_format) LOGGER.info('indices 2D added to archive') except Exception as ex: msg = 'failed adding 2D indices to archive: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) archive_references = tempfile.mkstemp(suffix='.tar', prefix='foobar-', dir='.') try: archive_prediction = archive(species_files, format=archive_format) LOGGER.info('species_files added to archive') except Exception as ex: msg = 'failed adding species_files indices to archive: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) archive_predicion = tempfile.mkstemp(suffix='.tar', prefix='foobar-', dir='.') try: from flyingpigeon.visualisation import pdfmerge, concat_images stat_infosconcat = pdfmerge(stat_infos) LOGGER.debug('pngs: {}'.format(PAmask_pngs)) PAmask_png = concat_images(PAmask_pngs, orientation='h') LOGGER.info('stat infos pdfs and mask pngs merged') except Exception as ex: msg = 'failed to concat images: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) _, stat_infosconcat = tempfile.mkstemp(suffix='.pdf', prefix='foobar-', dir='.') _, PAmask_png = tempfile.mkstemp(suffix='.png', prefix='foobar-', dir='.') response.outputs['output_gbif'].file = occurence_map response.outputs['output_PA'].file = PAmask_png response.outputs['output_reference'].file = archive_references response.outputs['output_prediction'].file = archive_prediction response.outputs['output_info'].file = stat_infosconcat response.update_status('done', 100) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' from os.path import basename from flyingpigeon import sdm from flyingpigeon.utils import archive, archiveextract, download response.update_status('Start process', 0) try: LOGGER.info('reading the arguments') resources_raw = archiveextract( resource=rename_complexinputs(request.inputs['resources'])) csv_url = request.inputs['gbif'][0].data period = request.inputs['period'] period = period[0].data indices = request.inputs['input_indices'] archive_format = request.inputs['archive_format'] LOGGER.info('indices %s ' % indices) LOGGER.debug('csv_url %s' % csv_url) except Exception: LOGGER.exception('failed to read in the arguments') raise try: LOGGER.info('set up the environment') csv_file = download(csv_url) resources = archiveextract(resources_raw) except: LOGGER.exception('failed to set up the environment') raise try: response.update_status('read in latlon coordinates', 10) latlon = sdm.latlon_gbifcsv(csv_file) LOGGER.info('got occurence coordinates %s ' % csv_file) except: LOGGER.exception( 'failed to extract the latlon points from file: %s' % (csv_file)) try: response.update_status('plot map', 20) from flyingpigeon.visualisation import map_gbifoccurrences # latlon = sdm.latlon_gbifdic(gbifdic) occurence_map = map_gbifoccurrences(latlon) except: LOGGER.exception('failed to plot occurence map') ################################# # calculate the climate indices ################################# # get the indices ncs_indices = None try: response.update_status( 'start calculation of climate indices for %s' % indices, 30) ncs_indices = sdm.get_indices(resources=resources, indices=indices) LOGGER.info('indice calculation done') except: msg = 'failed to calculate indices' LOGGER.exception(msg) raise Exception(msg) try: response.update_status('get domain', 30) domains = set() for resource in ncs_indices: # get_domain works only if metadata are set in a correct way domains = domains.union([basename(resource).split('_')[1]]) if len(domains) == 1: domain = list(domains)[0] LOGGER.debug('Domain %s found in indices files' % domain) else: LOGGER.error('Not a single domain in indices files %s' % domains) except: LOGGER.exception('failed to get domains') try: response.update_status('generating the PA mask', 20) PAmask = sdm.get_PAmask(coordinates=latlon, domain=domain) LOGGER.info('PA mask sucessfully generated') except: LOGGER.exception('failed to generate the PA mask') try: response.update_status('Ploting PA mask', 25) from flyingpigeon.visualisation import map_PAmask PAmask_png = map_PAmask(PAmask) except: LOGGER.exception('failed to plot the PA mask') try: # sort indices indices_dic = None indices_dic = sdm.sort_indices(ncs_indices) LOGGER.info('indice files sorted for %s Datasets' % len(indices_dic.keys())) except: msg = 'failed to sort indices' LOGGER.exception(msg) raise Exception(msg) ncs_references = [] species_files = [] stat_infos = [] for count, key in enumerate(indices_dic.keys()): try: staus_nr = 40 + count * 10 response.update_status('Start processing of %s' % key, staus_nr) ncs = indices_dic[key] LOGGER.info('with %s files' % len(ncs)) try: ncs_reference = sdm.get_reference(ncs_indices=ncs, period=period) ncs_references.extend(ncs_reference) LOGGER.info('reference indice calculated %s ' % ncs_references) except: msg = 'failed to calculate the reference' LOGGER.exception(msg) raise Exception(msg) try: gam_model, predict_gam, gam_info = sdm.get_gam( ncs_reference, PAmask) stat_infos.append(gam_info) response.update_status('GAM sucessfully trained', staus_nr + 5) except: msg = 'failed to train GAM for %s' % (key) LOGGER.debug(msg) try: prediction = sdm.get_prediction(gam_model, ncs) response.update_status('prediction done', staus_nr + 7) except: msg = 'failed to predict tree occurence' LOGGER.exception(msg) # raise Exception(msg) try: response.update_status('land sea mask for predicted data', staus_nr + 8) from numpy import invert, isnan, nan, broadcast_arrays # , array, zeros, linspace, meshgrid mask = invert(isnan(PAmask)) mask = broadcast_arrays(prediction, mask)[1] prediction[mask is False] = nan except: LOGGER.debug('failed to mask predicted data') try: species_files.append(sdm.write_to_file(ncs[0], prediction)) LOGGER.info('Favourabillity written to file') except: msg = 'failed to write species file' LOGGER.debug(msg) # raise Exception(msg) except: msg = 'failed to calculate reference indices.' LOGGER.exception(msg) raise Exception(msg) try: archive_indices = None archive_indices = archive(ncs_indices, format=archive_format) LOGGER.info('indices added to archive') except: msg = 'failed adding indices to archive' LOGGER.exception(msg) raise Exception(msg) archive_references = None try: archive_references = archive(ncs_references, format=archive_format) LOGGER.info('indices reference added to archive') except: msg = 'failed adding reference indices to archive' LOGGER.exception(msg) raise Exception(msg) archive_predicion = None try: archive_predicion = archive(species_files, format=archive_format) LOGGER.info('species_files added to archive') except: msg = 'failed adding species_files indices to archive' LOGGER.exception(msg) raise Exception(msg) try: from flyingpigeon.visualisation import pdfmerge stat_infosconcat = pdfmerge(stat_infos) LOGGER.info('stat infos pdfs merged') except: LOGGER.exception('failed to concat images') _, stat_infosconcat = tempfile.mkstemp(suffix='.pdf', prefix='foobar-', dir='.') # self.output_csv.setValue(csv_file) response.outputs['output_gbif'].file = occurence_map response.outputs['output_PA'].file = PAmask_png response.outputs['output_indices'].file = archive_indices response.outputs['output_reference'].file = archive_references response.outputs['archive_predicion'].file = archive_predicion response.outputs['output_info'].file = stat_infosconcat response.update_status('done', 100) return response