def execute(self): ###################### # start execution ###################### init_process_logger('log.txt') self.output_log.setValue('log.txt') from flyingpigeon import analogs as anlg from flyingpigeon import config from os.path import basename ########################################### # reorganize analog txt file for javascript # and find associated config file ########################################### # Reformat data file output by the analogs detection process so that # it can be read by the analogues viewer template. try: # Get the output csv file of analogs process (input by user in # text box) analogs = self.getInputValues(identifier='resource')[0] configfile = anlg.get_viewer_configfile(analogs) f = anlg.reformat_analogs(analogs) logger.info('Analog file reformatted') self.status.set('Successfully reformatted analog file', 50) output_av = anlg.get_viewer(f, configfile) logger.info('Viewer html page generated') self.status.set( 'Successfully generated analogs viewer html page', 90) outputUrl_path = config.outputUrl_path()
def execute(self): ###################### # start execution ###################### from flyingpigeon import analogs as anlg from flyingpigeon import config from os.path import basename ########################################### # reorganize analog txt file for javascript # and find associated config file ########################################### # Reformat data file output by the analogs detection process so that # it can be read by the analogues viewer template. try: # Get the output csv file of analogs process (input by user in # text box) analogs = self.getInputValues(identifier='resource')[0] configfile = anlg.get_viewer_configfile(analogs) f = anlg.reformat_analogs(analogs) logger.info('Analog file reformatted') self.status.set('Successfully reformatted analog file', 50) output_av = anlg.get_viewer(f, configfile) logger.info('Viewer html page generated') self.status.set( 'Successfully generated analogs viewer html page', 90) outputUrl_path = config.outputUrl_path() output_data = outputUrl_path + '/' + basename(f) logger.info('Data url: %s ' % output_data) logger.info('output_av: %s ' % output_av) except Exception as e: msg = 'Failed to reformat analogs file or generate viewer%s ' % e logger.debug(msg) ################################ # set the outputs ################################ self.output_txt.setValue(output_data) self.output_html.setValue(output_av)
def execute(self): ###################### # start execution ###################### from flyingpigeon import analogs as anlg from flyingpigeon import config from os.path import basename ########################################### # reorganize analog txt file for javascript # and find associated config file ########################################### # Reformat data file output by the analogs detection process so that # it can be read by the analogues viewer template. try: # Get the output csv file of analogs process (input by user in # text box) analogs = self.getInputValues(identifier='resource')[0] configfile = anlg.get_viewer_configfile(analogs) f = anlg.reformat_analogs(analogs) logger.info('Analog file reformatted') self.status.set('Successfully reformatted analog file', 50) output_av = anlg.get_viewer(f, configfile) logger.info('Viewer html page generated') self.status.set('Successfully generated analogs viewer html page', 90) outputUrl_path = config.outputUrl_path() output_data = outputUrl_path + '/' + basename(f) logger.info('Data url: %s ' % output_data) logger.info('output_av: %s ' % output_av) except Exception as e: msg = 'Failed to reformat analogs file or generate viewer%s ' % e logger.debug(msg) ################################ # set the outputs ################################ self.output_txt.setValue(output_data) self.output_html.setValue(output_av)
def copy_configfile(configfile): """ copy configuration file into output folder :param configfile: configuration file (path/to/file.txt) in working dir :return str,str: output_path, output_url """ from flyingpigeon import config from shutil import copyfile from os import path outputUrl_path = config.outputUrl_path() output_path = config.output_path() # configfile = path.basename(configfile) #just file name # Add server path to file name config_output_path = path.join(output_path, path.basename(configfile)) # Copy out of local working dir to output_path copyfile(configfile, config_output_path) config_output_url = path.join(outputUrl_path, configfile) return config_output_path, config_output_url
def get_viewer_configfile(analogs): """ finds or generates configuration file for an analogs file to be used by the analogs viewer. The configuration file will be copied into the output folder. :param analogs: text file containing the analogs values :return str: configuration file path/name.txt """ from flyingpigeon import config from flyingpigeon.config import www_url import requests from shutil import copyfile from tempfile import mkstemp from os import path from os.path import basename try: outputUrl_path = config.outputUrl_path() output_path = config.output_path() # Config file with path (server URL address) configfile = analogs.replace('analogs-', 'config-') logger.info('config filename generated %s' % configfile) configfile_with_path = path.join(outputUrl_path, configfile) logger.debug('configfile_with_path: %s' % configfile_with_path) # Check if config file exists r = requests.get(configfile_with_path) if r.status_code != 404: logger.debug('Config file exists on server URL address.') else: logger.debug( 'Config file does not exist on server address. Check local disk.') # Make config file name and get its path on local disk configfile = 'config_' + analogs logger.debug('local disk configfile: %s' % configfile) p, name = path.split(path.realpath(analogs)) configfile_localAddress = path.join(p, configfile) logger.debug('local disk configfile_localAddress: %s' % configfile_localAddress) # Check if config file exists if path.isfile(configfile_localAddress): logger.debug('Config file exists on local disk.') # Copy config file to output_path # (~/birdhouse/var/lib/pywps/outputs/flyingpigeon) configfile_outputlocation = path.join(output_path, configfile) copyfile(configfile_localAddress, configfile_outputlocation) logger.info(' time for coffee ') configfile_outputlocation_edited = config_edits( configfile_outputlocation) logger.info('outputlocation_edited: %s' % configfile_outputlocation_edited) configfile = path.basename(configfile_outputlocation_edited) logger.info(' configfile %s ' % configfile) else: logger.debug( 'There is no config file on local disk. Generating a default one.') # Insert analogs filename into config file. # The rest of the params are unknown. configfile_wkdir = get_configfile( files=['dummyconfig', 'dummyconfig', analogs], nanalog='DUMMY!!!', varname='DUMMY!!!', seacyc='DUMMY!!!', cycsmooth='DUMMY!!!', timewin='DUMMY!!!', seasonwin='DUMMY!!!', distfun='DUMMY!!!', calccor='DUMMY!!!', outformat='DUMMY!!!', silent='DUMMY!!!', period=['dummy', 'dummy'], bbox='DUMMY!!!' ) configfile = path.basename(configfile_wkdir) # just file name # Add server path to file name configfile_inplace = path.join(output_path, configfile) # Copy out of local working dir to output_path copyfile(configfile_wkdir, configfile_inplace) except Exception as e: msg = 'failed to read number of analogues from config file %s ' % e logger.debug(msg) return configfile
def execute(self): ###################### # start execution ###################### from flyingpigeon.config import JSsrc_dir tmpl = JSsrc_dir() + '/template_analogviewer.html' #Output csv file of analogs process analogs = self.getInputValues(identifier='resource')[0] ########################################### # reorganize analog txt file for javascript ########################################### from flyingpigeon import config from tempfile import mkstemp from flyingpigeon.config import www_url #my_css_url = www_url() + "/static/css/style.css" #begin CN #use as test input file: http://birdhouse-lsce.extra.cea.fr:8090/wpsoutputs/flyingpigeon/output_txt-0797016c-378e-11e6-91dd-41d8cd554993.txt import numpy as np import pandas as pd import collections import os try: num_analogues = 20 #number of analogues searched for num_cols = 3 #dateAnlg, Dis, Corr #Create dataframe and read in output csv file of analogs process dfS = pd.DataFrame() dfS = pd.read_csv(analogs, delimiter=r"\s+", index_col=0) #Define temporary df df_anlg = dfS.iloc[:, 0:20] #store only anlg dates df_dis = dfS.iloc[:, 20:40] #store only dis df_corr = dfS.iloc[:, 40:60] #store only corr #remove index name before stacking df_anlg.index.name = "" df_dis.index.name = "" df_corr.index.name = "" #Stack (transpose)into single col dateStack = df_anlg.stack() disStack = df_dis.stack().abs() #raw values < 0 so take abs corrStack = df_corr.stack() # #BUILD NEW DF IN CORRECT FORMAT #Create df of correct dimensions (n x num_cols) using dfS df_all = dfS.iloc[:, 0:num_cols] #NB data are placeholders #Rename cols df_all.columns = ['dateAnlg', 'Dis', 'Corr'] #Replicate each row 20 times (for dcjs format) df_all = df_all.loc[np.repeat(df_all.index.values,num_analogues)] #Replace data placeholders with correct values df_all['dateAnlg'] = list(dateStack) df_all['Dis'] = list(disStack) df_all['Corr'] = list(corrStack) #Name index col df_all.index.name = 'dateRef' # #SAVE TO TSV FILE output_path = config.output_path() ip , f = mkstemp(suffix='.tsv', prefix='modified-analogfile', dir=output_path, text=False) df_all.to_csv(f, sep='\t') logger.info('successfully reformatted analog file') self.status.set('successfully reformatted analog file', 90) except Exception as e: msg = 'failed to reformat analog file %s ' % e logger.debug(msg) # raise Exception(msg) ################################ # modify JS template ################################ from os.path import basename ip, output_av = mkstemp(suffix='.html', prefix='analogviewer', dir='.', text=False) #copyfile(tmpl, output_av) tmpl_file = open(tmpl).read() out = open(output_av, 'w') #replacements = {'analogues_placeholder.json':basename(f)} #for i in replacements.keys(): tmpl_file = tmpl_file.replace('analogues_placeholder.json', basename(f) ) out.write(tmpl_file) out.close() # with open(output_av, 'r+') as f: # content = f.read() # f.seek(0) # f.truncate() # f.write(content.replace('analogues_placeholder.json', basename(f))) ################################ # set the outputs ################################ outputUrl_path = config.outputUrl_path() output_data = outputUrl_path + '/' + basename(f) logger.info('Data url: %s ' % output_data) logger.info('output_av: %s ' % output_av) self.output_txt.setValue( output_data ) self.output_html.setValue( output_av )