def prepare_static_folder():
"""
Link static folder to output folder.
"""
destination = os.path.join(config.output_path(), 'static')
if not os.path.exists(destination):
os.symlink(config.static_path(), destination)
def reformat_analogs(analogs):
"""
Reformats analogs results file for analogues viewer code.
:param analogs: output from analog_detection process
:return str: reformatted analogs file for analogues viewer
"""
from flyingpigeon import config
import numpy as np
import pandas as pd
try:
num_cols = 3 #dateAnlg, Dis, Corr
#Create dataframe and read in output csv file of analogs process
dfS = pd.DataFrame()
dfS = pd.read_csv(analogs, delimiter=r"\s+", index_col=0)
#Find number of analogues
num_analogues = (dfS.shape[1])/3
logger.debug('num_analogues: %s ' % num_analogues)
#Define temporary df
df_anlg = dfS.iloc[:, 0:num_analogues] #store only anlg dates
df_dis = dfS.iloc[:, num_analogues:2*num_analogues] #store only dis
df_corr = dfS.iloc[:, 2*num_analogues:3*num_analogues] #store only corr
#remove index name before stacking
df_anlg.index.name = ""
df_dis.index.name = ""
df_corr.index.name = ""
dateStack = df_anlg.stack()
disStack = df_dis.stack().abs() #raw values < 0 so take abs
corrStack = df_corr.stack()
#Create df of correct dimensions (n x num_cols) using dfS
df_all = dfS.iloc[:, 0:num_cols] #NB data are placeholders
#Rename cols
df_all.columns = ['dateAnlg', 'Dis', 'Corr']
#Replicate each row 20 times (for dcjs format)
df_all = df_all.loc[np.repeat(df_all.index.values,num_analogues)]
#Replace data placeholders with correct values
df_all['dateAnlg'] = list(dateStack)
df_all['Dis'] = list(disStack)
df_all['Corr'] = list(corrStack)
#Name index col
df_all.index.name = 'dateRef'
#save to tsv file
output_path = config.output_path()
ip , analogs_mod = mkstemp(suffix='.tsv', prefix='modified-analogfile', dir=output_path, text=False)
df_all.to_csv(analogs_mod, sep='\t')
logger.info('successfully reformatted analog file')
def copy_configfile(configfile):
"""
copy configuration file into output folder
:param configfile: configuration file (path/to/file.txt) in working dir
:return str,str: output_path, output_url
"""
from flyingpigeon import config
from shutil import copyfile
from os import path
outputUrl_path = config.outputUrl_path()
output_path = config.output_path()
# configfile = path.basename(configfile) #just file name
# Add server path to file name
config_output_path = path.join(output_path, path.basename(configfile))
# Copy out of local working dir to output_path
copyfile(configfile, config_output_path)
config_output_url = path.join(outputUrl_path, configfile)
return config_output_path, config_output_url
def copy_configfile(configfile):
"""
copy configuration file into output folder
:param configfile: configuration file (path/to/file.txt) in working dir
:return str,str: output_path, output_url
"""
from flyingpigeon import config
from shutil import copyfile
from os import path
outputUrl_path = config.outputUrl_path()
output_path = config.output_path()
# configfile = path.basename(configfile) #just file name
# Add server path to file name
config_output_path = path.join(output_path, path.basename(configfile))
# Copy out of local working dir to output_path
copyfile(configfile, config_output_path)
config_output_url = path.join(outputUrl_path, configfile)
return config_output_path, config_output_url
def get_viewer_configfile(analogs):
"""
finds or generates configuration file for an analogs file
to be used by the analogs viewer. The configuration file
will be copied into the output folder.
:param analogs: text file containing the analogs values
:return str: configuration file path/name.txt
"""
from flyingpigeon import config
from flyingpigeon.config import www_url
import requests
from shutil import copyfile
from tempfile import mkstemp
from os import path
from os.path import basename
try:
outputUrl_path = config.outputUrl_path()
output_path = config.output_path()
# Config file with path (server URL address)
configfile = analogs.replace('analogs-', 'config-')
logger.info('config filename generated %s' % configfile)
configfile_with_path = path.join(outputUrl_path, configfile)
logger.debug('configfile_with_path: %s' % configfile_with_path)
# Check if config file exists
r = requests.get(configfile_with_path)
if r.status_code != 404:
logger.debug('Config file exists on server URL address.')
else:
logger.debug(
'Config file does not exist on server address. Check local disk.')
# Make config file name and get its path on local disk
configfile = 'config_' + analogs
logger.debug('local disk configfile: %s' % configfile)
p, name = path.split(path.realpath(analogs))
configfile_localAddress = path.join(p, configfile)
logger.debug('local disk configfile_localAddress: %s' %
configfile_localAddress)
# Check if config file exists
if path.isfile(configfile_localAddress):
logger.debug('Config file exists on local disk.')
# Copy config file to output_path
# (~/birdhouse/var/lib/pywps/outputs/flyingpigeon)
configfile_outputlocation = path.join(output_path, configfile)
copyfile(configfile_localAddress, configfile_outputlocation)
logger.info(' time for coffee ')
configfile_outputlocation_edited = config_edits(
configfile_outputlocation)
logger.info('outputlocation_edited: %s' %
configfile_outputlocation_edited)
configfile = path.basename(configfile_outputlocation_edited)
logger.info(' configfile %s ' % configfile)
else:
logger.debug(
'There is no config file on local disk. Generating a default one.')
# Insert analogs filename into config file.
# The rest of the params are unknown.
configfile_wkdir = get_configfile(
files=['dummyconfig', 'dummyconfig', analogs],
nanalog='DUMMY!!!',
varname='DUMMY!!!',
seacyc='DUMMY!!!',
cycsmooth='DUMMY!!!',
timewin='DUMMY!!!',
seasonwin='DUMMY!!!',
distfun='DUMMY!!!',
calccor='DUMMY!!!',
outformat='DUMMY!!!',
silent='DUMMY!!!',
period=['dummy', 'dummy'],
bbox='DUMMY!!!'
)
configfile = path.basename(configfile_wkdir) # just file name
# Add server path to file name
configfile_inplace = path.join(output_path, configfile)
# Copy out of local working dir to output_path
copyfile(configfile_wkdir, configfile_inplace)
except Exception as e:
msg = 'failed to read number of analogues from config file %s ' % e
logger.debug(msg)
return configfile
def get_viewer_configfile(analogs):
"""
finds or generates configuration file for an analogs file
to be used by the analogs viewer. The configuration file
will be copied into the output folder.
:param analogs: text file containing the analogs values
:return str: configuration file path/name.txt
"""
from flyingpigeon import config
from flyingpigeon.config import www_url
import requests
from shutil import copyfile
from tempfile import mkstemp
from os import path
from os.path import basename
try:
outputUrl_path = config.outputUrl_path()
output_path = config.output_path()
# Config file with path (server URL address)
configfile = analogs.replace('analogs-', 'config-')
logger.info('config filename generated %s' % configfile)
configfile_with_path = path.join(outputUrl_path, configfile)
logger.debug('configfile_with_path: %s' % configfile_with_path)
# Check if config file exists
r = requests.get(configfile_with_path)
if r.status_code != 404:
logger.debug('Config file exists on server URL address.')
else:
logger.debug(
'Config file does not exist on server address. Check local disk.')
# Make config file name and get its path on local disk
configfile = 'config_' + analogs
logger.debug('local disk configfile: %s' % configfile)
p, name = path.split(path.realpath(analogs))
configfile_localAddress = path.join(p, configfile)
logger.debug('local disk configfile_localAddress: %s' %
configfile_localAddress)
# Check if config file exists
if path.isfile(configfile_localAddress):
logger.debug('Config file exists on local disk.')
# Copy config file to output_path
# (~/birdhouse/var/lib/pywps/outputs/flyingpigeon)
configfile_outputlocation = path.join(output_path, configfile)
copyfile(configfile_localAddress, configfile_outputlocation)
logger.info(' time for coffee ')
configfile_outputlocation_edited = config_edits(
configfile_outputlocation)
logger.info('outputlocation_edited: %s' %
configfile_outputlocation_edited)
configfile = path.basename(configfile_outputlocation_edited)
logger.info(' configfile %s ' % configfile)
else:
logger.debug(
'There is no config file on local disk. Generating a default one.')
# Insert analogs filename into config file.
# The rest of the params are unknown.
configfile_wkdir = get_configfile(
files=['dummyconfig', 'dummyconfig', analogs],
nanalog='DUMMY!!!',
varname='DUMMY!!!',
seacyc='DUMMY!!!',
cycsmooth='DUMMY!!!',
timewin='DUMMY!!!',
seasonwin='DUMMY!!!',
distfun='DUMMY!!!',
calccor='DUMMY!!!',
outformat='DUMMY!!!',
silent='DUMMY!!!',
period=['dummy', 'dummy'],
bbox='DUMMY!!!'
)
configfile = path.basename(configfile_wkdir) # just file name
# Add server path to file name
configfile_inplace = path.join(output_path, configfile)
# Copy out of local working dir to output_path
copyfile(configfile_wkdir, configfile_inplace)
except Exception as e:
msg = 'failed to read number of analogues from config file %s ' % e
logger.debug(msg)
return configfile
def reformat_analogs(analogs):
"""
Reformats analogs results file for analogues viewer code.
:param analogs: output from analog_detection process
:return str: reformatted analogs file for analogues viewer
"""
from flyingpigeon import config
import numpy as np
import pandas as pd
try:
num_cols = 3 # dateAnlg, Dis, Corr
# Create dataframe and read in output csv file of analogs process
dfS = pd.DataFrame()
dfS = pd.read_csv(analogs, delimiter=r"\s+", index_col=0)
# Find number of analogues
num_analogues = (dfS.shape[1]) / 3
logger.debug('num_analogues: %s ' % num_analogues)
# Define temporary df
df_anlg = dfS.iloc[:, 0:num_analogues] # store only anlg dates
df_dis = dfS.iloc[:, num_analogues:2 * num_analogues] # store only dis
df_corr = dfS.iloc[:, 2 * num_analogues:3 *
num_analogues] # store only corr
# remove index name before stacking
df_anlg.index.name = ""
df_dis.index.name = ""
df_corr.index.name = ""
dateStack = df_anlg.stack()
disStack = df_dis.stack().abs() # raw values < 0 so take abs
corrStack = df_corr.stack()
# Create df of correct dimensions (n x num_cols) using dfS
df_all = dfS.iloc[:, 0:num_cols] # NB data are placeholders
# Rename cols
df_all.columns = ['dateAnlg', 'Dis', 'Corr']
# Replicate each row 20 times (for dcjs format)
df_all = df_all.loc[np.repeat(df_all.index.values, num_analogues)]
# Replace data placeholders with correct values
df_all['dateAnlg'] = list(dateStack)
df_all['Dis'] = list(disStack)
df_all['Corr'] = list(corrStack)
# Name index col
df_all.index.name = 'dateRef'
# save to tsv file
output_path = config.output_path()
ip, analogs_mod = mkstemp(
suffix='.tsv', prefix='modified-analogfile', dir=output_path, text=False)
df_all.to_csv(analogs_mod, sep='\t')
logger.info('successfully reformatted analog file')
except Exception as e:
msg = 'failed to reformat analog file %s ' % e
logger.debug(msg)
return analogs_mod
def execute(self):
######################
# start execution
######################
from flyingpigeon.config import JSsrc_dir
tmpl = JSsrc_dir() + '/template_analogviewer.html'
#Output csv file of analogs process
analogs = self.getInputValues(identifier='resource')[0]
###########################################
# reorganize analog txt file for javascript
###########################################
from flyingpigeon import config
from tempfile import mkstemp
from flyingpigeon.config import www_url
#my_css_url = www_url() + "/static/css/style.css"
#begin CN
#use as test input file: http://birdhouse-lsce.extra.cea.fr:8090/wpsoutputs/flyingpigeon/output_txt-0797016c-378e-11e6-91dd-41d8cd554993.txt
import numpy as np
import pandas as pd
import collections
import os
try:
num_analogues = 20 #number of analogues searched for
num_cols = 3 #dateAnlg, Dis, Corr
#Create dataframe and read in output csv file of analogs process
dfS = pd.DataFrame()
dfS = pd.read_csv(analogs, delimiter=r"\s+", index_col=0)
#Define temporary df
df_anlg = dfS.iloc[:, 0:20] #store only anlg dates
df_dis = dfS.iloc[:, 20:40] #store only dis
df_corr = dfS.iloc[:, 40:60] #store only corr
#remove index name before stacking
df_anlg.index.name = ""
df_dis.index.name = ""
df_corr.index.name = ""
#Stack (transpose)into single col
dateStack = df_anlg.stack()
disStack = df_dis.stack().abs() #raw values < 0 so take abs
corrStack = df_corr.stack()
# #BUILD NEW DF IN CORRECT FORMAT
#Create df of correct dimensions (n x num_cols) using dfS
df_all = dfS.iloc[:, 0:num_cols] #NB data are placeholders
#Rename cols
df_all.columns = ['dateAnlg', 'Dis', 'Corr']
#Replicate each row 20 times (for dcjs format)
df_all = df_all.loc[np.repeat(df_all.index.values,num_analogues)]
#Replace data placeholders with correct values
df_all['dateAnlg'] = list(dateStack)
df_all['Dis'] = list(disStack)
df_all['Corr'] = list(corrStack)
#Name index col
df_all.index.name = 'dateRef'
# #SAVE TO TSV FILE
output_path = config.output_path()
ip , f = mkstemp(suffix='.tsv', prefix='modified-analogfile', dir=output_path, text=False)
df_all.to_csv(f, sep='\t')
logger.info('successfully reformatted analog file')
self.status.set('successfully reformatted analog file', 90)
except Exception as e:
msg = 'failed to reformat analog file %s ' % e
logger.debug(msg)
# raise Exception(msg)
################################
# modify JS template
################################
from os.path import basename
ip, output_av = mkstemp(suffix='.html', prefix='analogviewer', dir='.', text=False)
#copyfile(tmpl, output_av)
tmpl_file = open(tmpl).read()
out = open(output_av, 'w')
#replacements = {'analogues_placeholder.json':basename(f)}
#for i in replacements.keys():
tmpl_file = tmpl_file.replace('analogues_placeholder.json', basename(f) )
out.write(tmpl_file)
out.close()
# with open(output_av, 'r+') as f:
# content = f.read()
# f.seek(0)
# f.truncate()
# f.write(content.replace('analogues_placeholder.json', basename(f)))
################################
# set the outputs
################################
outputUrl_path = config.outputUrl_path()
output_data = outputUrl_path + '/' + basename(f)
logger.info('Data url: %s ' % output_data)
logger.info('output_av: %s ' % output_av)
self.output_txt.setValue( output_data )
self.output_html.setValue( output_av )