if os.path.isfile(datafilename):
datafile = open(datafilename, 'rb')
else:
continue
thedata = pickle.load(datafile)
simfile = constant.simresultfolder + str(
stid) + '/exptemp_' + stname + '_' + str(t) + '_' + str(
p) + '_' + str(y) + '_' + str(d) + '.txt'
[h, m, temp] = utils.readtempfile(simfile)
sec = 3600 * h + 60 * m
hour = h * 100 + m * (100. / 60.)
thedata.sim = temp
thedata.timesim = hour
day = utils.doytodate(y, d)
an = analyse.Analyse(day, 0, 0, thedata)
simparam = an.getsimparam()
an.correctbaseline(stname)
signalwindow = 4
fitresult = an.fitdata2(signalwindow)
[fithourarray, fitradio] = an.getdataforfit(4)
if (fitresult.success == True):
# fig = plt.figure()
# plt.plot(fithourarray,fitradio)
# plt.plot(fithourarray, fitresult.best_fit, 'r-')
# plt.gca().text(np.mean(fithourarray),np.mean(fitradio), r'$\frac{\chi^2}{ndl} = $' +str(fitresult.redchi), fontsize=15)
# saving part:
outname = constant.datafit2folder + str(
stid) + '/fit2_' + stname + '_' + str(t) + '_' + str(
p) + '_' + str(y) + '_' + str(d) + '.txt'
from flask import Flask, request, render_template
import analyse as an
app = Flask(__name__)
# @app.before_request
# def load_analyser():
processor = an.Analyse()
@app.route("/")
def home():
return render_template("home.html")
@app.route("/about")
def about():
return render_template("about.html")
@app.route('/', methods=['POST'])
def my_form_post():
text = request.form['text']
#processed_text = text.upper()
processed_text = processor.processText(text) # call to Analyse object
return processed_text
if __name__ == "__main__":
app.run(host='0.0.0.0', port='80')
print datafilename
if os.path.isfile(datafilename):
datafile = open(datafilename, 'rb')
else:
continue
thedata = pickle.load(datafile)
simfile = constant.simresultfolder + '/fake/' + '/exptemp_fake' + stname + '_' + str(
delt) + '_' + str(delp) + '_' + str(y) + '_' + str(d) + '.txt'
[h, m, temp] = utils.readtempfile(simfile)
sec = 3600 * h + 60 * m
hour = h * 100 + m * (100. / 60.)
thedata.sim = temp
thedata.timesim = hour
day = utils.doytodate(y, d)
an = analyse.Analyse(day, delt, delp, thedata)
simparam = an.getsimparam()
an.correctbaseline(goodlistname)
signalwindow = 3
[fithours, fittedradio] = an.fitdata(signalwindow, 1)
an.isgoodfit()
######################################
######## check fit of the data #######
if an.goodfit:
print an.fitresult[0]
an.computetsys()
an.geterrorsonfit()
datearray = np.append(datearray, an.day)
det = detector.Detector(temp = tsys, type=dettype)
det.loadspectrum()
scales = np.array([1,10,50,100,200])
iter = [1,2,3,4,5,6,7,8,9,10]
a_mean = np.array([])
a_rms = np.array([])
for scale in scales:
meannr = 0
rmsnr = 0
a_ev = np.array([])
for it in iter:
folder = '/Users/romain/work/Auger/EASIER/IPNcode/script/results/afterelec/' + '/scaling' + str(int(scale)) + '/' + str(it) + '/'
filenames = 'ev_'
names = glob.glob(folder+ filenames+'*.pkl')
ana = analyse.Analyse(det=det)
evcount = 0
a_max = np.array([])
# print names
for n in names[::1]:
file = open(n, 'rb')
revent = pickle.load(file)
if revent.shower.energy < 5:
continue
for ant in revent.antennas:
size = len(ant.trace)
if size==0:
continue
time = np.arange(0,size*binsize,binsize)
wf = waveform.Waveform(time,ant.trace)
# time = ant.maketimearray()
def main_func(pprocess=False,
analyze=False,
meta=False,
classification=False,
func='summary_stat',
type='#anwps_freq',
sex=False,
age=False):
# Do pre_processing task
if (pprocess):
# Read data_set with limited columns
cupid_df = pd.read_csv('../data/raw/profiles.csv',
usecols=[
'education', 'essay0', 'essay1', 'essay2',
'essay3', 'essay7', 'essay8', 'essay9',
'age', 'sex'
])
# Define an object of pre_processing class
cupid = pre_processing.PreProcess()
cupid_df = cupid.missing_value(cupid_df)
cupid_df = cupid.merge_essay(cupid_df)
cupid_df = cupid.remove_tag(cupid_df)
cupid_df = cupid.recode_edcuaction(cupid_df)
cupid_df = cupid.count_words_sentences(cupid_df)
cupid_df = cupid.text_cleaning(cupid_df)
# Save pre_processed dat_set on disk
cupid_df.to_csv(r'../data/processed/preprocessed_cupid.csv',
index=None,
header=True)
# Final message
print(
colored(
'preprocessed_cupid.csv is written in data/preprocessed\
folder...', 'red'))
# ************************************************************************
# Do analyses task
elif (analyze):
# Read pre_processed data_set with limited columns
cupid_dfa = pd.read_csv('../data/processed/preprocessed_cupid.csv',
usecols=[
'education', 'age', 'sex', 'text', 'isced',
'isced2', '#words', '#sentences', '#anwps',
'clean_text'
])
# cupid_dfa.rename(columns={'removed_stopwords': 'clean_text'},
# inplace=True)
# Define an object of pre_processing class
a_cupid = analyse.Analyse()
if (func == 'summary_stat'):
summary_df = a_cupid.summary(cupid_dfa)
summary_df.to_json(r'../results/figures/summary_statistics.json')
summary_df.to_csv(r'../results/figures/summary_statistics.csv')
print(
colored(
'summary_statistics.csv is written in '
'results/figure folder...', 'magenta'))
elif (func == 'plot'):
a_cupid.plot_func(cupid_dfa, type, sex)
# *************************************************************************
# Calculate meta_data
if (meta):
style = meta_data.Stylo()
# Read data_set
df_preprocessed = pd.read_csv(
'../data/processed/preprocessed_cupid.csv',
usecols=[
'age', 'sex', '#anwps', 'clean_text', "text", 'isced', 'isced2'
])
df_preprocessed.dropna(subset=['text', 'isced'], inplace=True)
# Print the progress number
print(colored('\nCalculating count_char:\n', 'green'))
df_preprocessed['count_char'] = df_preprocessed.progress_apply(
lambda x: style.count_char(x['text']), axis=1)
# Print the progress number
print(colored('\nCalculating count_punct:\n', 'green'))
df_preprocessed['count_punct'] = df_preprocessed.progress_apply(
lambda x: style.count_punc(x['text']), axis=1)
# df_preprocessed['count_digit'] = sum(c.isdigit() for c in
# df_preprocessed['text'])
# Print the progress number
print(colored('\nCalculating count_word:\n', 'green'))
df_preprocessed['count_word'] = df_preprocessed.progress_apply(
lambda x: style.count_words(x['text']), axis=1)
# Print the progress number
print(colored('\nCalculating avg_wordlength:\n', 'green'))
df_preprocessed['avg_wordlength'] = round(
df_preprocessed['count_char'] / df_preprocessed['count_word'], 2)
# Print the progress number
print(colored('\nCalculating count_misspelled:\n', 'green'))
df_preprocessed['count_misspelled'] = \
df_preprocessed.progress_apply(lambda x:
style.count_spellerror(x[
'text']), axis=1)
# df_preprocessed['readability'] = df_preprocessed.progress_apply(
# lambda x: style.text_readability(x['text']), axis=1)
# Print the progress number
print(colored('\nCalculating words uniqueness:\n', 'green'))
df_preprocessed['word_uniqueness'] = df_preprocessed.progress_apply(
lambda x: style.uniqueness(x['text']), axis=1)
# Save calculated meta_data on disk
df_preprocessed.to_csv(r'../data/processed/stylo_cupid_test.csv',
index=None,
header=True)
# Final message
print(
colored(
'stylo_cupid.csv is written in data/preprocessed\
folder...', 'red'))
# **************************************************************************
if (classification):
cls = classify.Classifier()
df_cls = pd.read_csv(r'../data/processed/stylo_cupid2.csv')
cls.logistic_text_meta(df_cls)
else:
continue
listofangle.append((delt,delp))
popt = fitresult[0]
#data name
simtime = np.linspace(0,24,500)
sim = utils.expofunc0(simtime,float(ls[2]),float(ls[3]),float(ls[4]),float(ls[5]))
fitresult = datadict[(y,d)]
popt = fitresult[0]
tofmax_data = popt[2]
data = utils.expofunc2(simtime,popt[0],popt[1],tofmax_data-3,popt[3],popt[4],popt[5])
data = data - (popt[3]*simtime**2 + popt[4]*simtime + popt[5])
datapoint = np.interp(float(ls[4]),simtime,data)
if datapoint < 0.0001:
datapoint =0.001
theres = analyse.Analyse()
theres.fitresult = fitresult
theres.computetsys(float(ls[2]),datapoint)
theres.geterrorsonfit2(float(ls[4]))
# plt.plot(simtime,sim)
# plt.plot(simtime,data,'--')
# plt.show()
day = utils.doytodate(y,d)
a_tsys = np.append(a_tsys,theres.tsys)
# print ' theres.tsys = ', theres.tsys
a_errtsys = np.append(a_errtsys,theres.errortsys)
a_simtofmax = np.append(a_simtofmax,float(ls[4]))
a_tofmax = np.append(a_tofmax,tofmax_data -3)
a_errortofmax = np.append(a_errortofmax,theres.errortofmax)