def analysis(inputfn):
"""
note some of the following code will be redundant once datazilla is up and running
"""
# read in data
datadict = dataproc(inputfn)
# store push ids
pushes = datadict.keys()
pushes.sort()
# store list of pages
pages = datadict[pushes[0]].keys()
# set starting value for the trend
trend = {}
for i in range(0, len(pages)):
trend[pages[i]] = {
"mean": mean(datadict[pushes[0]].values()[i]),
"stddev": std(datadict[pushes[0]].values()[i],ddof=1),
"n": float(len(datadict[pushes[0]].values()[i]))
}
# delete first push as no parent data
del pushes[0]
"""
ok now the data is prepared we can loop through the pushes and compare each new push to the current trend line. If a page is rejected we do not include it in the trend line calculation
"""
results = {}
counts = []
for push in pushes:
pvalues = {}
x1summary = {}
for page in pages:
x1 = datadict[push][page]
x1summary[page] = {
"n": float(len(x1)),
"stddev": std(x1,ddof=1),
"mean": mean(x1)
}
# calculate pvalue
pvalues[page] = welchs_ttest_internal(x1summary[page]["n"],x1summary[page]["stddev"],x1summary[page]["mean"],trend[page]["n"],trend[page]["stddev"],trend[page]["mean"])
# do fdr
output = rejector(pvalues.values(), 0.05)
# store count
results[push] = {}
results[push]["count"] = output["count"]
counts.append(output["count"])
# store other results and update trend
for i in range(0, len(pages)):
# store individual page results
results[push][pages[i]] = output["status"][i]
# update trend
if(results[push][pages[i]] == False):
trend[pages[i]] = exp_smooth(x1summary[pages[i]]["n"],x1summary[pages[i]]["stddev"],x1summary[pages[i]]["mean"],trend[pages[i]]["n"],trend[pages[i]]["stddev"],trend[pages[i]]["mean"])
# write output
datafn = inputfn + '_python_results.csv'
try:
os.remove(datafn)
except:
pass
processWriter = csv.writer(open(datafn,'wb'),delimiter=',',quotechar='"',quoting=csv.QUOTE_MINIMAL)
processWriter.writerow(pushes)
processWriter.writerow(counts)
return results
if i % 10 == 0:
result = sess.run(merged, feed_dict={x: x_train, y: y_train})
writer.add_summary(result, i)
############模型训练--第二层#################
for i in range(1400):
sess.run(Train_op2, feed_dict={x: x_train, y: y_train})
if i % 10 == 0:
result = sess.run(merged, feed_dict={x: x_train, y: y_train})
writer.add_summary(result, 300 + i)
###############单次测试##################
x_singleTest = np.zeros((1, 1, dataLength, 1))
y_singleTest = np.zeros((1, 1, dataLength, 1))
y_singleTest[0, 0, :, 0] = np.sin(2 * 3.14 * 0.015 *
np.linspace(0, dataLength, dataLength))
noise = np.random.normal(loc=0, scale=noise_amp, size=(dataLength))
x_singleTest[0, 0, :, 0] = y_singleTest[0, 0, :, 0] + noise
with open('F:/Files/Program/py/TensorFlow/Model2/ori.txt', 'w') as f:
f.write(str(np.reshape(y_singleTest, dataLength)))
print('Origin data:\n', np.reshape(y_singleTest, dataLength))
with open('F:/Files/Program/py/TensorFlow/Model2/detor.txt', 'w') as f:
f.write(str(np.reshape(x_singleTest, dataLength)))
print('Detor data:\n', np.reshape(x_singleTest, dataLength))
flattenResult = tf.reshape(x_wave2, [-1])
with open('F:/Files/Program/py/TensorFlow/Model2/calc.txt', 'w') as f:
f.write(str(sess.run(flattenResult, feed_dict={x: x_singleTest})))
print('calc result:\n', sess.run(flattenResult, feed_dict={x: x_singleTest}))
dataproc.dataproc()