def getModel(data): numInputs = len(data[0][0]) numOutputs = len(data[0][1]) model = Model(numInputs, numOutputs) for input in range(numInputs): #extract fourier bin from training data x = map(lambda x: x[0][input], data) for output in range(numOutputs): #extract outputs to train against y = map(lambda x: x[1][output], data) #calculate pearson score between input and output p = stats.pearson(x, y) model.pearsons[input][output] = p #get the slope and y-int of the line of best fit between in put and output (slope, yInt) = fittingFunction(x, y) model.slopes[input][output] = slope model.yInts[input][output] = yInt return model
def alphasense_compute(dataFrame, t_incl=False, h_incl=False):
# lambdas for columns
col_skip = 3
if t_incl and h_incl:
col_temp = (lambda i: (col_skip + 6 * i))
col_hum = (lambda i: (col_skip + 6 * i + 1))
col_no2 = (lambda i: range((col_skip + 6 * i + 2),
(col_skip + 6 * i + 4)))
col_ox = (lambda i: range((col_skip + 6 * i + 4),
(col_skip + 6 * i + 6)))
elif h_incl:
col_hum = (lambda i: (col_skip + 5 * i))
col_no2 = (lambda i: range((col_skip + 5 * i + 1),
(col_skip + 5 * i + 3)))
col_ox = (lambda i: range((col_skip + 5 * i + 3),
(col_skip + 5 * i + 5)))
elif t_incl:
col_temp = (lambda i: (col_skip + 5 * i))
col_no2 = (lambda i: range((col_skip + 5 * i + 1),
(col_skip + 5 * i + 3)))
col_ox = (lambda i: range((col_skip + 5 * i + 3),
(col_skip + 5 * i + 5)))
else:
col_no2 = (lambda i: range((col_skip + 4 * i), (col_skip + 4 * i + 2)))
col_ox = (lambda i: range((col_skip + 4 * i + 2),
(col_skip + 4 * i + 4)))
#dataFrame = dataFrame.values
err_no2 = np.zeros([len(NO2_WE_0T), 4, 5])
err_o3 = np.zeros([len(NO2_WE_0T), 4, 5])
# iterate over sensors
for i in xrange(np.size(dataFrame, 1)):
if col_ox(i)[-1] >= np.size(dataFrame, 1):
break
nx = dataFrame[:, col_no2(i)]
ox = dataFrame[:, col_ox(i)]
# formula 1
pred_no2 = formula1(nx[:, 0], nx[:, 1], NO2_WE_0E[i], NO2_AE_0E[i],
NTn, SENSITIVITY_NO2[i])
err_no2[i, 0, 0] = stats.mae(dataFrame[:, 1], pred_no2)
err_no2[i, 0, 1] = stats.rmse(dataFrame[:, 1], pred_no2)
err_no2[i, 0, 2] = stats.mape(dataFrame[:, 1], pred_no2)
err_no2[i, 0, 3] = stats.coeff_deter(dataFrame[:, 1], pred_no2)
err_no2[i, 0, 4] = stats.pearson(dataFrame[:, 1], pred_no2)
#print np.mean(pred_no2)
#print np.std(pred_no2)
pred = formula1(ox[:, 0], ox[:, 1], O3_WE_0E[i], O3_AE_0E[i], NTo,
SENSITIVITY_O3[i])
err_o3[i, 0, 0] = stats.mae(dataFrame[:, 2], pred - pred_no2)
err_o3[i, 0, 1] = stats.rmse(dataFrame[:, 2], pred - pred_no2)
err_o3[i, 0, 2] = stats.mape(dataFrame[:, 2], pred - pred_no2)
err_o3[i, 0, 3] = stats.coeff_deter(dataFrame[:, 2], pred - pred_no2)
err_o3[i, 0, 4] = stats.pearson(dataFrame[:, 2], pred - pred_no2)
#print np.mean(pred)
#print np.std(pred)
# formula 2
pred_no2 = formula2(nx[:, 0], nx[:, 1], NO2_WE_0E[i], NO2_AE_0E[i],
NO2_WE_0T[i], NO2_AE_0T[i], KTn,
SENSITIVITY_NO2[i])
err_no2[i, 1, 0] = stats.mae(dataFrame[:, 1], pred)
err_no2[i, 1, 1] = stats.rmse(dataFrame[:, 1], pred)
err_no2[i, 1, 2] = stats.mape(dataFrame[:, 1], pred)
err_no2[i, 1, 3] = stats.coeff_deter(dataFrame[:, 1], pred)
err_no2[i, 1, 4] = stats.pearson(dataFrame[:, 1], pred)
#print np.mean(pred_no2)
#print np.std(pred_no2)
pred = formula2(ox[:, 0], ox[:, 1], O3_WE_0E[i], O3_AE_0E[i],
O3_WE_0T[i], O3_AE_0T[i], KTo, SENSITIVITY_O3[i])
err_o3[i, 1, 0] = stats.mae(dataFrame[:, 2], pred - pred_no2)
err_o3[i, 1, 1] = stats.rmse(dataFrame[:, 2], pred - pred_no2)
err_o3[i, 1, 2] = stats.mape(dataFrame[:, 2], pred - pred_no2)
err_o3[i, 1, 3] = stats.coeff_deter(dataFrame[:, 2], pred - pred_no2)
err_o3[i, 1, 4] = stats.pearson(dataFrame[:, 2], pred - pred_no2)
#print np.mean(pred)
#print np.std(pred)
# formula 3
pred_no2 = formula3(nx[:, 0], nx[:, 1], NO2_WE_0E[i], NO2_AE_0E[i],
NO2_WE_0T[i], NO2_AE_0T[i], K_Tn,
SENSITIVITY_NO2[i])
err_no2[i, 2, 0] = stats.mae(dataFrame[:, 1], pred_no2)
err_no2[i, 2, 1] = stats.rmse(dataFrame[:, 1], pred_no2)
err_no2[i, 2, 2] = stats.mape(dataFrame[:, 1], pred_no2)
err_no2[i, 2, 3] = stats.coeff_deter(dataFrame[:, 1], pred_no2)
err_no2[i, 2, 4] = stats.pearson(dataFrame[:, 1], pred_no2)
#print np.mean(pred_no2)
#print np.std(pred_no2)
pred = formula3(ox[:, 0], ox[:, 1], O3_WE_0E[i], O3_AE_0E[i],
O3_WE_0T[i], O3_AE_0T[i], K_To, SENSITIVITY_O3[i])
err_o3[i, 2, 0] = stats.mae(dataFrame[:, 2], pred - pred_no2)
err_o3[i, 2, 1] = stats.rmse(dataFrame[:, 2], pred - pred_no2)
err_o3[i, 2, 2] = stats.mape(dataFrame[:, 2], pred - pred_no2)
err_o3[i, 2, 3] = stats.coeff_deter(dataFrame[:, 2], pred - pred_no2)
err_o3[i, 2, 4] = stats.pearson(dataFrame[:, 2], pred - pred_no2)
#print np.mean(pred)
#print np.std(pred)
# formula 4
pred_no2 = formula4(nx[:, 0], nx[:, 1], NO2_WE_0E[i], NO2_AE_0E[i],
NO2_WE_0T[i], NO2_AE_0T[i], K__Tn,
SENSITIVITY_NO2[i])
err_no2[i, 3, 0] = stats.mae(dataFrame[:, 1], pred_no2)
err_no2[i, 3, 1] = stats.rmse(dataFrame[:, 1], pred_no2)
err_no2[i, 3, 2] = stats.mape(dataFrame[:, 1], pred_no2)
err_no2[i, 3, 3] = stats.coeff_deter(dataFrame[:, 1], pred_no2)
err_no2[i, 3, 4] = stats.pearson(dataFrame[:, 1], pred_no2)
#print np.mean(pred_no2)
#print np.std(pred_no2)
pred = formula4(ox[:, 0], ox[:, 1], O3_WE_0E[i], O3_AE_0E[i],
O3_WE_0T[i], O3_AE_0T[i], K__To, SENSITIVITY_O3[i])
err_o3[i, 3, 0] = stats.mae(dataFrame[:, 2], pred - pred_no2)
err_o3[i, 3, 1] = stats.rmse(dataFrame[:, 2], pred - pred_no2)
err_o3[i, 3, 2] = stats.mape(dataFrame[:, 2], pred - pred_no2)
err_o3[i, 3, 3] = stats.coeff_deter(dataFrame[:, 2], pred - pred_no2)
err_o3[i, 3, 4] = stats.pearson(dataFrame[:, 2], pred - pred_no2)
#print np.mean(pred)
#print np.std(pred)
#np.savetxt("alpha-no2-err1.csv", err_no2[0].T, fmt='%0.4g', delimiter=',')
#np.savetxt("alpha-no2-err2.csv", err_no2[1].T, fmt='%0.4g', delimiter=',')
#np.savetxt("alpha-o3-err1.csv", err_o3[0].T, fmt='%0.4g', delimiter=',')
#np.savetxt("alpha-o3-err2.csv", err_o3[1].T, fmt='%0.4g', delimiter=',')
#print err_no2
#print err_o3
return err_no2, err_o3
(stats.peak_snr(post1, post2)))
if opts.structural_similarity:
if opts.Verbose:
print " structural_similarity"
messages.append(
"structural similarity : %.5f" % stats.structural_similarity(
post1,
post2,
c1=0.01 * np.min(np.mean(post1), np.mean(post2)),
c2=0.01 * np.min(np.var(post1), np.var(post2))))
if opts.pearson:
if opts.Verbose:
print " pearson"
messages.append("pearson : %.5f" % stats.pearson(post1, post2))
if opts.dot:
if opts.Verbose:
print " dot"
messages.append("dot : %.5f" % stats.dot(post1, post2))
if opts.spotcheck:
if opts.Verbose:
print " spotcheck"
p12, p21 = stats.spotcheck(post1, post2, opts.spotcheck)
for conf, a, b in zip(opts.spotcheck, p12, p21):
messages.append("spotcheck %.3f %s: (%.5f, %.5f)" %
(conf * 100, "%", a, b))
if opts.Verbose:
def pearsonDistance(lx, ly): return stats.pearson(lx, ly)