if not np.isclose(noiseStrengthTM[-1], float(noise), rtol=0.1): print "Warning: Estimated noise strength is different from the given noise" groundTruth = np.roll(data['value'], -5) tmTruth = np.roll(predData_TM['value'], -5) predDataTMFiveStep = np.array(predData_TM['prediction5']) nrmse = computeNRMSE(tmTruth, predDataTMFiveStep, startFrom) altMAPE = computeAltMAPE(tmTruth, predDataTMFiveStep, startFrom) mapeGroundTruth = computeAltMAPE(tmTruth, groundTruth, startFrom) tm_prediction = np.load('./result/' + dataSet + 'TMprediction.npy') tmTruth = np.load('./result/' + dataSet + 'TMtruth.npy') negLL = computeLikelihood(tm_prediction, tmTruth, encoder) negLL = np.nanmean(negLL[startFrom:]) nrmseTM = pd.concat([nrmseTM, pd.DataFrame([nrmse], columns=['TM'])]) mapeTM = pd.concat([ mapeTM, pd.DataFrame(np.reshape(np.array([altMAPE, mapeGroundTruth]), newshape=(1, 2)), columns=['TM', 'GT']) ]) negLLTM = pd.concat([negLLTM, pd.DataFrame([negLL], columns=['TM'])]) lstmExptDir = 'results/nyc_taxi_experiment_continuous_likelihood_noise/' noiseList = ['0.0', '0.020', '0.040', '0.060', '0.080', '0.10']
from plot import computeLikelihood, plotAccuracy bucketIndex2 = [] negLL = [] minProb = 0.0001 for i in xrange(len(truth)): bucketIndex2.append(np.where(encoder.encode(truth[i]))[0]) outOfBucketProb = 1 - sum(predictions[i, :]) prob = predictions[i, bucketIndex2[i]] if prob == 0: prob = outOfBucketProb if prob < minProb: prob = minProb negLL.append(-np.log(prob)) negLL = computeLikelihood(predictions, truth, encoder) negLL[:5000] = np.nan x = range(len(negLL)) plt.figure() plotAccuracy((negLL, x), truth, window=480, errorType='negLL') np.save('./result/' + dataSet + classifierType + 'TMprediction.npy', predictions) np.save('./result/' + dataSet + classifierType + 'TMtruth.npy', truth) plt.figure() activeCellNumAvg = movingAverage(activeCellNum, 100) plt.plot(np.array(activeCellNumAvg) / tm.numberOfCells()) plt.xlabel('data records') plt.ylabel('sparsity') plt.xlim([0, 5000])
MAPECLA = MAPE(actualData[nTrain:nTrain + nTest], shiftedPredDataCLA[nTrain:nTrain + nTest]) MAPENN = MAPE(actualData[nTrain:nTrain + nTest], shiftedPredDataNN[nTrain:nTrain + nTest]) print "NRMSE on test data, CLA: ", NRMSECLA print "NRMSE on test data, NN: ", NRMSENN # calculate neg-likelihood encoder = NupicScalarEncoder(w=1, minval=0, maxval=40000, n=22, forced=True) truth = np.roll(actualData, -5) predictions = np.transpose(likelihoodsVecAll) negLLCLA = computeLikelihood(predictions, truth, encoder) negLLCLA[:5904] = np.nan predictions = np.transpose(likelihoodsVecAllNN) negLLNN = computeLikelihood(predictions, truth, encoder) negLLNN[:5904] = np.nan # save predicted distribution for likelihood calculation np.save('./result/' + datasetName + 'TMprediction.npy', predictions) np.save('./result/' + datasetName + 'TMtruth.npy', truth) plt.figure() plotAccuracy((negLLCLA, range(len(negLLCLA))), truth, window=480, errorType='negLL') plotAccuracy((negLLNN, range(len(negLLNN))), truth, window=480, errorType='negLL')
window, xaxis=xaxisDate, label='continuous LSTM-3000') (negLLLSTM6000, expResultLSTM6000negLL) = \ plotLSTMresult('results/nyc_taxi_experiment_continuous_likelihood/learning_window6001.0/', window, xaxis=xaxisDate, label='continuous LSTM-6000') dataSet = 'nyc_taxi' tmPredictionLL = np.load('./result/' + dataSet + 'TMprediction.npy') tmTruth = np.load('./result/' + dataSet + 'TMtruth.npy') encoder = NupicScalarEncoder(w=1, minval=0, maxval=40000, n=22, forced=True) negLL = computeLikelihood(tmPredictionLL, tmTruth, encoder) negLL[:skipTrain] = None negLLTM = plotAccuracy((negLL, xaxisDate), tmTruth, window=window, errorType='negLL', label='TM') plt.legend() plt.savefig(figPath + 'continuous_likelihood.pdf') startFrom = skipTrain altMAPELSTM6000 = computeAltMAPE(expResultLSTM6000.truth, expResultLSTM6000.predictions, startFrom) altMAPELSTM3000 = computeAltMAPE(expResultLSTM3000.truth, expResultLSTM3000.predictions, startFrom) altMAPELSTM1000 = computeAltMAPE(expResultLSTM1000.truth,
truth = np.roll(actual_data, -5) from nupic.encoders.scalar import ScalarEncoder as NupicScalarEncoder encoder = NupicScalarEncoder(w=1, minval=0, maxval=40000, n=22, forced=True) from plot import computeLikelihood, plotAccuracy bucketIndex2 = [] negLL = [] minProb = 0.0001 for i in xrange(len(truth)): bucketIndex2.append(np.where(encoder.encode(truth[i]))[0]) outOfBucketProb = 1 - sum(predictions[i,:]) prob = predictions[i, bucketIndex2[i]] if prob == 0: prob = outOfBucketProb if prob < minProb: prob = minProb negLL.append( -np.log(prob)) negLL = computeLikelihood(predictions, truth, encoder) negLL[:5000] = np.nan x = range(len(negLL)) plt.figure() plotAccuracy((negLL, x), truth, window=480, errorType='negLL') np.save('./result/'+dataSet+classifierType+'TMprediction.npy', predictions) np.save('./result/'+dataSet+classifierType+'TMtruth.npy', truth)
predDataTMfiveStep = np.array(predDataTM['prediction5']) iteration = predDataTM.index tmPredPerturbTruth = np.roll(predDataTM['value'], -5) tmPredPerturb = np.array(predDataTM['prediction5']) filePath = './prediction/' + dataSet + '_esn_pred.csv' predDataESN = pd.read_csv(filePath, header=0, skiprows=[1, 2], names=['step', 'value', 'prediction5']) esnPredPerturbTruth = np.roll(predDataESN['value'], -5) esnPredPerturb = np.array(predDataESN['prediction5']) negLLTMPerturb = computeLikelihood(tmPredictionPerturb, tmTruthPerturb, encoder) negLLTMPerturb[:6000] = None nrmseTMPerturb = computeSquareDeviation(tmPredPerturb, tmPredPerturbTruth) mapeTMPerturb = np.abs(tmPredPerturb - tmPredPerturbTruth) mapeESNPerturb = np.abs(esnPredPerturb - esnPredPerturbTruth) expResultPerturb1000 = ExperimentResult( 'results/nyc_taxi_experiment_continuous_perturb/learning_window' + str(1001.0) + '/') expResultPerturb3000 = ExperimentResult( 'results/nyc_taxi_experiment_continuous_perturb/learning_window' + str(3001.0) + '/') expResultPerturb6000 = ExperimentResult( 'results/nyc_taxi_experiment_continuous_perturb/learning_window' +
window, xaxis=xaxisDate, label='continuous LSTM-3000') (negLLLSTM6000, expResultLSTM6000negLL) = plotLSTMresult( 'results/nyc_taxi_experiment_continuous_likelihood/learning_window6001.0/', window, xaxis=xaxisDate, label='continuous LSTM-6000') (negLLLSTMOnline, expResultLSTMOnlinenegLL) = plotLSTMresult( 'results/nyc_taxi_experiment_continuous_likelihood_online/learning_window100.0/', window, xaxis=xaxisDate, label='continuous LSTM-6000') dataSet = 'nyc_taxi' tmPredictionLL = np.load('./result/'+dataSet+'TMprediction.npy') tmTruth = np.load('./result/' + dataSet + 'TMtruth.npy') encoder = NupicScalarEncoder(w=1, minval=0, maxval=40000, n=22, forced=True) negLL = computeLikelihood(tmPredictionLL, tmTruth, encoder) negLL[:skipTrain] = None negLLTM = plotAccuracy((negLL, xaxisDate), tmTruth, window=window, errorType='negLL', label='TM') plt.legend() plt.savefig(figPath + 'continuous_likelihood.pdf') startFrom = skipTrain altMAPELSTM6000 = computeAltMAPE(expResultLSTM6000.truth, expResultLSTM6000.predictions, startFrom) altMAPELSTM3000 = computeAltMAPE(expResultLSTM3000.truth, expResultLSTM3000.predictions, startFrom) altMAPELSTM1000 = computeAltMAPE(expResultLSTM1000.truth, expResultLSTM1000.predictions, startFrom) altMAPELSTMonline = computeAltMAPE(expResultLSTMonline.truth, expResultLSTMonline.predictions, startFrom) altMAPETM = computeAltMAPE(tmTruth, tmPrediction, startFrom) altMAPEARIMA = computeAltMAPE(arimaTruth, arimaPrediction, startFrom) altMAPEESN = computeAltMAPE(esnTruth, esnPrediction, startFrom)
predData_TM_five_step = np.array(predData_TM['prediction5']) iteration = predData_TM.index tm_pred_perturb_truth = np.roll(predData_TM['value'], -5) tm_pred_perturb = np.array(predData_TM['prediction5']) filePath = './prediction/' + dataSet + '_esn_pred.csv' predDataESN = pd.read_csv(filePath, header=0, skiprows=[1, 2], names=['step', 'value', 'prediction5']) esnPredPerturbTruth = np.roll(predDataESN['value'], -5) esnPredPerturb = np.array(predDataESN['prediction5']) negLL_tm_perturb = computeLikelihood(tm_prediction_perturb, tm_truth_perturb, encoder) negLL_tm_perturb[:6000] = None nrmse_tm_perturb = computeSquareDeviation(tm_pred_perturb, tm_pred_perturb_truth) mape_tm_perturb = np.abs(tm_pred_perturb - tm_pred_perturb_truth) mape_esn_perturb = np.abs(esnPredPerturb - esnPredPerturbTruth) plt.figure() plotAccuracy((negLL_LSTM3000_perturb, xaxis_datetime), truth_LSTM3000_perturb, window=window, errorType='negLL', label='LSTM3000', train=expResult_perturb.train) # plotAccuracy((negLL_LSTM3000_perturb_baseline, xaxis_datetime), truth_LSTM3000_perturb, window=window, errorType='negLL', label='TM')
window, xaxis=xaxis_datetime, label='continuous LSTM-1000') (negLL_LSTM3000, expResult_LSTM3000_negLL) = \ plotLSTMresult('results/nyc_taxi_experiment_continuous_likelihood/learning_window3001.0/', window, xaxis=xaxis_datetime, label='continuous LSTM-3000') (negLL_LSTM6000, expResult_LSTM6000_negLL) = \ plotLSTMresult('results/nyc_taxi_experiment_continuous_likelihood/learning_window6001.0/', window, xaxis=xaxis_datetime, label='continuous LSTM-6000') dataSet = 'nyc_taxi' tm_prediction = np.load('./result/'+dataSet+'TMprediction.npy') tm_truth = np.load('./result/'+dataSet+'TMtruth.npy') from nupic.encoders.scalar import ScalarEncoder as NupicScalarEncoder encoder = NupicScalarEncoder(w=1, minval=0, maxval=40000, n=22, forced=True) negLL = computeLikelihood(tm_prediction, tm_truth, encoder) negLL[:6000] = None negLL_TM = \ plotAccuracy((negLL, xaxis_datetime), tm_truth, window=window, errorType='negLL', label='TM') plt.legend() plt.savefig(figPath + 'continuous_likelihood.pdf') ### Figure 4: Continuous LSTM with different window size using the likelihood metric fig = plt.figure() # negLL_StaticLSTM = \ # plotLSTMresult('results/nyc_taxi_experiment_one_shot_likelihood/', # window, xaxis=xaxis_datetime, label='static LSTM ') plt.clf()
predDataTMfiveStep = np.array(predDataTM['prediction5']) iteration = predDataTM.index tmPredPerturbTruth = np.roll(predDataTM['value'], -5) tmPredPerturb = np.array(predDataTM['prediction5']) filePath = './prediction/' + dataSet + '_esn_pred.csv' predDataESN = pd.read_csv(filePath, header=0, skiprows=[1, 2], names=['step', 'value', 'prediction5']) esnPredPerturbTruth = np.roll(predDataESN['value'], -5) esnPredPerturb = np.array(predDataESN['prediction5']) negLLTMPerturb = computeLikelihood(tmPredictionPerturb, tmTruthPerturb, encoder) negLLTMPerturb[:6000] = None nrmseTMPerturb = computeSquareDeviation(tmPredPerturb, tmPredPerturbTruth) mapeTMPerturb = np.abs(tmPredPerturb - tmPredPerturbTruth) mapeESNPerturb = np.abs(esnPredPerturb - esnPredPerturbTruth) expResultPerturb1000 = ExperimentResult( 'results/nyc_taxi_experiment_continuous_perturb/learning_window'+str(1001.0)+'/') expResultPerturb3000 = ExperimentResult( 'results/nyc_taxi_experiment_continuous_perturb/learning_window'+str(3001.0)+'/') expResultPerturb6000 = ExperimentResult( 'results/nyc_taxi_experiment_continuous_perturb/learning_window'+str(6001.0)+'/')
dataSet = 'nyc_taxi_perturb' tm_prediction_perturb = np.load('./result/'+dataSet+'TMprediction.npy') tm_truth_perturb = np.load('./result/'+dataSet+'TMtruth.npy') from nupic.encoders.scalar import ScalarEncoder as NupicScalarEncoder encoder = NupicScalarEncoder(w=1, minval=0, maxval=40000, n=22, forced=True) filePath = './prediction/' + dataSet + '_TM_pred.csv' predData_TM = pd.read_csv(filePath, header=0, skiprows=[1, 2], names=['step', 'value', 'prediction5']) predData_TM_five_step = np.array(predData_TM['prediction5']) iteration = predData_TM.index tm_pred_perturb_truth = np.roll(predData_TM['value'], -5) tm_pred_perturb = np.array(predData_TM['prediction5']) negLL_tm_perturb = computeLikelihood(tm_prediction_perturb, tm_truth_perturb, encoder) negLL_tm_perturb[:6000] = None nrmse_tm_perturb = computeSquareDeviation(tm_pred_perturb, tm_pred_perturb_truth) mape_tm_perturb = np.abs(tm_pred_perturb - tm_pred_perturb_truth) plt.figure() window = 480 plotAccuracy((negLL_LSTM3000_perturb, xaxis_datetime), truth_LSTM3000_perturb, window=window, errorType='negLL', label='LSTM3000', train=expResult_perturb.train) # plotAccuracy((negLL_LSTM3000_perturb_baseline, xaxis_datetime), truth_LSTM3000_perturb, window=window, errorType='negLL', label='TM') plotAccuracy((negLL_LSTM6000_perturb, xaxis_datetime), truth_LSTM6000_perturb, window=window, errorType='negLL', label='LSTM6000') # plotAccuracy((negLL_LSTM6000_perturb_baseline, xaxis_datetime), truth_LSTM3000_perturb, window=window, errorType='negLL', label='TM') plotAccuracy((negLL_tm_perturb, xaxis_datetime), tm_truth_perturb, window=window, errorType='negLL', label='TM') plt.axvline(xaxis_datetime[13152], color='black', linestyle='--')